Skip to content

Commit e4047c4

Browse files
robmoffatrobmoffat
committed
Added verification
1 parent 4e27efe commit e4047c4

File tree

7 files changed

+286
-79
lines changed

7 files changed

+286
-79
lines changed

‎.github/workflows/validate-gemara.yml‎

Lines changed: 2 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,7 @@ name: Validate Gemara Front Matter
22

33
on:
44
push:
5-
branches: [main]
6-
paths:
7-
- 'risks/**/*.md'
8-
- 'practices/**/*.md'
9-
- 'capabilities/**/*.md'
10-
- 'cue/**/*.cue'
115
pull_request:
12-
branches: [main]
13-
paths:
14-
- 'risks/**/*.md'
15-
- 'practices/**/*.md'
16-
- 'capabilities/**/*.md'
17-
- 'cue/**/*.cue'
186

197
jobs:
208
validate:
@@ -33,57 +21,5 @@ jobs:
3321
sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
3422
sudo chmod +x /usr/local/bin/yq
3523
36-
- name: Validate risk files
37-
run: |
38-
echo "Validating risk files against #Threat schema..."
39-
for file in risks/*.md; do
40-
if [ -f "$file" ]; then
41-
echo "Checking $file"
42-
# Extract gemara YAML from front matter
43-
gemara_yaml=$(sed -n '/^---$/,/^---$/p' "$file" | yq '.gemara' -o json 2>/dev/null)
44-
if [ "$gemara_yaml" != "null" ] && [ -n "$gemara_yaml" ]; then
45-
echo "$gemara_yaml" > /tmp/threat.json
46-
cue vet /tmp/threat.json cue/gemara/layer-2.cue -d '#Threat'
47-
echo "✓ $file validated"
48-
else
49-
echo "⚠ $file has no gemara front matter, skipping"
50-
fi
51-
fi
52-
done
53-
54-
- name: Validate practice files
55-
run: |
56-
echo "Validating practice files against #Control schema..."
57-
for file in practices/*.md; do
58-
if [ -f "$file" ]; then
59-
echo "Checking $file"
60-
gemara_yaml=$(sed -n '/^---$/,/^---$/p' "$file" | yq '.gemara' -o json 2>/dev/null)
61-
if [ "$gemara_yaml" != "null" ] && [ -n "$gemara_yaml" ]; then
62-
echo "$gemara_yaml" > /tmp/control.json
63-
cue vet /tmp/control.json cue/gemara/layer-2.cue -d '#Control'
64-
echo "✓ $file validated"
65-
else
66-
echo "⚠ $file has no gemara front matter, skipping"
67-
fi
68-
fi
69-
done
70-
71-
- name: Validate capability files
72-
run: |
73-
echo "Validating capability files against #Capability schema..."
74-
for file in capabilities/*.md; do
75-
if [ -f "$file" ]; then
76-
echo "Checking $file"
77-
gemara_yaml=$(sed -n '/^---$/,/^---$/p' "$file" | yq '.gemara' -o json 2>/dev/null)
78-
if [ "$gemara_yaml" != "null" ] && [ -n "$gemara_yaml" ]; then
79-
echo "$gemara_yaml" > /tmp/capability.json
80-
cue vet /tmp/capability.json cue/gemara/layer-2.cue -d '#Capability'
81-
echo "✓ $file validated"
82-
else
83-
echo "⚠ $file has no gemara front matter, skipping"
84-
fi
85-
fi
86-
done
87-
88-
- name: Validation complete
89-
run: echo "All Gemara front matter validated successfully!"
24+
- name: Validate Gemara front matter
25+
run: ./scripts/validate.sh

‎cue/gemara/base.cue‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Schema lifecycle: experimental | stable | deprecated
2+
@status("stable")
3+
package gemara
4+
5+
import "time"
6+
7+
@go(gemara)
8+
9+
// Contact is the contact information for a person or group
10+
#Contact: {
11+
// name is the preferred descriptor for the contact entity
12+
name: string
13+
14+
// affiliation is the organization with which the contact entity is associated, such as a team, school, or employer
15+
affiliation?: string @go(Affiliation,type=*string)
16+
17+
// email is the preferred email address to reach the contact
18+
email?: #Email @go(Email,type=*Email)
19+
20+
// social is a social media handle or other profile for the contact, such as GitHub
21+
social?: string @go(Social,type=*string)
22+
}
23+
24+
// Actor represents an entity (human or tool) that can perform actions in evaluations
25+
#Actor: {
26+
// id uniquely identifies the actor and allows this entry to be referenced by other elements
27+
id: string
28+
29+
// name is the name of the actor
30+
name: string
31+
32+
// type specifies the type of entity interacting in the workflow
33+
type: #ActorType @go(Type)
34+
35+
// version is the version of the actor (for tools; if applicable)
36+
version?: string
37+
38+
// description provides additional context about the actor
39+
description?: string
40+
41+
// uri is a general URI for the actor information
42+
uri?: =~"^https?://[^\\s]+$"
43+
44+
// contact is contact information for the actor
45+
contact?: #Contact @go(Contact)
46+
}
47+
48+
// ActorType specifies what entity is interacting in the workflow
49+
#ActorType: "Human" | "Software" | "Software-Assisted" @go(-)
50+
51+
// Email represents a validated email address pattern
52+
#Email: =~"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$"
53+
54+
// Datetime represents an ISO 8601 formatted datetime string
55+
#Datetime: time.Format("2006-01-02T15:04:05Z07:00") @go(Datetime,format="date-time")
56+
57+
// Date represents a date string (ISO 8601 date format)
58+
#Date: time.Format("2006-01-02") @go(Date,format="date")
59+
60+
// Category represents a category used for applicability or classification
61+
#Category: {
62+
// id allows this entry to be referenced by other elements
63+
id: string
64+
65+
// title describes the purpose of this category at a glance
66+
title: string
67+
68+
// description explains the significance and traits of entries to this category
69+
description: string
70+
}
71+
72+
// Family represents a logical grouping of guidelines or controls which share a common purpose or function
73+
#Family: {
74+
// id allows this entry to be referenced by other elements
75+
id: string
76+
77+
// title describes the purpose of this family at a glance
78+
title: string
79+
80+
// description explains the significance and traits of entries to this entity family
81+
description: string
82+
}

‎cue/gemara/mapping.cue‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// Schema lifecycle: experimental | stable | deprecated
2+
@status("stable")
3+
4+
package gemara
5+
6+
// MappingReference represents a reference to an external document with full metadata.
7+
#MappingReference: {
8+
// id allows this entry to be referenced by other elements
9+
id: string
10+
11+
// title describes the purpose of this mapping reference at a glance
12+
title: string
13+
14+
// version is the version identifier of the artifact being mapped to
15+
version: string
16+
17+
// description is prose regarding the artifact's purpose or content
18+
description?: string
19+
20+
// url is the path where the artifact may be retrieved; preferrably responds with Gemara-compatible YAML/JSON
21+
url?: =~"^(https?|file)://[^\\s]+$"
22+
}
23+
24+
#ArtifactMapping: {
25+
// ReferenceId should reference the corresponding MappingReference id from metadata
26+
"reference-id": string @go(ReferenceId)
27+
28+
// remarks is prose regarding the mapped artifact or the mapping relationship
29+
"remarks": string
30+
}
31+
32+
// MultiEntryMapping represents a mapping to an external reference with one or more entries.
33+
#MultiEntryMapping: {
34+
// ReferenceId should reference the corresponding MappingReference id from metadata
35+
"reference-id": string @go(ReferenceId)
36+
37+
// entries is a list of mapping entries
38+
entries: [#MappingEntry, ...#MappingEntry] @go(Entries)
39+
40+
// remarks is prose regarding the mapped artifact or the mapping relationship
41+
remarks?: string
42+
}
43+
44+
// EntryMapping represents how a specific entry (control/requirement/procedure) maps to a MappingReference.
45+
#EntryMapping: {
46+
// reference-id is the id for a MappingReference entry in the artifact's metadata
47+
"reference-id"?: string @go(ReferenceId)
48+
49+
// entry-id is the identifier being mapped to in the referenced artifact
50+
"entry-id": string @go(EntryId)
51+
52+
// strength is the author's estimate of how completely the current/source material satisfies the target/reference material;
53+
// Range: 1-10. Zero value means not yet quantified.
54+
strength?: int & >=1 & <=10
55+
56+
// remarks is prose describing the mapping relationship
57+
remarks?: string
58+
}
59+
60+
// MappingEntry represents a single entry within a mapping
61+
#MappingEntry: {
62+
// reference-id is the id for a MappingReference entry in the artifact's metadata
63+
"reference-id": string @go(ReferenceId)
64+
65+
// strength is the author's estimate of how completely the current/source material satisfies the target/reference material;
66+
// Range: 1-10. Zero value means not yet quantified.
67+
strength?: int & >=1 & <=10
68+
69+
// remarks is prose describing the mapping relationship
70+
remarks?: string
71+
}

‎cue/gemara/metadata.cue‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// Schema lifecycle: experimental | stable | deprecated
2+
@status("stable")
3+
package gemara
4+
5+
// Metadata represents common metadata fields shared across all layers
6+
#Metadata: {
7+
// id allows this entry to be referenced by other elements
8+
id: string
9+
10+
// version is the version identifier of this artifact
11+
version?: string
12+
13+
// date is the publication or effective date of this artifact
14+
date?: #Date @go(Date)
15+
16+
// description provides a high-level summary of the artifact's purpose and scope
17+
description: string
18+
19+
// author is the person or group primarily responsible for this artifact
20+
author: #Actor
21+
22+
// mapping-references is a list of external documents referenced within this artifact
23+
"mapping-references"?: [...#MappingReference] @go(MappingReferences) @yaml("mapping-references,omitempty")
24+
25+
// applicability-categories is a list of categories used to classify within this artifact to specify scope
26+
"applicability-categories"?: [...#Category] @go(ApplicabilityCategories) @yaml("applicability-categories,omitempty")
27+
28+
// draft indicates whether this artifact is a pre-release version; open to modification
29+
draft?: bool
30+
31+
// lexicon is a URI pointing to a controlled vocabulary or glossary relevant to this artifact
32+
lexicon?: #ArtifactMapping @go(Lexicon,optional=nillable)
33+
}

‎practices/Independent-Verification.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,11 @@ gemara:
2727
- autonomous-deployment
2828
- ci-cd-integration
2929
threat-mappings:
30-
- id: verification-illusion
31-
relationship: mitigates
30+
- reference-id: verification-illusion
31+
entries:
32+
- reference-id: verification-illusion
33+
strength: 8
34+
remarks: Primary control for preventing circular verification
3235
---
3336

3437
# Independent Verification

‎risks/Verification-Illusion.md‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,20 +10,28 @@ gemara:
1010
title: Verification Illusion
1111
description: When an AI agent both writes code and generates its own tests, the tests tend to verify what the code does rather than what it should do. This creates a dangerous illusion of quality — high test coverage with low actual assurance.
1212
capabilities:
13-
- id: code-generation
14-
relationship: exploits
15-
- id: execution
16-
relationship: exploits
13+
- reference-id: code-generation
14+
entries:
15+
- reference-id: code-generation
16+
remarks: AI generates both code and tests from the same cognitive process
17+
- reference-id: execution
18+
entries:
19+
- reference-id: execution
20+
remarks: Tests execute without independent verification of intent
1721
actors:
18-
- type: unintentional
22+
- id: ai-agent
23+
name: AI Coding Agent
24+
type: Software
1925
description: AI agents optimizing for coverage metrics rather than correctness
2026
external-mappings:
21-
- id: nist-ai-rmf
22-
relationship: gap
23-
notes: Not addressed at code verification level
24-
- id: iso-42001
25-
relationship: gap
26-
notes: No assurance independence requirement
27+
- reference-id: nist-ai-rmf
28+
entries:
29+
- reference-id: nist-ai-rmf
30+
remarks: Not addressed at code verification level
31+
- reference-id: iso-42001
32+
entries:
33+
- reference-id: iso-42001
34+
remarks: No assurance independence requirement
2735
---
2836

2937
# Verification Illusion

0 commit comments

Comments
 (0)