Skip to content

Commit e1cb461

Browse files
darklight3itdarklight3it
authored
Merge pull request #612 from yhamano0312/feat/add-s3-event-fields
Add S3 event optional fields
2 parents 9dac8a5 + a66ce2d commit e1cb461

7 files changed

Lines changed: 413 additions & 9 deletions

File tree

‎events/s3.go‎

Lines changed: 43 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,20 @@ type S3Event struct {
1515

1616
// S3EventRecord which wrap record data
1717
type S3EventRecord struct {
18-
EventVersion string `json:"eventVersion"`
19-
EventSource string `json:"eventSource"`
20-
AWSRegion string `json:"awsRegion"`
21-
EventTime time.Time `json:"eventTime"`
22-
EventName string `json:"eventName"`
23-
PrincipalID S3UserIdentity `json:"userIdentity"`
24-
RequestParameters S3RequestParameters `json:"requestParameters"`
25-
ResponseElements map[string]string `json:"responseElements"`
26-
S3 S3Entity `json:"s3"`
18+
EventVersion string `json:"eventVersion"`
19+
EventSource string `json:"eventSource"`
20+
AWSRegion string `json:"awsRegion"`
21+
EventTime time.Time `json:"eventTime"`
22+
EventName string `json:"eventName"`
23+
PrincipalID S3UserIdentity `json:"userIdentity"`
24+
RequestParameters S3RequestParameters `json:"requestParameters"`
25+
ResponseElements map[string]string `json:"responseElements"`
26+
S3 S3Entity `json:"s3"`
27+
GlacierEventData *S3GlacierEventData `json:"glacierEventData,omitempty"`
28+
RestoreEventData *S3RestoreEventData `json:"restoreEventData,omitempty"`
29+
ReplicationEventData *S3ReplicationEventData `json:"replicationEventData,omitempty"`
30+
IntelligentTieringEventData *S3IntelligentTieringEventData `json:"intelligentTieringEventData,omitempty"`
31+
LifecycleEventData *S3LifecycleEventData `json:"lifecycleEventData,omitempty"`
2732
}
2833

2934
type S3UserIdentity struct {
@@ -70,6 +75,35 @@ func (o *S3Object) UnmarshalJSON(data []byte) error {
7075
return nil
7176
}
7277

78+
type S3GlacierEventData struct {
79+
RestoreEventData *S3RestoreEventData `json:"restoreEventData"`
80+
}
81+
82+
type S3RestoreEventData struct {
83+
LifecycleRestorationExpiryTime time.Time `json:"lifecycleRestorationExpiryTime"`
84+
LifecycleRestoreStorageClass string `json:"lifecycleRestoreStorageClass"`
85+
}
86+
87+
type S3ReplicationEventData struct {
88+
ReplicationRuleID string `json:"replicationRuleId"`
89+
DestinationBucket string `json:"destinationBucket"`
90+
S3Operation string `json:"s3Operation"`
91+
RequestTime time.Time `json:"requestTime"`
92+
FailureReason string `json:"failureReason"`
93+
}
94+
95+
type S3IntelligentTieringEventData struct {
96+
DestinationAccessTier string `json:"destinationAccessTier"`
97+
}
98+
99+
type S3LifecycleEventData struct {
100+
TransitionEventData *S3TransitionEventData `json:"transitionEventData"`
101+
}
102+
103+
type S3TransitionEventData struct {
104+
DestinationStorageClass string `json:"destinationStorageClass"`
105+
}
106+
73107
type S3TestEvent struct {
74108
Service string `json:"Service"`
75109
Bucket string `json:"Bucket"`

‎events/s3_test.go‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,3 +56,148 @@ func TestS3TestEventMarshaling(t *testing.T) {
5656
func TestS3MarshalingMalformedJSON(t *testing.T) {
5757
test.TestMalformedJson(t, S3Event{})
5858
}
59+
60+
func TestS3GlacierEventMarshaling(t *testing.T) {
61+
// 1. read JSON from file
62+
inputJSON := test.ReadJSONFromFile(t, "./testdata/s3-glacier-event.json")
63+
64+
// 2. de-serialize into Go object
65+
var inputEvent S3Event
66+
if err := json.Unmarshal(inputJSON, &inputEvent); err != nil {
67+
t.Errorf("could not unmarshal event. details: %v", err)
68+
}
69+
70+
// 3. verify glacierEventData is correctly parsed
71+
if inputEvent.Records[0].GlacierEventData == nil {
72+
t.Error("glacierEventData should not be nil for glacier restore events")
73+
}
74+
75+
// 4. verify restoreEventData is correctly parsed
76+
if inputEvent.Records[0].GlacierEventData.RestoreEventData == nil {
77+
t.Error("restoreEventData should not be nil")
78+
}
79+
80+
// 5. serialize to JSON
81+
outputJSON, err := json.Marshal(inputEvent)
82+
if err != nil {
83+
t.Errorf("could not marshal event. details: %v", err)
84+
}
85+
86+
// 6. check result
87+
assert.JSONEq(t, string(inputJSON), string(outputJSON))
88+
}
89+
90+
func TestS3RestoreEventMarshaling(t *testing.T) {
91+
// 1. read JSON from file
92+
inputJSON := test.ReadJSONFromFile(t, "./testdata/s3-restore-event.json")
93+
94+
// 2. de-serialize into Go object
95+
var inputEvent S3Event
96+
if err := json.Unmarshal(inputJSON, &inputEvent); err != nil {
97+
t.Errorf("could not unmarshal event. details: %v", err)
98+
}
99+
100+
// 3. verify restoreEventData is correctly parsed
101+
if inputEvent.Records[0].RestoreEventData == nil {
102+
t.Error("restoreEventData should not be nil")
103+
}
104+
105+
// 4. serialize to JSON
106+
outputJSON, err := json.Marshal(inputEvent)
107+
if err != nil {
108+
t.Errorf("could not marshal event. details: %v", err)
109+
}
110+
111+
// 5. check result
112+
assert.JSONEq(t, string(inputJSON), string(outputJSON))
113+
}
114+
115+
func TestS3IntelligentTieringEventMarshaling(t *testing.T) {
116+
// 1. read JSON from file
117+
inputJSON := test.ReadJSONFromFile(t, "./testdata/s3-intelligenttier-event.json")
118+
119+
// 2. de-serialize into Go object
120+
var inputEvent S3Event
121+
if err := json.Unmarshal(inputJSON, &inputEvent); err != nil {
122+
t.Errorf("could not unmarshal event. details: %v", err)
123+
}
124+
125+
// 3. verify intelligentTieringEventData is correctly parsed
126+
if inputEvent.Records[0].IntelligentTieringEventData == nil {
127+
t.Error("intelligentTieringEventData should not be nil for intelligent tiering events")
128+
}
129+
130+
// 4. verify destinationAccessTier is correctly parsed
131+
if inputEvent.Records[0].IntelligentTieringEventData.DestinationAccessTier == "" {
132+
t.Error("destinationAccessTier should not be empty")
133+
}
134+
135+
// 5. serialize to JSON
136+
outputJSON, err := json.Marshal(inputEvent)
137+
if err != nil {
138+
t.Errorf("could not marshal event. details: %v", err)
139+
}
140+
141+
// 6. check result
142+
assert.JSONEq(t, string(inputJSON), string(outputJSON))
143+
}
144+
145+
func TestS3LifecycleEventMarshaling(t *testing.T) {
146+
// 1. read JSON from file
147+
inputJSON := test.ReadJSONFromFile(t, "./testdata/s3-lifecycle-event.json")
148+
149+
// 2. de-serialize into Go object
150+
var inputEvent S3Event
151+
if err := json.Unmarshal(inputJSON, &inputEvent); err != nil {
152+
t.Errorf("could not unmarshal event. details: %v", err)
153+
}
154+
155+
// 3. verify lifecycleEventData is correctly parsed
156+
if inputEvent.Records[0].LifecycleEventData == nil {
157+
t.Error("lifecycleEventData should not be nil for lifecycle events")
158+
}
159+
160+
// 4. verify transitionEventData is correctly parsed
161+
if inputEvent.Records[0].LifecycleEventData.TransitionEventData == nil {
162+
t.Error("transitionEventData should not be nil")
163+
}
164+
165+
// 5. verify destinationStorageClass is correctly parsed
166+
if inputEvent.Records[0].LifecycleEventData.TransitionEventData.DestinationStorageClass == "" {
167+
t.Error("destinationStorageClass should not be empty")
168+
}
169+
170+
// 6. serialize to JSON
171+
outputJSON, err := json.Marshal(inputEvent)
172+
if err != nil {
173+
t.Errorf("could not marshal event. details: %v", err)
174+
}
175+
176+
// 7. check result
177+
assert.JSONEq(t, string(inputJSON), string(outputJSON))
178+
}
179+
180+
func TestS3ReplicationEventMarshaling(t *testing.T) {
181+
// 1. read JSON from file
182+
inputJSON := test.ReadJSONFromFile(t, "./testdata/s3-replication-event.json")
183+
184+
// 2. de-serialize into Go object
185+
var inputEvent S3Event
186+
if err := json.Unmarshal(inputJSON, &inputEvent); err != nil {
187+
t.Errorf("could not unmarshal event. details: %v", err)
188+
}
189+
190+
// 3. verify replicationEventData is correctly parsed
191+
if inputEvent.Records[0].ReplicationEventData == nil {
192+
t.Error("replicationEventData should not be nil for replication events")
193+
}
194+
195+
// 4. serialize to JSON
196+
outputJSON, err := json.Marshal(inputEvent)
197+
if err != nil {
198+
t.Errorf("could not marshal event. details: %v", err)
199+
}
200+
201+
// 5. check result
202+
assert.JSONEq(t, string(inputJSON), string(outputJSON))
203+
}

‎events/testdata/s3-glacier-event.json‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{
2+
"Records": [
3+
{
4+
"eventVersion": "2.1",
5+
"eventSource": "aws:s3",
6+
"awsRegion": "us-west-2",
7+
"eventTime": "2023-05-10T15:00:00.123Z",
8+
"eventName": "s3:ObjectRestore:Completed",
9+
"userIdentity": {
10+
"principalId": "EXAMPLE"
11+
},
12+
"requestParameters": {
13+
"sourceIPAddress": "127.0.0.1"
14+
},
15+
"responseElements": {
16+
"x-amz-request-id": "EXAMPLE123456789",
17+
"x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH"
18+
},
19+
"s3": {
20+
"s3SchemaVersion": "1.0",
21+
"configurationId": "testConfigRule",
22+
"bucket": {
23+
"name": "example-bucket",
24+
"ownerIdentity": {
25+
"principalId": "EXAMPLE"
26+
},
27+
"arn": "arn:aws:s3:::example-bucket"
28+
},
29+
"object": {
30+
"key": "glacier%2Dtest.txt",
31+
"urlDecodedKey": "glacier-test.txt",
32+
"size": 1024,
33+
"versionId": "abcdeH0Xp66ep__QDjR76LK7Gc9X4wKO",
34+
"eTag": "0123456789abcdef0123456789abcdef",
35+
"sequencer": "0A1B2C3D4E5F678901"
36+
}
37+
},
38+
"glacierEventData": {
39+
"restoreEventData": {
40+
"lifecycleRestorationExpiryTime": "2023-05-17T15:00:00.456Z",
41+
"lifecycleRestoreStorageClass": "STANDARD"
42+
}
43+
}
44+
}
45+
]
46+
}

‎events/testdata/s3-intelligenttier-event.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"Records": [
3+
{
4+
"eventVersion": "2.3",
5+
"eventSource": "aws:s3",
6+
"awsRegion": "us-east-1",
7+
"eventTime": "2023-11-15T10:30:00.789Z",
8+
"eventName": "s3:IntelligentTiering:TransitionAccess",
9+
"userIdentity": {
10+
"principalId": "s3.amazonaws.com"
11+
},
12+
"requestParameters": {
13+
"sourceIPAddress": "s3.amazonaws.com"
14+
},
15+
"responseElements": {
16+
"x-amz-request-id": "EXAMPLE123456789",
17+
"x-amz-id-2": "EXAMPLEabcdefghijklmnopqrstuvwxyz"
18+
},
19+
"s3": {
20+
"s3SchemaVersion": "1.0",
21+
"configurationId": "IntelligentTieringConfig",
22+
"bucket": {
23+
"name": "example-bucket",
24+
"ownerIdentity": {
25+
"principalId": "EXAMPLE"
26+
},
27+
"arn": "arn:aws:s3:::example-bucket"
28+
},
29+
"object": {
30+
"key": "intelligent%2Dtier%2Dtest.dat",
31+
"urlDecodedKey": "intelligent-tier-test.dat",
32+
"size": 5242880,
33+
"versionId": "versionExample123",
34+
"eTag": "d41d8cd98f00b204e9800998ecf8427e",
35+
"sequencer": "0066ABC1234567890"
36+
}
37+
},
38+
"intelligentTieringEventData": {
39+
"destinationAccessTier": "ARCHIVE_ACCESS"
40+
}
41+
}
42+
]
43+
}

‎events/testdata/s3-lifecycle-event.json‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{
2+
"Records": [
3+
{
4+
"eventVersion": "2.3",
5+
"eventSource": "aws:s3",
6+
"awsRegion": "us-west-2",
7+
"eventTime": "2023-08-20T14:25:00.321Z",
8+
"eventName": "s3:LifecycleTransition",
9+
"userIdentity": {
10+
"principalId": "s3.amazonaws.com"
11+
},
12+
"requestParameters": {
13+
"sourceIPAddress": "s3.amazonaws.com"
14+
},
15+
"responseElements": {
16+
"x-amz-request-id": "LIFECYCLE123456789",
17+
"x-amz-id-2": "LIFECYCLEabcdefghijklmnopqrstuvwxyz"
18+
},
19+
"s3": {
20+
"s3SchemaVersion": "1.0",
21+
"configurationId": "LifecycleTransitionRule",
22+
"bucket": {
23+
"name": "example-bucket",
24+
"ownerIdentity": {
25+
"principalId": "EXAMPLE"
26+
},
27+
"arn": "arn:aws:s3:::example-bucket"
28+
},
29+
"object": {
30+
"key": "lifecycle%2Dtest.dat",
31+
"urlDecodedKey": "lifecycle-test.dat",
32+
"size": 10485760,
33+
"versionId": "lifecycleVersion123",
34+
"eTag": "abcd1234ef5678901234567890abcdef",
35+
"sequencer": "0066LIFECYCLE890"
36+
}
37+
},
38+
"lifecycleEventData": {
39+
"transitionEventData": {
40+
"destinationStorageClass": "GLACIER"
41+
}
42+
}
43+
}
44+
]
45+
}

‎events/testdata/s3-replication-event.json‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
{
2+
"Records": [
3+
{
4+
"eventVersion": "2.2",
5+
"eventSource": "aws:s3",
6+
"awsRegion": "us-east-1",
7+
"eventTime": "2024-09-05T21:04:32.527Z",
8+
"eventName": "s3:Replication:OperationFailedReplication",
9+
"userIdentity": {
10+
"principalId": "s3.amazonaws.com"
11+
},
12+
"requestParameters": {
13+
"sourceIPAddress": "s3.amazonaws.com"
14+
},
15+
"responseElements": {
16+
"x-amz-request-id": "123bf045-2b4b-4ca8-a211-c34a63c59426",
17+
"x-amz-id-2": "12VAWNDIHnwJsRhTccqQTeAPoXQmRt22KkewMV8G3XZihAuf9CLDdmkApgZzudaIe2KlLfDqGS0="
18+
},
19+
"s3": {
20+
"s3SchemaVersion": "1.0",
21+
"configurationId": "ReplicationEventName",
22+
"bucket": {
23+
"name": "amzn-s3-demo-bucket1",
24+
"ownerIdentity": {
25+
"principalId": "111122223333"
26+
},
27+
"arn": "arn:aws:s3:::amzn-s3-demo-bucket1"
28+
},
29+
"object": {
30+
"key": "replication%2Dtest.png",
31+
"urlDecodedKey": "replication-test.png",
32+
"size": 520080,
33+
"versionId": "abcdeH0Xp66ep__QDjR76LK7Gc9X4wKO",
34+
"eTag": "e12345ca7e88a38428305d3ff7fcb99f",
35+
"sequencer": "0066DA1CBF104C0D51"
36+
}
37+
},
38+
"replicationEventData": {
39+
"replicationRuleId": "notification-test-replication-rule",
40+
"destinationBucket": "arn:aws:s3:::amzn-s3-demo-bucket2",
41+
"s3Operation": "OBJECT_PUT",
42+
"requestTime": "2024-09-05T21:03:59.168Z",
43+
"failureReason": "AssumeRoleNotPermitted"
44+
}
45+
}
46+
]
47+
}

0 commit comments

Comments
 (0)