Skip to content

Commit 0268ef8

Browse files
kostkokostko
authored
Merge pull request #6233 from oasisprotocol/kostko/feature/sgx-pcs-tcb-evalnums-cache
go/common/sgx: Also cache TCB evaluation data numbers
2 parents 64afc8c + 489a19f commit 0268ef8

4 files changed

Lines changed: 159 additions & 49 deletions

File tree

‎.changelog/6233.trivial.md‎

Whitespace-only changes.

‎go/common/sgx/pcs/cache.go‎

Lines changed: 72 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,21 @@ import (
1111
)
1212

1313
const (
14-
tcbCacheKey = "tcb_bundle_cache"
14+
tcbBundleCacheKeyPrefix = "tcb_bundle_cache"
15+
tcbEvaluationDataNumbersCacheKeyPrefix = "tcb_evaluation_data_numbers_cache"
1516

1617
tcbCacheRefreshThreshold = 14 * 24 * time.Hour
1718
tcbCacheSlowRefreshInterval = 24 * time.Hour
1819
)
1920

21+
func tcbBundleCacheKey(teeType TeeType) []byte {
22+
return []byte(fmt.Sprintf("%s.%d", tcbBundleCacheKeyPrefix, teeType))
23+
}
24+
25+
func tcbEvaluationDataNumbersCacheKey(teeType TeeType) []byte {
26+
return []byte(fmt.Sprintf("%s.%d", tcbEvaluationDataNumbersCacheKeyPrefix, teeType))
27+
}
28+
2029
func readBundleMinTimestamp(bundle *TCBBundle) (time.Time, error) {
2130
var err error
2231
var info TCBInfo
@@ -50,18 +59,57 @@ type tcbBundleCache struct {
5059
LastUpdate time.Time `json:"last_update"`
5160
}
5261

62+
type tcbEvaluationDataNumbersCache struct {
63+
Numbers []uint32 `json:"numbers"`
64+
LastUpdate time.Time `json:"last_update"`
65+
}
66+
5367
type tcbCache struct {
5468
serviceStore *persistent.ServiceStore
5569
logger *logging.Logger
5670
now func() time.Time
5771
}
5872

59-
func (tc *tcbCache) check(fmspc []byte) (*TCBBundle, bool) {
73+
func (tc *tcbCache) checkEvaluationDataNumbers(teeType TeeType) ([]uint32, bool) {
74+
var stored tcbEvaluationDataNumbersCache
75+
switch err := tc.serviceStore.GetCBOR(tcbEvaluationDataNumbersCacheKey(teeType), &stored); err {
76+
case nil:
77+
// No error, continues below.
78+
case persistent.ErrNotFound:
79+
// Not cached yet. Not an error, but needs refresh.
80+
return nil, true
81+
default:
82+
// Can't get it... an error, but we can still try downloading it.
83+
tc.logger.Warn("error checking common store for cached TCB evaluation data numbers",
84+
"err", err,
85+
)
86+
return nil, true
87+
}
88+
89+
now := tc.now()
90+
delta := now.Sub(stored.LastUpdate)
91+
refresh := delta > tcbCacheSlowRefreshInterval
92+
return stored.Numbers, refresh
93+
}
94+
95+
func (tc *tcbCache) cacheEvaluationDataNumbers(teeType TeeType, numbers []uint32) {
96+
cached := tcbEvaluationDataNumbersCache{
97+
Numbers: numbers,
98+
LastUpdate: tc.now(),
99+
}
100+
if err := tc.serviceStore.PutCBOR(tcbEvaluationDataNumbersCacheKey(teeType), cached); err != nil {
101+
tc.logger.Error("could not store new TCB evaluation data numbers to cache, ignoring",
102+
"err", err,
103+
)
104+
}
105+
}
106+
107+
func (tc *tcbCache) checkBundle(teeType TeeType, fmspc []byte) (*TCBBundle, bool) {
60108
var err error
61109

62110
// Check if we have a copy in the local store.
63111
var stored tcbBundleCache
64-
switch err = tc.serviceStore.GetCBOR([]byte(tcbCacheKey), &stored); err {
112+
switch err = tc.serviceStore.GetCBOR(tcbBundleCacheKey(teeType), &stored); err {
65113
case nil:
66114
// No error, continues below.
67115
case persistent.ErrNotFound:
@@ -97,7 +145,7 @@ func (tc *tcbCache) check(fmspc []byte) (*TCBBundle, bool) {
97145
return stored.Bundle, refresh
98146
}
99147

100-
func (tc *tcbCache) cache(tcbBundle *TCBBundle, fmspc []byte) {
148+
func (tc *tcbCache) cacheBundle(teeType TeeType, tcbBundle *TCBBundle, fmspc []byte) {
101149
expectedExpiry, err := readBundleMinTimestamp(tcbBundle)
102150
if err != nil {
103151
tc.logger.Error("could not determine next update timestamp from TCB bundle",
@@ -112,25 +160,42 @@ func (tc *tcbCache) cache(tcbBundle *TCBBundle, fmspc []byte) {
112160
ExpectedExpiry: expectedExpiry,
113161
LastUpdate: tc.now(),
114162
}
115-
if err = tc.serviceStore.PutCBOR([]byte(tcbCacheKey), cached); err != nil {
163+
if err = tc.serviceStore.PutCBOR(tcbBundleCacheKey(teeType), cached); err != nil {
116164
tc.logger.Error("could not store new TCB bundle to cache, ignoring",
117165
"err", err,
118166
)
119167
}
120168
}
121169

170+
func (tc *tcbCache) migrate() {
171+
// Migrate any old (without TEE type) cached entries.
172+
var stored tcbBundleCache
173+
switch err := tc.serviceStore.GetCBOR([]byte(tcbBundleCacheKeyPrefix), &stored); err {
174+
case nil:
175+
// No error, migrate. Any errors during migration are ignored as this is a cache.
176+
_ = tc.serviceStore.PutCBOR(tcbBundleCacheKey(TeeTypeSGX), stored)
177+
_ = tc.serviceStore.Delete([]byte(tcbBundleCacheKeyPrefix))
178+
default:
179+
// No migration needed.
180+
}
181+
}
182+
122183
func newTcbCache(serviceStore *persistent.ServiceStore, logger *logging.Logger) *tcbCache {
123-
return &tcbCache{
184+
tc := &tcbCache{
124185
serviceStore: serviceStore,
125186
logger: logger,
126187
now: time.Now,
127188
}
189+
tc.migrate()
190+
return tc
128191
}
129192

130193
func newMockTcbCache(serviceStore *persistent.ServiceStore, logger *logging.Logger, now func() time.Time) *tcbCache {
131-
return &tcbCache{
194+
tc := &tcbCache{
132195
serviceStore: serviceStore,
133196
logger: logger,
134197
now: now,
135198
}
199+
tc.migrate()
200+
return tc
136201
}

‎go/common/sgx/pcs/cache_test.go‎

Lines changed: 66 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,17 @@ func (ft *fakeTime) get() time.Time {
2525
func testStorageRoundtrip(t *testing.T, store *persistent.ServiceStore, bundle *TCBBundle) {
2626
require := require.New(t)
2727
fmspc := []byte("fmspc")
28+
numbers := []uint32{17, 18, 19}
2829

2930
tcbCache := newMockTcbCache(store, logging.GetLogger(loggerModule), time.Now)
30-
tcbCache.cache(bundle, fmspc)
31+
tcbCache.cacheBundle(TeeTypeSGX, bundle, fmspc)
32+
tcbCache.cacheEvaluationDataNumbers(TeeTypeSGX, numbers)
3133

32-
cached, _ := tcbCache.check(fmspc)
33-
require.EqualValues(cached, bundle, "tcbCache.check")
34+
cachedBundle, _ := tcbCache.checkBundle(TeeTypeSGX, fmspc)
35+
require.EqualValues(cachedBundle, bundle, "tcbCache.checkBundle")
36+
37+
cachedNumbers, _ := tcbCache.checkEvaluationDataNumbers(TeeTypeSGX)
38+
require.EqualValues(cachedNumbers, numbers, "tcbCache.checkEvaluationDataNumbers")
3439
}
3540

3641
func testFMSPCInvalidation(t *testing.T, store *persistent.ServiceStore, bundle *TCBBundle) {
@@ -47,18 +52,18 @@ func testFMSPCInvalidation(t *testing.T, store *persistent.ServiceStore, bundle
4752
var refresh bool
4853

4954
// Cache initial and check.
50-
tcbCache.cache(bundle, fmspc)
51-
cached, refresh = tcbCache.check(fmspc)
55+
tcbCache.cacheBundle(TeeTypeSGX, bundle, fmspc)
56+
cached, refresh = tcbCache.checkBundle(TeeTypeSGX, fmspc)
5257
require.NotNil(cached, "tcbCache.check 1")
5358
require.False(refresh, "tcbCache.check 1")
5459

5560
// Check again with bogus fmspc; shouldn't return anything
5661
// but should still be available.
57-
cached, refresh = tcbCache.check([]byte("different"))
62+
cached, refresh = tcbCache.checkBundle(TeeTypeSGX, []byte("different"))
5863
require.Nil(cached, "tcbCache.check 2")
5964
require.True(refresh, "tcbCache.check 2")
6065

61-
cached, refresh = tcbCache.check(fmspc)
66+
cached, refresh = tcbCache.checkBundle(TeeTypeSGX, fmspc)
6267
require.NotNil(cached, "tcbCache.check 3")
6368
require.False(refresh, "tcbCache.check 3")
6469
}
@@ -75,54 +80,81 @@ func testCheckIntervals(t *testing.T, store *persistent.ServiceStore, bundle *TC
7580
tcbCache := newMockTcbCache(store, logging.GetLogger(loggerModule), timer.get)
7681

7782
// Initially, always needs to be refreshed.
78-
cache, refresh := tcbCache.check(fmspc)
79-
require.Nil(cache, "tcbCache.check pre-cache")
80-
require.True(refresh, "tcbCache.check pre-cache")
83+
cache, refresh := tcbCache.checkBundle(TeeTypeSGX, fmspc)
84+
require.Nil(cache, "tcbCache.checkBundle pre-cache")
85+
require.True(refresh, "tcbCache.checkBundle pre-cache")
86+
87+
cachedNumbers, refresh := tcbCache.checkEvaluationDataNumbers(TeeTypeSGX)
88+
require.Nil(cachedNumbers, "tcbCache.checkEvaluationDataNumbers pre-cache")
89+
require.True(refresh, "tcbCache.checkEvaluationDataNumbers pre-cache")
8190

8291
// Cache it, pretend it's a day before the first check will need to be performed.
8392
timer.now = expiryTime.Add(-(tcbCacheRefreshThreshold + 24*time.Hour))
84-
tcbCache.cache(bundle, fmspc)
93+
tcbCache.cacheBundle(TeeTypeSGX, bundle, fmspc)
94+
tcbCache.cacheEvaluationDataNumbers(TeeTypeSGX, []uint32{17, 18, 19})
8595

8696
// An hour after the initial cache, shouldn't be refreshed.
8797
timer.now = timer.now.Add(time.Hour)
88-
cache, refresh = tcbCache.check(fmspc)
89-
require.NotNil(cache, "tcbCache.check 1")
90-
require.False(refresh, "tcbCache.check 1")
98+
cache, refresh = tcbCache.checkBundle(TeeTypeSGX, fmspc)
99+
require.NotNil(cache, "tcbCache.checkBundle 1")
100+
require.False(refresh, "tcbCache.checkBundle 1")
101+
102+
cachedNumbers, refresh = tcbCache.checkEvaluationDataNumbers(TeeTypeSGX)
103+
require.NotNil(cachedNumbers, "tcbCache.checkEvaluationDataNumbers 1")
104+
require.False(refresh, "tcbCache.checkEvaluationDataNumbers 1")
91105

92106
// Another day later, we're in the slow refresh cycle. First check should refresh.
93107
// Advance by 25 hours, because 24 would still be within the slow refresh interval.
94108
timer.now = timer.now.Add(25 * time.Hour)
95-
cache, refresh = tcbCache.check(fmspc)
96-
require.NotNil(cache, "tcbCache.check 2")
97-
require.True(refresh, "tcbCache.check 2")
98-
tcbCache.cache(bundle, fmspc)
109+
cache, refresh = tcbCache.checkBundle(TeeTypeSGX, fmspc)
110+
require.NotNil(cache, "tcbCache.checkBundle 2")
111+
require.True(refresh, "tcbCache.checkBundle 2")
112+
tcbCache.cacheBundle(TeeTypeSGX, bundle, fmspc)
113+
114+
cachedNumbers, refresh = tcbCache.checkEvaluationDataNumbers(TeeTypeSGX)
115+
require.NotNil(cachedNumbers, "tcbCache.checkEvaluationDataNumbers 2")
116+
require.True(refresh, "tcbCache.checkEvaluationDataNumbers 2")
117+
tcbCache.cacheEvaluationDataNumbers(TeeTypeSGX, []uint32{17, 18, 19})
99118

100119
// An hour later, don't check again.
101120
timer.now = timer.now.Add(time.Hour)
102-
cache, refresh = tcbCache.check(fmspc)
103-
require.NotNil(cache, "tcbCache.check 3")
104-
require.False(refresh, "tcbCache.check 3")
121+
cache, refresh = tcbCache.checkBundle(TeeTypeSGX, fmspc)
122+
require.NotNil(cache, "tcbCache.checkBundle 3")
123+
require.False(refresh, "tcbCache.checkBundle 3")
124+
125+
cachedNumbers, refresh = tcbCache.checkEvaluationDataNumbers(TeeTypeSGX)
126+
require.NotNil(cachedNumbers, "tcbCache.checkEvaluationDataNumbers 3")
127+
require.False(refresh, "tcbCache.checkEvaluationDataNumbers 3")
105128

106129
// 22 hours later, still don't check (within slow refresh interval).
107130
// Two hours after that, do check.
108131
timer.now = timer.now.Add(22 * time.Hour)
109-
cache, refresh = tcbCache.check(fmspc)
110-
require.NotNil(cache, "tcbCache.check 4")
111-
require.False(refresh, "tcbCache.check 4")
132+
cache, refresh = tcbCache.checkBundle(TeeTypeSGX, fmspc)
133+
require.NotNil(cache, "tcbCache.checkBundle 4")
134+
require.False(refresh, "tcbCache.checkBundle 4")
135+
136+
cachedNumbers, refresh = tcbCache.checkEvaluationDataNumbers(TeeTypeSGX)
137+
require.NotNil(cachedNumbers, "tcbCache.checkEvaluationDataNumbers 4")
138+
require.False(refresh, "tcbCache.checkEvaluationDataNumbers 4")
112139

113140
timer.now = timer.now.Add(2 * time.Hour)
114-
cache, refresh = tcbCache.check(fmspc)
115-
require.NotNil(cache, "tcbCache.check 5")
116-
require.True(refresh, "tcbCache.check 5")
117-
tcbCache.cache(bundle, fmspc)
141+
cache, refresh = tcbCache.checkBundle(TeeTypeSGX, fmspc)
142+
require.NotNil(cache, "tcbCache.checkBundle 5")
143+
require.True(refresh, "tcbCache.checkBundle 5")
144+
tcbCache.cacheBundle(TeeTypeSGX, bundle, fmspc)
145+
146+
cachedNumbers, refresh = tcbCache.checkEvaluationDataNumbers(TeeTypeSGX)
147+
require.NotNil(cachedNumbers, "tcbCache.checkEvaluationDataNumbers 5")
148+
require.True(refresh, "tcbCache.checkEvaluationDataNumbers 5")
149+
tcbCache.cacheEvaluationDataNumbers(TeeTypeSGX, []uint32{17, 18, 19})
118150

119151
// After the bundle expires, check all the time.
120152
timer.now = expiryTime
121153
for i := 0; i < 4; i++ {
122-
cache, refresh = tcbCache.check(fmspc)
123-
require.NotNil(cache, "tcbCache.check loop")
124-
require.True(refresh, "tcbCache.check loop")
125-
tcbCache.cache(bundle, fmspc)
154+
cache, refresh = tcbCache.checkBundle(TeeTypeSGX, fmspc)
155+
require.NotNil(cache, "tcbCache.checkBundle loop")
156+
require.True(refresh, "tcbCache.checkBundle loop")
157+
tcbCache.cacheBundle(TeeTypeSGX, bundle, fmspc)
126158
timer.now = timer.now.Add(time.Hour)
127159
}
128160
}
@@ -169,7 +201,8 @@ func TestTCBCache(t *testing.T) {
169201
} {
170202
t.Run(name, func(t *testing.T) {
171203
fun(t, store, &tcbBundle)
172-
_ = store.Delete([]byte(tcbCacheKey))
204+
_ = store.Delete(tcbBundleCacheKey(TeeTypeSGX))
205+
_ = store.Delete(tcbEvaluationDataNumbersCacheKey(TeeTypeSGX))
173206
})
174207
}
175208
}

‎go/common/sgx/pcs/service.go‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -100,21 +100,23 @@ func (qs *cachingQuoteService) ResolveQuote(ctx context.Context, rawQuote []byte
100100
return nil, fmt.Errorf("PCK verification failed: %w", err)
101101
}
102102

103+
teeType := quote.Header().TeeType()
104+
103105
// Verify the quote so we can catch errors early (the runtime and later consensus layer will
104106
// also do their own verification).
105107
getTcbBundle := func(tcbEvaluationDataNumber uint32) (*TCBBundle, error) {
106108
var fresh *TCBBundle
107109

108-
cached, refresh := qs.cache.check(pckInfo.FMSPC)
110+
cached, refresh := qs.cache.checkBundle(teeType, pckInfo.FMSPC)
109111
if refresh {
110-
if fresh, err = qs.client.GetTCBBundle(ctx, quote.Header().TeeType(), pckInfo.FMSPC, tcbEvaluationDataNumber); err != nil {
112+
if fresh, err = qs.client.GetTCBBundle(ctx, teeType, pckInfo.FMSPC, tcbEvaluationDataNumber); err != nil {
111113
qs.logger.Warn("error downloading TCB refresh",
112114
"err", err,
113115
"tcb_evaluation_data_number", tcbEvaluationDataNumber,
114116
)
115117
}
116118
if err = qs.verifyBundle(quote, quotePolicy, fresh, "fresh"); err == nil {
117-
qs.cache.cache(fresh, pckInfo.FMSPC)
119+
qs.cache.cacheBundle(teeType, fresh, pckInfo.FMSPC)
118120
return fresh, nil
119121
}
120122
qs.logger.Warn("error verifying downloaded TCB refresh",
@@ -137,7 +139,7 @@ func (qs *cachingQuoteService) ResolveQuote(ctx context.Context, rawQuote []byte
137139
}
138140

139141
// If not downloaded yet this time round, try forcing. Any errors are fatal.
140-
if fresh, err = qs.client.GetTCBBundle(ctx, quote.Header().TeeType(), pckInfo.FMSPC, tcbEvaluationDataNumber); err != nil {
142+
if fresh, err = qs.client.GetTCBBundle(ctx, teeType, pckInfo.FMSPC, tcbEvaluationDataNumber); err != nil {
141143
qs.logger.Warn("error downloading TCB",
142144
"err", err,
143145
"tcb_evaluation_data_number", tcbEvaluationDataNumber,
@@ -147,15 +149,22 @@ func (qs *cachingQuoteService) ResolveQuote(ctx context.Context, rawQuote []byte
147149
if err = qs.verifyBundle(quote, quotePolicy, fresh, "downloaded"); err != nil {
148150
return nil, err
149151
}
150-
qs.cache.cache(fresh, pckInfo.FMSPC)
152+
qs.cache.cacheBundle(teeType, fresh, pckInfo.FMSPC)
151153
return fresh, nil
152154
}
153155

154156
// Fetch a list of all available TCB evaluation data numbers and try one by one.
155-
var tcbEvaluationDataNumbers []uint32
156-
tcbEvaluationDataNumbers, err = qs.client.GetTCBEvaluationDataNumbers(ctx, quote.Header().TeeType())
157-
if err != nil {
158-
return nil, err
157+
tcbEvaluationDataNumbers, refresh := qs.cache.checkEvaluationDataNumbers(teeType)
158+
if refresh {
159+
tcbEvaluationDataNumbers, err = qs.client.GetTCBEvaluationDataNumbers(ctx, teeType)
160+
switch err {
161+
case nil:
162+
qs.cache.cacheEvaluationDataNumbers(teeType, tcbEvaluationDataNumbers)
163+
default:
164+
qs.logger.Warn("error downloading TCB evaluation data numbers",
165+
"err", err,
166+
)
167+
}
159168
}
160169
// Ensure they are sorted from highest to lowest.
161170
slices.SortFunc(tcbEvaluationDataNumbers, func(a, b uint32) int {
@@ -171,6 +180,9 @@ func (qs *cachingQuoteService) ResolveQuote(ctx context.Context, rawQuote []byte
171180
if err != nil {
172181
return nil, err
173182
}
183+
if tcbBundle == nil {
184+
return nil, fmt.Errorf("failed to fetch any valid TCB bundles")
185+
}
174186

175187
// Prepare quote structure.
176188
return &QuoteBundle{

0 commit comments

Comments
 (0)