Skip to content

Commit ec31ede

Browse files
ekjotmultaniekjotmultani
authored
feat(firehose): record cache extraction (#6846)
* refactor(kinesis): extract shared record cache into amplify_record_cache_dart Create amplify_record_cache_dart package with shared caching infrastructure: - RecordCacheException hierarchy (const constructors) - Record/RecordInput models (partitionKey optional, dataSize caller-computed) - RecordStorage base + SqliteRecordStorage, InMemoryRecordStorage, IndexedDbRecordStorage - RecordCacheDatabase (Drift, parameterized dbPrefix) - Sender interface + SendResult (replaces KDS-specific PutRecordsResult) - RecordClient, AutoFlushScheduler, FlushStrategy, FlushData, RecordData, ClearCacheData - Platform resolution (VM/web/stub conditional exports) Update amplify_kinesis_dart to depend on shared package: - KinesisSender implements Sender interface (sendBatch replaces putRecords) - Partition key validation moved from RecordStorage to AmplifyKinesisClient - createKinesisRecordInputNow computes dataSize with partition key - All test imports updated, zero behavioral change
1 parent 95c1669 commit ec31ede

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+885
-517
lines changed

‎.github/workflows/amplify_firehose_dart.yaml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Generated with aft. To update, run: `aft generate workflows`
2+
name: amplify_firehose_dart
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- stable
8+
paths:
9+
- '.github/workflows/amplify_firehose_dart.yaml'
10+
- '.github/workflows/dart_vm.yaml'
11+
- 'packages/amplify_core/lib/**/*.dart'
12+
- 'packages/amplify_core/pubspec.yaml'
13+
- 'packages/amplify_foundation/amplify_foundation_dart/lib/**/*.dart'
14+
- 'packages/amplify_foundation/amplify_foundation_dart/pubspec.yaml'
15+
- 'packages/amplify_foundation/amplify_foundation_dart_bridge/lib/**/*.dart'
16+
- 'packages/amplify_foundation/amplify_foundation_dart_bridge/pubspec.yaml'
17+
- 'packages/amplify_lints/lib/**/*.yaml'
18+
- 'packages/amplify_lints/pubspec.yaml'
19+
- 'packages/aws_common/lib/**/*.dart'
20+
- 'packages/aws_common/pubspec.yaml'
21+
- 'packages/aws_signature_v4/lib/**/*.dart'
22+
- 'packages/aws_signature_v4/pubspec.yaml'
23+
- 'packages/common/amplify_db_common_dart/lib/**/*.dart'
24+
- 'packages/common/amplify_db_common_dart/pubspec.yaml'
25+
- 'packages/kinesis/amplify_firehose_dart/**/*.dart'
26+
- 'packages/kinesis/amplify_firehose_dart/**/*.yaml'
27+
- 'packages/kinesis/amplify_firehose_dart/lib/**/*'
28+
- 'packages/kinesis/amplify_firehose_dart/test/**/*'
29+
- 'packages/smithy/smithy/lib/**/*.dart'
30+
- 'packages/smithy/smithy/pubspec.yaml'
31+
- 'packages/smithy/smithy_aws/lib/**/*.dart'
32+
- 'packages/smithy/smithy_aws/pubspec.yaml'
33+
pull_request:
34+
paths:
35+
- '.github/workflows/amplify_firehose_dart.yaml'
36+
- '.github/workflows/dart_vm.yaml'
37+
- 'packages/amplify_core/lib/**/*.dart'
38+
- 'packages/amplify_core/pubspec.yaml'
39+
- 'packages/amplify_foundation/amplify_foundation_dart/lib/**/*.dart'
40+
- 'packages/amplify_foundation/amplify_foundation_dart/pubspec.yaml'
41+
- 'packages/amplify_foundation/amplify_foundation_dart_bridge/lib/**/*.dart'
42+
- 'packages/amplify_foundation/amplify_foundation_dart_bridge/pubspec.yaml'
43+
- 'packages/amplify_lints/lib/**/*.yaml'
44+
- 'packages/amplify_lints/pubspec.yaml'
45+
- 'packages/aws_common/lib/**/*.dart'
46+
- 'packages/aws_common/pubspec.yaml'
47+
- 'packages/aws_signature_v4/lib/**/*.dart'
48+
- 'packages/aws_signature_v4/pubspec.yaml'
49+
- 'packages/common/amplify_db_common_dart/lib/**/*.dart'
50+
- 'packages/common/amplify_db_common_dart/pubspec.yaml'
51+
- 'packages/kinesis/amplify_firehose_dart/**/*.dart'
52+
- 'packages/kinesis/amplify_firehose_dart/**/*.yaml'
53+
- 'packages/kinesis/amplify_firehose_dart/lib/**/*'
54+
- 'packages/kinesis/amplify_firehose_dart/test/**/*'
55+
- 'packages/smithy/smithy/lib/**/*.dart'
56+
- 'packages/smithy/smithy/pubspec.yaml'
57+
- 'packages/smithy/smithy_aws/lib/**/*.dart'
58+
- 'packages/smithy/smithy_aws/pubspec.yaml'
59+
schedule:
60+
- cron: "0 13 * * 1" # Every Monday at 06:00 PST
61+
workflow_dispatch:
62+
defaults:
63+
run:
64+
shell: bash
65+
66+
# These permissions are needed to interact with GitHub's OIDC Token endpoint.
67+
permissions:
68+
id-token: write
69+
contents: read
70+
71+
# Cancels in-progress job when there is another push to same ref.
72+
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-only-cancel-in-progress-jobs-or-runs-for-the-current-workflow
73+
concurrency:
74+
group: ${{ github.workflow }}-${{ github.ref }}
75+
cancel-in-progress: true
76+
77+
jobs:
78+
test:
79+
uses: ./.github/workflows/dart_vm.yaml
80+
secrets: inherit
81+
with:
82+
package-name: amplify_firehose_dart
83+
working-directory: packages/kinesis/amplify_firehose_dart

‎.github/workflows/amplify_kinesis.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ on:
2828
- 'packages/kinesis/amplify_kinesis/test/**/*'
2929
- 'packages/kinesis/amplify_kinesis_dart/lib/**/*.dart'
3030
- 'packages/kinesis/amplify_kinesis_dart/pubspec.yaml'
31+
- 'packages/kinesis/amplify_record_cache_dart/lib/**/*.dart'
32+
- 'packages/kinesis/amplify_record_cache_dart/pubspec.yaml'
3133
- 'packages/smithy/smithy/lib/**/*.dart'
3234
- 'packages/smithy/smithy/pubspec.yaml'
3335
- 'packages/smithy/smithy_aws/lib/**/*.dart'
@@ -56,6 +58,8 @@ on:
5658
- 'packages/kinesis/amplify_kinesis/test/**/*'
5759
- 'packages/kinesis/amplify_kinesis_dart/lib/**/*.dart'
5860
- 'packages/kinesis/amplify_kinesis_dart/pubspec.yaml'
61+
- 'packages/kinesis/amplify_record_cache_dart/lib/**/*.dart'
62+
- 'packages/kinesis/amplify_record_cache_dart/pubspec.yaml'
5963
- 'packages/smithy/smithy/lib/**/*.dart'
6064
- 'packages/smithy/smithy/pubspec.yaml'
6165
- 'packages/smithy/smithy_aws/lib/**/*.dart'

‎.github/workflows/amplify_kinesis_dart.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ on:
2727
- 'packages/kinesis/amplify_kinesis_dart/**/*.yaml'
2828
- 'packages/kinesis/amplify_kinesis_dart/lib/**/*'
2929
- 'packages/kinesis/amplify_kinesis_dart/test/**/*'
30+
- 'packages/kinesis/amplify_record_cache_dart/lib/**/*.dart'
31+
- 'packages/kinesis/amplify_record_cache_dart/pubspec.yaml'
3032
- 'packages/smithy/smithy/lib/**/*.dart'
3133
- 'packages/smithy/smithy/pubspec.yaml'
3234
- 'packages/smithy/smithy_aws/lib/**/*.dart'
@@ -54,6 +56,8 @@ on:
5456
- 'packages/kinesis/amplify_kinesis_dart/**/*.yaml'
5557
- 'packages/kinesis/amplify_kinesis_dart/lib/**/*'
5658
- 'packages/kinesis/amplify_kinesis_dart/test/**/*'
59+
- 'packages/kinesis/amplify_record_cache_dart/lib/**/*.dart'
60+
- 'packages/kinesis/amplify_record_cache_dart/pubspec.yaml'
5761
- 'packages/smithy/smithy/lib/**/*.dart'
5862
- 'packages/smithy/smithy/pubspec.yaml'
5963
- 'packages/smithy/smithy_aws/lib/**/*.dart'

‎.github/workflows/amplify_kinesis_example.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ on:
5757
- 'packages/kinesis/amplify_kinesis/pubspec.yaml'
5858
- 'packages/kinesis/amplify_kinesis_dart/lib/**/*.dart'
5959
- 'packages/kinesis/amplify_kinesis_dart/pubspec.yaml'
60+
- 'packages/kinesis/amplify_record_cache_dart/lib/**/*.dart'
61+
- 'packages/kinesis/amplify_record_cache_dart/pubspec.yaml'
6062
- 'packages/secure_storage/amplify_secure_storage/android/**/*'
6163
- 'packages/secure_storage/amplify_secure_storage/ios/**/*'
6264
- 'packages/secure_storage/amplify_secure_storage/lib/**/*.dart'
@@ -127,6 +129,8 @@ on:
127129
- 'packages/kinesis/amplify_kinesis/pubspec.yaml'
128130
- 'packages/kinesis/amplify_kinesis_dart/lib/**/*.dart'
129131
- 'packages/kinesis/amplify_kinesis_dart/pubspec.yaml'
132+
- 'packages/kinesis/amplify_record_cache_dart/lib/**/*.dart'
133+
- 'packages/kinesis/amplify_record_cache_dart/pubspec.yaml'
130134
- 'packages/secure_storage/amplify_secure_storage/android/**/*'
131135
- 'packages/secure_storage/amplify_secure_storage/ios/**/*'
132136
- 'packages/secure_storage/amplify_secure_storage/lib/**/*.dart'

‎.github/workflows/amplify_record_cache_dart.yaml‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# Generated with aft. To update, run: `aft generate workflows`
2+
name: amplify_record_cache_dart
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- stable
8+
paths:
9+
- '.github/workflows/amplify_record_cache_dart.yaml'
10+
- '.github/workflows/dart_vm.yaml'
11+
- 'packages/amplify_core/lib/**/*.dart'
12+
- 'packages/amplify_core/pubspec.yaml'
13+
- 'packages/amplify_foundation/amplify_foundation_dart/lib/**/*.dart'
14+
- 'packages/amplify_foundation/amplify_foundation_dart/pubspec.yaml'
15+
- 'packages/amplify_lints/lib/**/*.yaml'
16+
- 'packages/amplify_lints/pubspec.yaml'
17+
- 'packages/aws_common/lib/**/*.dart'
18+
- 'packages/aws_common/pubspec.yaml'
19+
- 'packages/aws_signature_v4/lib/**/*.dart'
20+
- 'packages/aws_signature_v4/pubspec.yaml'
21+
- 'packages/common/amplify_db_common_dart/lib/**/*.dart'
22+
- 'packages/common/amplify_db_common_dart/pubspec.yaml'
23+
- 'packages/kinesis/amplify_record_cache_dart/**/*.dart'
24+
- 'packages/kinesis/amplify_record_cache_dart/**/*.yaml'
25+
- 'packages/kinesis/amplify_record_cache_dart/lib/**/*'
26+
- 'packages/kinesis/amplify_record_cache_dart/test/**/*'
27+
- 'packages/smithy/smithy/lib/**/*.dart'
28+
- 'packages/smithy/smithy/pubspec.yaml'
29+
pull_request:
30+
paths:
31+
- '.github/workflows/amplify_record_cache_dart.yaml'
32+
- '.github/workflows/dart_vm.yaml'
33+
- 'packages/amplify_core/lib/**/*.dart'
34+
- 'packages/amplify_core/pubspec.yaml'
35+
- 'packages/amplify_foundation/amplify_foundation_dart/lib/**/*.dart'
36+
- 'packages/amplify_foundation/amplify_foundation_dart/pubspec.yaml'
37+
- 'packages/amplify_lints/lib/**/*.yaml'
38+
- 'packages/amplify_lints/pubspec.yaml'
39+
- 'packages/aws_common/lib/**/*.dart'
40+
- 'packages/aws_common/pubspec.yaml'
41+
- 'packages/aws_signature_v4/lib/**/*.dart'
42+
- 'packages/aws_signature_v4/pubspec.yaml'
43+
- 'packages/common/amplify_db_common_dart/lib/**/*.dart'
44+
- 'packages/common/amplify_db_common_dart/pubspec.yaml'
45+
- 'packages/kinesis/amplify_record_cache_dart/**/*.dart'
46+
- 'packages/kinesis/amplify_record_cache_dart/**/*.yaml'
47+
- 'packages/kinesis/amplify_record_cache_dart/lib/**/*'
48+
- 'packages/kinesis/amplify_record_cache_dart/test/**/*'
49+
- 'packages/smithy/smithy/lib/**/*.dart'
50+
- 'packages/smithy/smithy/pubspec.yaml'
51+
schedule:
52+
- cron: "0 13 * * 1" # Every Monday at 06:00 PST
53+
workflow_dispatch:
54+
defaults:
55+
run:
56+
shell: bash
57+
58+
# These permissions are needed to interact with GitHub's OIDC Token endpoint.
59+
permissions:
60+
id-token: write
61+
contents: read
62+
63+
# Cancels in-progress job when there is another push to same ref.
64+
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-only-cancel-in-progress-jobs-or-runs-for-the-current-workflow
65+
concurrency:
66+
group: ${{ github.workflow }}-${{ github.ref }}
67+
cancel-in-progress: true
68+
69+
jobs:
70+
test:
71+
uses: ./.github/workflows/dart_vm.yaml
72+
secrets: inherit
73+
with:
74+
package-name: amplify_record_cache_dart
75+
working-directory: packages/kinesis/amplify_record_cache_dart

‎packages/kinesis/amplify_kinesis_dart/lib/amplify_kinesis_dart.dart‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,22 @@
44
/// Amplify Kinesis Data Streams client for Dart.
55
library;
66

7+
// Re-export shared types used in the public API
8+
export 'package:amplify_record_cache_dart/amplify_record_cache_dart.dart'
9+
show
10+
FlushStrategy,
11+
FlushInterval,
12+
FlushNone,
13+
FlushData,
14+
RecordData,
15+
ClearCacheData;
16+
717
// Main client
818
export 'src/amplify_kinesis_client.dart';
919
// Options
1020
export 'src/amplify_kinesis_client_options.dart';
1121
// Exceptions
1222
export 'src/exception/amplify_kinesis_exception.dart';
13-
// Flush strategies
14-
export 'src/flush_strategy/flush_strategy.dart';
15-
// Return types
16-
export 'src/model/clear_cache_data.dart';
17-
export 'src/model/flush_data.dart';
18-
export 'src/model/record_data.dart';
1923
// SDK client (for escape hatch)
2024
export 'src/sdk/kinesis.dart'
2125
show

‎packages/kinesis/amplify_kinesis_dart/lib/src/amplify_kinesis_client.dart‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,12 @@ import 'package:amplify_foundation_dart/amplify_foundation_dart.dart'
1212
import 'package:amplify_foundation_dart_bridge/amplify_foundation_dart_bridge.dart';
1313
import 'package:amplify_kinesis_dart/src/amplify_kinesis_client_options.dart';
1414
import 'package:amplify_kinesis_dart/src/exception/amplify_kinesis_exception.dart';
15-
import 'package:amplify_kinesis_dart/src/flush_strategy/flush_strategy.dart';
16-
import 'package:amplify_kinesis_dart/src/impl/auto_flush_scheduler.dart';
1715
import 'package:amplify_kinesis_dart/src/impl/kinesis_record.dart';
1816
import 'package:amplify_kinesis_dart/src/impl/kinesis_sender.dart';
19-
import 'package:amplify_kinesis_dart/src/impl/record_client.dart';
20-
import 'package:amplify_kinesis_dart/src/impl/storage/platform/record_storage_platform.dart';
21-
import 'package:amplify_kinesis_dart/src/model/clear_cache_data.dart';
22-
import 'package:amplify_kinesis_dart/src/model/flush_data.dart';
23-
import 'package:amplify_kinesis_dart/src/model/record_data.dart';
17+
import 'package:amplify_kinesis_dart/src/kinesis_limits.dart' as limits;
2418
import 'package:amplify_kinesis_dart/src/sdk/kinesis.dart';
2519
import 'package:amplify_kinesis_dart/src/version.dart';
20+
import 'package:amplify_record_cache_dart/amplify_record_cache_dart.dart';
2621
import 'package:smithy/smithy.dart' show WithUserAgent;
2722

2823
/// User agent component identifying this library.
@@ -114,6 +109,11 @@ class AmplifyKinesisClient {
114109
identifier: region,
115110
storagePath: storagePath,
116111
maxCacheBytes: opts.cacheMaxBytes,
112+
maxRecordsPerBatch: limits.maxRecordsPerStream,
113+
maxBytesPerBatch: limits.maxPutRecordsSizeBytes,
114+
maxRecordSizeBytes: limits.maxRecordSizeBytes,
115+
dbPrefix: 'kinesis_records',
116+
storeName: 'kinesis_records',
117117
);
118118

119119
final kinesisClient = KinesisClient(
@@ -208,8 +208,21 @@ class AmplifyKinesisClient {
208208
_logger.debug('Record collection is disabled, dropping record');
209209
return const Result.ok(RecordData());
210210
}
211+
// KDS-specific partition key validation
212+
final codePoints = partitionKey.runes.length;
213+
if (codePoints == 0 || codePoints > limits.maxPartitionKeyLength) {
214+
return Result.error(
215+
KinesisValidationException(
216+
'Partition key length ($codePoints) is outside the allowed '
217+
'range of 1-${limits.maxPartitionKeyLength} characters.',
218+
recoverySuggestion:
219+
'Use a partition key between 1 and '
220+
'${limits.maxPartitionKeyLength} characters.',
221+
),
222+
);
223+
}
211224
_logger.verbose('Recording to stream: $streamName');
212-
final kinesisRecord = RecordInput.now(
225+
final kinesisRecord = createKinesisRecordInputNow(
213226
data: data,
214227
partitionKey: partitionKey,
215228
streamName: streamName,

‎packages/kinesis/amplify_kinesis_dart/lib/src/amplify_kinesis_client_options.dart‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
import 'package:amplify_kinesis_dart/src/amplify_kinesis_client.dart'
55
show AmplifyKinesisClient;
6-
import 'package:amplify_kinesis_dart/src/flush_strategy/flush_strategy.dart';
6+
import 'package:amplify_record_cache_dart/amplify_record_cache_dart.dart';
77

88
/// {@template amplify_kinesis.amplify_kinesis_client_options}
99
/// Configuration options for [AmplifyKinesisClient].

‎packages/kinesis/amplify_kinesis_dart/lib/src/exception/amplify_kinesis_exception.dart‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,7 @@
22
// SPDX-License-Identifier: Apache-2.0
33

44
import 'package:amplify_core/amplify_core.dart';
5-
import 'package:amplify_kinesis_dart/src/exception/record_cache_exception.dart';
6-
7-
/// Default recovery suggestion for errors.
8-
const String defaultRecoverySuggestion =
9-
'Inspect the underlying error for more details.';
5+
import 'package:amplify_record_cache_dart/amplify_record_cache_dart.dart';
106

117
/// {@template amplify_kinesis.amplify_kinesis_exception}
128
/// Base exception for Amplify Kinesis Data Streams errors.

0 commit comments

Comments
 (0)