Skip to content

Commit 5a59f86

Browse files
hjgracahjgraca
committed
new example with class library
1 parent 40d3282 commit 5a59f86

10 files changed

Lines changed: 325 additions & 1 deletion

File tree

‎examples/Kafka/Avro/src/Readme.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ examples/Kafka/Avro/src/
1515
├── Function.cs # Entry point for the Lambda function
1616
├── aws-lambda-tools-defaults.json # Default argument settings for AWS Lambda deployment
1717
├── template.yaml # AWS SAM template for deploying the function
18+
├── CustomerProfile.avsc # Avro schema definition file for the data structure used in the Kafka messages
1819
└── kafka-avro-event.json # Sample Avro event to test the function
1920
```
2021

‎examples/Kafka/JsonClassLibrary/src/CustomerProfile.proto‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
syntax = "proto3";
2+
3+
package com.example;
4+
5+
enum PhoneType {
6+
HOME = 0;
7+
WORK = 1;
8+
MOBILE = 2;
9+
}
10+
11+
enum AccountStatus {
12+
ACTIVE = 0;
13+
INACTIVE = 1;
14+
SUSPENDED = 2;
15+
}
16+
17+
// EmailAddress message
18+
message EmailAddress {
19+
string address = 1;
20+
bool verified = 2;
21+
bool primary = 3;
22+
}
23+
24+
// Address message
25+
message Address {
26+
string street = 1;
27+
string city = 2;
28+
string state = 3;
29+
string country = 4;
30+
string zip_code = 5;
31+
}
32+
33+
// PhoneNumber message
34+
message PhoneNumber {
35+
string number = 1;
36+
PhoneType type = 2;
37+
}
38+
39+
// CustomerProfile message
40+
message CustomerProfile {
41+
string user_id = 1;
42+
string full_name = 2;
43+
EmailAddress email = 3;
44+
int32 age = 4;
45+
Address address = 5;
46+
repeated PhoneNumber phone_numbers = 6;
47+
map<string, string> preferences = 7;
48+
AccountStatus account_status = 8;
49+
}

‎examples/Kafka/JsonClassLibrary/src/Function.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
using Amazon.Lambda.Core;
2+
using AWS.Lambda.Powertools.Kafka;
3+
using AWS.Lambda.Powertools.Kafka.Protobuf;
4+
using AWS.Lambda.Powertools.Logging;
5+
using Com.Example;
6+
7+
// Assembly attribute to enable the Lambda function's JSON input to be converted into a .NET class.
8+
[assembly: LambdaSerializer(typeof(PowertoolsKafkaProtobufSerializer))]
9+
10+
namespace ProtoBufClassLibrary;
11+
12+
public class Function
13+
{
14+
/// <summary>
15+
/// A simple function that takes a string and does a ToUpper
16+
/// </summary>
17+
/// <param name="input">The event for the Lambda function handler to process.</param>
18+
/// <param name="context">The ILambdaContext that provides methods for logging and describing the Lambda environment.</param>
19+
/// <returns></returns>
20+
public string FunctionHandler(ConsumerRecords<string, CustomerProfile> records, ILambdaContext context)
21+
{
22+
foreach (var record in records)
23+
{
24+
Logger.LogInformation("Record Value: {@record}", record.Value);
25+
}
26+
27+
return "Processed " + records.Count() + " records";
28+
}
29+
}

‎examples/Kafka/JsonClassLibrary/src/ProtoBufClassLibrary.csproj‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<TargetFramework>net8.0</TargetFramework>
4+
<ImplicitUsings>enable</ImplicitUsings>
5+
<Nullable>enable</Nullable>
6+
<GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
7+
<AWSProjectType>Lambda</AWSProjectType>
8+
<!-- This property makes the build directory similar to a publish directory and helps the AWS .NET Lambda Mock Test Tool find project dependencies. -->
9+
<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
10+
<!-- Generate ready to run images during publishing to improve cold start time. -->
11+
<PublishReadyToRun>true</PublishReadyToRun>
12+
</PropertyGroup>
13+
<ItemGroup>
14+
<PackageReference Include="Amazon.Lambda.Core" Version="2.5.0"/>
15+
<PackageReference Include="Amazon.Lambda.Serialization.SystemTextJson" Version="2.4.4"/>
16+
<PackageReference Include="AWS.Lambda.Powertools.Logging" Version="2.0.0" />
17+
<PackageReference Include="Grpc.Tools" Version="2.72.0">
18+
<PrivateAssets>all</PrivateAssets>
19+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
20+
</PackageReference>
21+
</ItemGroup>
22+
<ItemGroup>
23+
<ProjectReference Include="..\..\..\..\libraries\src\AWS.Lambda.Powertools.Kafka.Protobuf\AWS.Lambda.Powertools.Kafka.Protobuf.csproj" />
24+
</ItemGroup>
25+
<ItemGroup>
26+
<Content Update="kafka-protobuf-event.json">
27+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
28+
</Content>
29+
</ItemGroup>
30+
<ItemGroup>
31+
<Protobuf Include="CustomerProfile.proto">
32+
<GrpcServices>Client</GrpcServices>
33+
<Access>Public</Access>
34+
<ProtoCompile>True</ProtoCompile>
35+
<CompileOutputs>True</CompileOutputs>
36+
<OutputDir>obj/Debug/net8.0/</OutputDir>
37+
<Generator>MSBuild:Compile</Generator>
38+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
39+
</Protobuf>
40+
</ItemGroup>
41+
42+
</Project>

‎examples/Kafka/JsonClassLibrary/src/Readme.md‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
# AWS Powertools for AWS Lambda .NET - Kafka Protobuf Example
2+
3+
This project demonstrates how to use AWS Lambda Powertools for .NET with Amazon MSK (Managed Streaming for Kafka) to process events from Kafka topics.
4+
5+
## Overview
6+
7+
This example showcases a Lambda functions that consume messages from Kafka topics with Protocol Buffers serialization format.
8+
9+
It uses the `AWS.Lambda.Powertools.Kafka.Protobuf` NuGet package to easily deserialize and process Kafka records.
10+
11+
## Project Structure
12+
13+
```bash
14+
examples/Kafka/Protobuf/src/
15+
├── Function.cs # Entry point for the Lambda function
16+
├── aws-lambda-tools-defaults.json # Default argument settings for AWS Lambda deployment
17+
├── template.yaml # AWS SAM template for deploying the function
18+
├── CustomerProfile.proto # Protocol Buffers definition file for the data structure used in the Kafka messages
19+
└── kafka-protobuf-event.json # Sample Protocol Buffers event to test the function
20+
```
21+
22+
## Prerequisites
23+
24+
- [Dotnet](https://dotnet.microsoft.com/en-us/download/dotnet) (dotnet8 or later)
25+
- [AWS SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.html)
26+
- [AWS CLI](https://aws.amazon.com/cli/)
27+
- An AWS account with appropriate permissions
28+
- [Amazon MSK](https://aws.amazon.com/msk/) cluster set up with a topic to consume messages from
29+
- [AWS.Lambda.Powertools.Kafka.Protobuf](https://www.nuget.org/packages/AWS.Lambda.Powertools.Kafka.Protobuf/) NuGet package installed in your project
30+
31+
## Installation
32+
33+
1. Clone the repository:
34+
35+
```bash
36+
git clone https://github.com/aws-powertools/powertools-lambda-dotnet.git
37+
```
38+
39+
2. Navigate to the project directory:
40+
41+
```bash
42+
cd powertools-lambda-dotnet/examples/Kafka/Protobuf/src
43+
```
44+
45+
3. Build the project:
46+
47+
```bash
48+
dotnet build
49+
```
50+
51+
## Deployment
52+
53+
Deploy the application using the AWS SAM CLI:
54+
55+
```bash
56+
sam build
57+
sam deploy --guided
58+
```
59+
60+
Follow the prompts to configure your deployment.
61+
62+
## Protocol Buffers Format
63+
64+
The Protobuf example handles messages serialized with Protocol Buffers. The schema is defined in a `.proto` file (which would need to be created), and the C# code is generated from that schema.
65+
66+
This requires the `Grpc.Tools` package to deserialize the messages correctly.
67+
68+
And update the `.csproj` file to include the `.proto` files.
69+
70+
```xml
71+
<Protobuf Include="CustomerProfile.proto">
72+
<GrpcServices>Client</GrpcServices>
73+
<Access>Public</Access>
74+
<ProtoCompile>True</ProtoCompile>
75+
<CompileOutputs>True</CompileOutputs>
76+
<OutputDir>obj\Debug/net8.0/</OutputDir>
77+
<Generator>MSBuild:Compile</Generator>
78+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
79+
</Protobuf>
80+
```
81+
82+
## Usage Examples
83+
84+
Once deployed, you can test the Lambda function by sending a sample Protocol Buffers event to the configured Kafka topic.
85+
You can use the `kafka-protobuf-event.json` file as a sample event to test the function.
86+
87+
### Testing
88+
89+
You can test the function locally using the AWS SAM CLI (Requires Docker to be installed):
90+
91+
```bash
92+
sam local invoke ProtobufDeserializationFunction --event kafka-protobuf-event.json
93+
```
94+
95+
This command simulates an invocation of the Lambda function with the provided event data.
96+
97+
## How It Works
98+
99+
1. **Event Source**: Configure your Lambda functions with an MSK or self-managed Kafka cluster as an event source.
100+
2. **Deserializing Records**: Powertools handles deserializing the records based on the specified format.
101+
3. **Processing**: Each record is processed within the handler function.
102+
103+
## Event Deserialization
104+
105+
Pass the `PowertoolsKafkaProtobufSerializer` to the `[assembly: LambdaSerializer(typeof(PowertoolsKafkaProtobufSerializer))]`:
106+
107+
```csharp
108+
[assembly: LambdaSerializer(typeof(PowertoolsKafkaProtobufSerializer))]
109+
```
110+
111+
## Configuration
112+
113+
The SAM template (`template.yaml`) defines three Lambda function:
114+
115+
- **ProtobufDeserializationFunction**: Handles Protobuf-formatted Kafka messages
116+
117+
## Customization
118+
119+
To customize the examples:
120+
121+
1. Modify the schema definitions to match your data structures
122+
2. Update the handler logic to process the records according to your requirements
123+
3. Ensure you have the proper `.proto` files and that they are included in your project for Protocol Buffers serialization/deserialization.
124+
125+
## Resources
126+
127+
- [AWS Lambda Powertools for .NET Documentation](https://docs.powertools.aws.dev/lambda/dotnet/)
128+
- [Amazon MSK Documentation](https://docs.aws.amazon.com/msk/)
129+
- [AWS Lambda Developer Guide](https://docs.aws.amazon.com/lambda/)
130+
- [Protocol Buffers Documentation](https://developers.google.com/protocol-buffers)

‎examples/Kafka/JsonClassLibrary/src/aws-lambda-tools-defaults.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"Information": [
3+
"This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.",
4+
"To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.",
5+
"dotnet lambda help",
6+
"All the command line options for the Lambda command can be specified in this file."
7+
],
8+
"profile": "",
9+
"region": "",
10+
"configuration": "Release",
11+
"function-architecture": "x86_64",
12+
"function-runtime": "dotnet8",
13+
"function-memory-size": 512,
14+
"function-timeout": 30,
15+
"function-handler": "ProtoBufClassLibrary::ProtoBufClassLibrary.Function::FunctionHandler"
16+
}

‎examples/Kafka/JsonClassLibrary/src/kafka-protobuf-event.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"eventSource": "aws:kafka",
3+
"eventSourceArn": "arn:aws:kafka:us-east-1:0123456789019:cluster/CustomerCluster/abcd1234-abcd-cafe-abab-9876543210ab-4",
4+
"bootstrapServers": "b-2.demo-cluster-1.a1bcde.c1.kafka.us-east-1.amazonaws.com:9092,b-1.demo-cluster-1.a1bcde.c1.kafka.us-east-1.amazonaws.com:9092",
5+
"records": {
6+
"customer-topic-0": [
7+
{
8+
"topic": "customer-topic",
9+
"partition": 0,
10+
"offset": 15,
11+
"timestamp": 1545084650987,
12+
"timestampType": "CREATE_TIME",
13+
"key": "dXNlcl85NzU0",
14+
"value": "Cgl1c2VyXzk3NTQSDlVzZXIgdXNlcl85NzU0GhgKFHVzZXJfOTc1NEBpY2xvdWQuY29tGAEgNSooCgw5MzQwIE1haW4gU3QSCFNhbiBKb3NlGgJDQSIDVVNBKgUzOTU5NjIQCgwyNDQtNDA3LTg4NzEQAToUCghsYW5ndWFnZRIIZGlzYWJsZWQ6FQoNbm90aWZpY2F0aW9ucxIEZGFyazoTCgh0aW1lem9uZRIHZW5hYmxlZEAC",
15+
"headers": [
16+
{
17+
"headerKey": [104, 101, 97, 100, 101, 114, 86, 97, 108, 117, 101]
18+
}
19+
]
20+
}
21+
]
22+
}
23+
}

‎examples/Kafka/JsonClassLibrary/src/template.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
AWSTemplateFormatVersion: '2010-09-09'
2+
Transform: AWS::Serverless-2016-10-31
3+
Description: >
4+
kafka
5+
6+
Sample SAM Template for kafka
7+
8+
# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst
9+
Globals:
10+
Function:
11+
Timeout: 15
12+
MemorySize: 512
13+
Runtime: dotnet8
14+
15+
Resources:
16+
ProtobufClassLibraryDeserializationFunction:
17+
Type: AWS::Serverless::Function
18+
Properties:
19+
Handler: ProtoBufClassLibrary::ProtoBufClassLibrary.Function::FunctionHandler
20+
Architectures:
21+
- x86_64
22+
Tracing: Active
23+
Environment: # Powertools env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables
24+
Variables:
25+
POWERTOOLS_SERVICE_NAME: PowertoolsHelloWorld
26+
POWERTOOLS_LOG_LEVEL: Info
27+
POWERTOOLS_LOGGER_CASE: PascalCase # Allowed values are: CamelCase, PascalCase and SnakeCase (Default)

‎examples/Kafka/Protobuf/src/Readme.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ This command simulates an invocation of the Lambda function with the provided ev
102102

103103
## Event Deserialization
104104

105-
Pass the `PowertoolsKafkaProtobufSerializer` to the `LambdaBootstrapBuilder.Create()` method to enable JSON deserialization of Kafka records:
105+
Pass the `PowertoolsKafkaProtobufSerializer` to the `LambdaBootstrapBuilder.Create()` method to enable Protobuf deserialization of Kafka records:
106106

107107
```csharp
108108
await LambdaBootstrapBuilder.Create((Func<ConsumerRecords<string, CustomerProfile>, ILambdaContext, string>?)Handler,

‎examples/examples.sln‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Protobuf", "Kafka\Protobuf\
117117
EndProject
118118
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro", "Kafka\Avro\src\Avro.csproj", "{B03F22B2-315C-429B-9CC0-C15BE94CBF77}"
119119
EndProject
120+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProtoBufClassLibrary", "Kafka\JsonClassLibrary\src\ProtoBufClassLibrary.csproj", "{B6B3136D-B739-4917-AD3D-30F19FE12D3F}"
121+
EndProject
120122
Global
121123
GlobalSection(SolutionConfigurationPlatforms) = preSolution
122124
Debug|Any CPU = Debug|Any CPU
@@ -222,6 +224,10 @@ Global
222224
{B03F22B2-315C-429B-9CC0-C15BE94CBF77}.Debug|Any CPU.Build.0 = Debug|Any CPU
223225
{B03F22B2-315C-429B-9CC0-C15BE94CBF77}.Release|Any CPU.ActiveCfg = Release|Any CPU
224226
{B03F22B2-315C-429B-9CC0-C15BE94CBF77}.Release|Any CPU.Build.0 = Release|Any CPU
227+
{B6B3136D-B739-4917-AD3D-30F19FE12D3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
228+
{B6B3136D-B739-4917-AD3D-30F19FE12D3F}.Debug|Any CPU.Build.0 = Debug|Any CPU
229+
{B6B3136D-B739-4917-AD3D-30F19FE12D3F}.Release|Any CPU.ActiveCfg = Release|Any CPU
230+
{B6B3136D-B739-4917-AD3D-30F19FE12D3F}.Release|Any CPU.Build.0 = Release|Any CPU
225231
EndGlobalSection
226232
GlobalSection(NestedProjects) = preSolution
227233
{0CC66DBC-C1DF-4AF6-8EEB-FFED6C578BF4} = {526F1EF7-5A9C-4BFF-ABAE-75992ACD8F78}
@@ -272,5 +278,6 @@ Global
272278
{58EC305E-353A-4996-A541-3CF7FC0EDD80} = {71027B81-CA39-498C-9A50-ADDAFA2AC2F5}
273279
{853F6FE9-1762-4BA3-BAF4-2FCD605B81CF} = {71027B81-CA39-498C-9A50-ADDAFA2AC2F5}
274280
{B03F22B2-315C-429B-9CC0-C15BE94CBF77} = {71027B81-CA39-498C-9A50-ADDAFA2AC2F5}
281+
{B6B3136D-B739-4917-AD3D-30F19FE12D3F} = {71027B81-CA39-498C-9A50-ADDAFA2AC2F5}
275282
EndGlobalSection
276283
EndGlobal

0 commit comments

Comments
 (0)