Skip to content

Commit 88cf4d5

Browse files
mateon01mateon01
committed
Update OpenSearch UBI implementation with webapp backend and deployment scripts
1 parent 686dc1f commit 88cf4d5

8 files changed

Lines changed: 593 additions & 6 deletions

File tree

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,6 @@ env/
8686
# === Java / Gradle ===
8787
.gradle/
8888
build/
89-
bin/
9089
!**/src/main/**/bin/
9190
!**/src/test/**/bin/
9291
*.class

‎opensearch/opensearch_ubi/.gitignore‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,33 @@ node_modules/
1010
*.d.ts
1111
!jest.config.js
1212

13-
# Lambda layer build
14-
lambda/layers/dependencies/python/*
13+
# Lambda layer dependencies (installed during deploy)
14+
lambda/layers/dependencies/python/
1515
!lambda/layers/dependencies/python/.gitkeep
16-
lambda/webapp-backend/
16+
17+
# Lambda webapp-backend dependencies (installed during deploy)
18+
# Exclude all except source files
19+
lambda/webapp-backend/*
20+
!lambda/webapp-backend/main.py
21+
!lambda/webapp-backend/requirements.txt
22+
!lambda/webapp-backend/.gitkeep
23+
24+
# Lambda function dependencies (if any)
25+
lambda/functions/**/package/
26+
lambda/functions/**/__pycache__/
27+
28+
# Python
29+
__pycache__/
30+
*.py[cod]
31+
*$py.class
32+
*.so
33+
.Python
34+
*.egg-info/
35+
.eggs/
36+
*.egg
37+
.dist-info/
38+
39+
# Archives
1740
*.zip
1841

1942
# IDE
@@ -33,3 +56,7 @@ npm-debug.log*
3356
# Test coverage
3457
coverage/
3558
.nyc_output/
59+
60+
# Environment
61+
.env
62+
.env.local

‎opensearch/opensearch_ubi/bin/app.ts‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
#!/usr/bin/env node
2+
/**
3+
* CDK App Entry Point - UBI to LTR Pipeline Infrastructure.
4+
*
5+
* This app deploys the complete infrastructure for:
6+
* 1. OpenSearch Service domain with UBI plugin support
7+
* 2. OSI (OpenSearch Ingestion) pipeline configurations
8+
* 3. S3 buckets for data storage
9+
* 4. Lambda functions for judgment generation
10+
* 5. Step Functions for pipeline orchestration
11+
*
12+
* Usage:
13+
* npm run build
14+
* cdk deploy --all
15+
*
16+
* Environment variables:
17+
* CDK_DEFAULT_ACCOUNT - AWS account ID
18+
* CDK_DEFAULT_REGION - AWS region (default: us-east-1)
19+
* ENV_PREFIX - Environment prefix (default: dev)
20+
*/
21+
22+
import 'source-map-support/register';
23+
import * as cdk from 'aws-cdk-lib';
24+
import { IamStack } from '../lib/iam-stack';
25+
import { StorageStack } from '../lib/storage-stack';
26+
import { OpenSearchStack } from '../lib/opensearch-stack';
27+
import { OsiStack } from '../lib/osi-stack';
28+
import { ProcessingStack } from '../lib/processing-stack';
29+
import { SetupStack } from '../lib/setup-stack';
30+
import { WebappStack } from '../lib/webapp-stack';
31+
32+
const app = new cdk.App();
33+
34+
// ===========================================================================
35+
// Configuration
36+
// ===========================================================================
37+
const envPrefix = app.node.tryGetContext('envPrefix') ?? process.env.ENV_PREFIX ?? 'dev';
38+
const region = app.node.tryGetContext('region') ?? process.env.CDK_DEFAULT_REGION ?? 'us-east-1';
39+
const account = process.env.CDK_DEFAULT_ACCOUNT ?? process.env.AWS_ACCOUNT_ID;
40+
41+
// OpenSearch configuration
42+
const openSearchConfig = {
43+
version: app.node.tryGetContext('opensearchVersion') ?? '3.3',
44+
instanceType: app.node.tryGetContext('instanceType') ?? 't3.small.search',
45+
instanceCount: parseInt(app.node.tryGetContext('instanceCount') ?? '1'),
46+
ebsVolumeSize: parseInt(app.node.tryGetContext('ebsVolumeSize') ?? '20'),
47+
dedicatedMasterEnabled: app.node.tryGetContext('dedicatedMaster') === 'true',
48+
multiAzEnabled: app.node.tryGetContext('multiAz') === 'true',
49+
};
50+
51+
// Processing configuration
52+
const processingConfig = {
53+
bedrockModelId: app.node.tryGetContext('bedrockModelId') ?? 'us.anthropic.claude-sonnet-4-5-20250929-v1:0',
54+
enableSchedule: app.node.tryGetContext('enableSchedule') === 'true',
55+
scheduleExpression: app.node.tryGetContext('scheduleExpression') ?? 'rate(1 day)',
56+
};
57+
58+
const env: cdk.Environment = {
59+
account,
60+
region,
61+
};
62+
63+
console.log(`
64+
===========================================================================
65+
UBI to LTR Pipeline Infrastructure
66+
===========================================================================
67+
Environment: ${envPrefix}
68+
Region: ${region}
69+
Account: ${account ?? 'Not specified (will use default)'}
70+
71+
OpenSearch Configuration:
72+
- Version: ${openSearchConfig.version}
73+
- Instance Type: ${openSearchConfig.instanceType}
74+
- Instance Count: ${openSearchConfig.instanceCount}
75+
- EBS Volume: ${openSearchConfig.ebsVolumeSize} GB
76+
77+
Processing Configuration:
78+
- Bedrock Model: ${processingConfig.bedrockModelId}
79+
- Scheduled: ${processingConfig.enableSchedule}
80+
===========================================================================
81+
`);
82+
83+
// ===========================================================================
84+
// Stack Deployment
85+
// ===========================================================================
86+
87+
// 1. Storage Stack (S3 buckets)
88+
const storageStack = new StorageStack(app, `UbiLtr-StorageStack-${envPrefix}`, {
89+
stackName: `${envPrefix}-ubi-ltr-storage`,
90+
description: 'S3 buckets for UBI-LTR pipeline data storage',
91+
env,
92+
envPrefix,
93+
iaTransitionDays: 30,
94+
retentionDays: 365,
95+
enableVersioning: true,
96+
});
97+
98+
// 2. IAM Stack (roles and policies)
99+
const iamStack = new IamStack(app, `UbiLtr-IamStack-${envPrefix}`, {
100+
stackName: `${envPrefix}-ubi-ltr-iam`,
101+
description: 'IAM roles and policies for UBI-LTR pipeline',
102+
env,
103+
envPrefix,
104+
dataBucketArn: storageStack.dataBucket.bucketArn,
105+
});
106+
107+
// Add dependency
108+
iamStack.addDependency(storageStack);
109+
110+
// 3. OpenSearch Stack (domain + OSI configs)
111+
const openSearchStack = new OpenSearchStack(app, `UbiLtr-OpenSearchStack-${envPrefix}`, {
112+
stackName: `${envPrefix}-ubi-ltr-opensearch`,
113+
description: 'OpenSearch Service domain with UBI support',
114+
env,
115+
envPrefix,
116+
openSearchVersion: openSearchConfig.version,
117+
instanceType: openSearchConfig.instanceType,
118+
instanceCount: openSearchConfig.instanceCount,
119+
ebsVolumeSize: openSearchConfig.ebsVolumeSize,
120+
dedicatedMasterEnabled: openSearchConfig.dedicatedMasterEnabled,
121+
multiAzEnabled: openSearchConfig.multiAzEnabled,
122+
osiPipelineRoleArn: iamStack.osiPipelineRole.roleArn,
123+
dlqBucketArn: storageStack.dlqBucket.bucketArn,
124+
});
125+
126+
// Add dependencies
127+
openSearchStack.addDependency(iamStack);
128+
openSearchStack.addDependency(storageStack);
129+
130+
// 3.5 OSI Stack (OpenSearch Ingestion pipelines for UBI)
131+
const osiStack = new OsiStack(app, `UbiLtr-OsiStack-${envPrefix}`, {
132+
stackName: `${envPrefix}-ubi-ltr-osi`,
133+
description: 'OSI pipelines for UBI queries and events ingestion',
134+
env,
135+
envPrefix,
136+
openSearchEndpoint: openSearchStack.domainEndpoint,
137+
osiPipelineRoleArn: iamStack.osiPipelineRole.roleArn,
138+
dlqBucketName: storageStack.dlqBucket.bucketName,
139+
});
140+
141+
// Add dependencies - OSI needs OpenSearch domain endpoint and IAM role
142+
osiStack.addDependency(openSearchStack);
143+
osiStack.addDependency(iamStack);
144+
osiStack.addDependency(storageStack);
145+
146+
// 4. Processing Stack (Lambda + Step Functions)
147+
// Pass role ARNs instead of role objects to avoid circular dependencies
148+
const processingStack = new ProcessingStack(app, `UbiLtr-ProcessingStack-${envPrefix}`, {
149+
stackName: `${envPrefix}-ubi-ltr-processing`,
150+
description: 'Lambda functions and Step Functions for UBI-LTR pipeline',
151+
env,
152+
envPrefix,
153+
openSearchEndpoint: openSearchStack.domainEndpoint,
154+
masterUserSecretArn: openSearchStack.masterUserSecret.secretArn,
155+
dataBucket: storageStack.dataBucket,
156+
lambdaExecutionRoleArn: iamStack.lambdaExecutionRole.roleArn,
157+
stepFunctionsRoleArn: iamStack.stepFunctionsRole.roleArn,
158+
bedrockModelId: processingConfig.bedrockModelId,
159+
enableSchedule: processingConfig.enableSchedule,
160+
scheduleExpression: processingConfig.scheduleExpression,
161+
});
162+
163+
// Add dependencies
164+
processingStack.addDependency(openSearchStack);
165+
processingStack.addDependency(iamStack);
166+
processingStack.addDependency(storageStack);
167+
168+
// 5. Setup Stack (OpenSearch initialization)
169+
// This stack creates indices, role mappings, and sample data
170+
const setupStack = new SetupStack(app, `UbiLtr-SetupStack-${envPrefix}`, {
171+
stackName: `${envPrefix}-ubi-ltr-setup`,
172+
description: 'OpenSearch initialization with indices, role mappings, and sample data',
173+
env,
174+
envPrefix,
175+
openSearchEndpoint: openSearchStack.domainEndpoint,
176+
masterUserSecretArn: openSearchStack.masterUserSecret.secretArn,
177+
lambdaExecutionRoleArn: iamStack.lambdaExecutionRole.roleArn,
178+
osiPipelineRoleArn: iamStack.osiPipelineRole.roleArn,
179+
});
180+
181+
// Add dependencies - setup must run after OpenSearch and IAM are ready
182+
setupStack.addDependency(openSearchStack);
183+
setupStack.addDependency(iamStack);
184+
185+
// 6. Webapp Stack (Lambda API + Frontend)
186+
// Frontend deployment is enabled by default - requires: cd webapp/frontend && npm run build
187+
const deployFrontend = app.node.tryGetContext('deployFrontend') !== 'false';
188+
const webappStack = new WebappStack(app, `UbiLtr-WebappStack-${envPrefix}`, {
189+
stackName: `${envPrefix}-ubi-ltr-webapp`,
190+
description: 'API Gateway, Lambda backend, and S3/CloudFront frontend for UBI webapp',
191+
env,
192+
envPrefix,
193+
masterUserSecretArn: openSearchStack.masterUserSecret.secretArn,
194+
deployFrontend,
195+
});
196+
197+
// Add dependencies - webapp needs OpenSearch to be ready (for SSM parameters)
198+
webappStack.addDependency(openSearchStack);
199+
200+
// ===========================================================================
201+
// App Synthesis
202+
// ===========================================================================
203+
app.synth();

‎opensearch/opensearch_ubi/deploy.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,22 @@ check_prerequisites() {
9595
print_success "npm: $(npm --version)"
9696
fi
9797

98+
# Check Python (for Lambda dependencies)
99+
if ! command -v python3 &> /dev/null; then
100+
missing+=("python3")
101+
else
102+
PYTHON_VERSION=$(python3 --version 2>&1 | sed 's/Python //')
103+
print_success "Python: $PYTHON_VERSION"
104+
fi
105+
106+
# Check pip
107+
if ! command -v pip3 &> /dev/null && ! python3 -m pip --version &> /dev/null; then
108+
missing+=("pip3")
109+
else
110+
PIP_VERSION=$(python3 -m pip --version 2>&1 | head -n1)
111+
print_success "pip: $PIP_VERSION"
112+
fi
113+
98114
# Check AWS credentials
99115
if ! aws sts get-caller-identity &> /dev/null; then
100116
print_error "AWS credentials not configured or invalid"
@@ -110,6 +126,7 @@ check_prerequisites() {
110126
echo "Please install missing dependencies:"
111127
echo " - AWS CLI: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html"
112128
echo " - Node.js: https://nodejs.org/ (v18 or higher)"
129+
echo " - Python 3: https://www.python.org/downloads/ (v3.9 or higher)"
113130
echo " - AWS credentials: aws configure"
114131
exit 1
115132
fi
@@ -130,6 +147,59 @@ install_dependencies() {
130147
cd "$SCRIPT_DIR"
131148
}
132149

150+
install_python_dependencies() {
151+
print_step "Installing Python dependencies for Lambda..."
152+
153+
# Install webapp-backend dependencies
154+
local WEBAPP_DIR="$SCRIPT_DIR/lambda/webapp-backend"
155+
if [ -f "$WEBAPP_DIR/requirements.txt" ]; then
156+
echo "Installing webapp-backend dependencies..."
157+
158+
# Clean up old packages (keep only main.py and requirements.txt)
159+
find "$WEBAPP_DIR" -mindepth 1 -maxdepth 1 \
160+
! -name "main.py" \
161+
! -name "requirements.txt" \
162+
! -name ".gitkeep" \
163+
-exec rm -rf {} + 2>/dev/null || true
164+
165+
# Install dependencies to the directory
166+
python3 -m pip install \
167+
--quiet \
168+
--target "$WEBAPP_DIR" \
169+
-r "$WEBAPP_DIR/requirements.txt"
170+
171+
# Count installed packages
172+
PKG_COUNT=$(find "$WEBAPP_DIR" -maxdepth 1 -type d | wc -l | tr -d ' ')
173+
print_success "webapp-backend dependencies installed ($PKG_COUNT packages)"
174+
else
175+
print_warning "webapp-backend/requirements.txt not found, skipping"
176+
fi
177+
178+
# Install layer dependencies
179+
local LAYER_DIR="$SCRIPT_DIR/lambda/layers/dependencies"
180+
if [ -f "$LAYER_DIR/requirements.txt" ]; then
181+
echo "Installing Lambda layer dependencies..."
182+
183+
# Create python directory for layer
184+
mkdir -p "$LAYER_DIR/python"
185+
186+
# Clean up old packages
187+
rm -rf "$LAYER_DIR/python/"* 2>/dev/null || true
188+
189+
# Install dependencies to python/ directory (Lambda layer structure)
190+
python3 -m pip install \
191+
--quiet \
192+
--target "$LAYER_DIR/python" \
193+
-r "$LAYER_DIR/requirements.txt"
194+
195+
# Count installed packages
196+
PKG_COUNT=$(find "$LAYER_DIR/python" -maxdepth 1 -type d | wc -l | tr -d ' ')
197+
print_success "Lambda layer dependencies installed ($PKG_COUNT packages)"
198+
else
199+
print_warning "lambda/layers/dependencies/requirements.txt not found, skipping"
200+
fi
201+
}
202+
133203
build_frontend() {
134204
print_step "Building frontend application..."
135205

@@ -322,6 +392,7 @@ fi
322392
# Execute deployment steps
323393
check_prerequisites
324394
install_dependencies
395+
install_python_dependencies
325396
build_frontend
326397
bootstrap_cdk
327398
deploy_stacks

‎opensearch/opensearch_ubi/lambda/layers/dependencies/python/.gitkeep‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎opensearch/opensearch_ubi/lambda/webapp-backend/.gitkeep‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# This file ensures the directory is tracked by git
2+
# Python dependencies are installed during deployment via requirements.txt

0 commit comments

Comments
 (0)