Skip to content

Commit 79ecda5

Browse files
AbdoKnbGitAbdoKnbGit
authored
Merge branch 'dev' into feature/simplify-skill
2 parents f51fb9f + 1ed1415 commit 79ecda5

69 files changed

Lines changed: 8937 additions & 46 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.dockerignore‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
.git
2+
.opencode
3+
.sst
4+
.turbo
5+
.wrangler
6+
node_modules
7+
**/node_modules
8+
**/.output
9+
**/dist
10+
**/.turbo
11+
**/.vite
12+
**/coverage

‎.github/workflows/deploy.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,15 @@ on:
99

1010
concurrency: ${{ github.workflow }}-${{ github.ref }}
1111

12+
permissions:
13+
contents: read
14+
id-token: write
15+
1216
jobs:
1317
deploy:
18+
if: github.repository == 'anomalyco/opencode' && (github.ref_name == 'dev' || github.ref_name == 'production')
1419
runs-on: ubuntu-latest
20+
environment: ${{ github.ref_name }}
1521
steps:
1622
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
1723

@@ -21,6 +27,12 @@ jobs:
2127
with:
2228
node-version: "24"
2329

30+
- uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
31+
with:
32+
role-to-assume: ${{ vars.AWS_DEPLOY_ROLE_ARN }}
33+
role-session-name: opencode-${{ github.run_id }}
34+
aws-region: us-east-1
35+
2436
- run: bun sst deploy --stage=${{ github.ref_name }}
2537
env:
2638
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}

‎bun.lock‎

Lines changed: 112 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎infra/app.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ export const api = new sst.cloudflare.Worker("Api", {
3030
transform: {
3131
worker: (args) => {
3232
args.logpush = true
33-
if ($app.stage === "vimtor") return
33+
if ($app.stage === "vimtor" || $app.stage === "adam") return
3434
args.bindings = $resolve(args.bindings).apply((bindings) => [
3535
...bindings,
3636
{

‎infra/console.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1-
import { domain } from "./stage"
1+
import { deployAws, domain } from "./stage"
22
import { EMAILOCTOPUS_API_KEY } from "./app"
33
import { SECRET } from "./secret"
44

5+
const lake = deployAws ? await import("./lake") : undefined
6+
57
////////////////
68
// DATABASE
79
////////////////
@@ -240,7 +242,7 @@ const SALESFORCE_INSTANCE_URL = new sst.Secret("SALESFORCE_INSTANCE_URL")
240242

241243
const logProcessor = new sst.cloudflare.Worker("LogProcessor", {
242244
handler: "packages/console/function/src/log-processor.ts",
243-
link: [new sst.Secret("HONEYCOMB_API_KEY")],
245+
link: [SECRET.HoneycombApiKey, ...(lake?.lakeIngest ? [lake.lakeIngest] : [])],
244246
})
245247

246248
new sst.cloudflare.x.SolidStart("Console", {

‎infra/lake.ts‎

Lines changed: 322 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,322 @@
1+
import { domain } from "./stage"
2+
3+
const current = aws.getCallerIdentityOutput({})
4+
const partition = aws.getPartitionOutput({})
5+
const region = aws.getRegionOutput({})
6+
7+
const tableBucketName = `opencode-${$app.stage}-lake`
8+
const glueCatalogName = "s3tablescatalog"
9+
const glueCatalogArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:catalog`
10+
const glueS3TablesCatalogArn = $interpolate`${glueCatalogArn}/${glueCatalogName}`
11+
const glueS3TablesChildCatalogArn = $interpolate`${glueS3TablesCatalogArn}/${tableBucketName}`
12+
const glueS3TablesDatabaseWildcardArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/${glueCatalogName}/${tableBucketName}/*`
13+
const glueS3TablesTableWildcardArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/${tableBucketName}/*/*`
14+
const s3TablesBucketWildcardArn = $interpolate`arn:${partition.partition}:s3tables:${region.region}:${current.accountId}:bucket/*`
15+
16+
export const tableBucket = new aws.s3tables.TableBucket("LakeTableBucket", {
17+
name: tableBucketName,
18+
forceDestroy: $app.stage !== "production",
19+
})
20+
21+
const s3TablesCatalog = new aws.cloudcontrol.Resource(
22+
"LakeS3TablesCatalog",
23+
{
24+
typeName: "AWS::Glue::Catalog",
25+
desiredState: $jsonStringify({
26+
Name: glueCatalogName,
27+
Description: "Federated catalog for S3 Tables",
28+
FederatedCatalog: {
29+
Identifier: s3TablesBucketWildcardArn,
30+
ConnectionName: "aws:s3tables",
31+
},
32+
CreateDatabaseDefaultPermissions: [
33+
{
34+
Principal: {
35+
DataLakePrincipalIdentifier: "IAM_ALLOWED_PRINCIPALS",
36+
},
37+
Permissions: ["ALL"],
38+
},
39+
],
40+
CreateTableDefaultPermissions: [
41+
{
42+
Principal: {
43+
DataLakePrincipalIdentifier: "IAM_ALLOWED_PRINCIPALS",
44+
},
45+
Permissions: ["ALL"],
46+
},
47+
],
48+
AllowFullTableExternalDataAccess: "True",
49+
}),
50+
},
51+
{ dependsOn: [tableBucket] },
52+
)
53+
54+
const athenaResultsBucket = new aws.s3.Bucket("LakeAthenaResults", {
55+
bucket: `opencode-${$app.stage}-lake-athena-results`,
56+
forceDestroy: $app.stage !== "production",
57+
})
58+
59+
const firehoseErrorBucket = new aws.s3.Bucket("LakeFirehoseErrors", {
60+
bucket: `opencode-${$app.stage}-lake-firehose-errors`,
61+
forceDestroy: $app.stage !== "production",
62+
})
63+
64+
const athenaWorkgroup = new aws.athena.Workgroup("LakeAthenaWorkgroup", {
65+
name: `opencode-${$app.stage}-lake-workgroup`,
66+
forceDestroy: $app.stage !== "production",
67+
configuration: {
68+
enforceWorkgroupConfiguration: true,
69+
publishCloudwatchMetricsEnabled: true,
70+
resultConfiguration: {
71+
outputLocation: $interpolate`s3://${athenaResultsBucket.bucket}/`,
72+
},
73+
},
74+
})
75+
76+
const firehoseRole = new aws.iam.Role("LakeFirehoseRole", {
77+
assumeRolePolicy: aws.iam.getPolicyDocumentOutput({
78+
statements: [
79+
{
80+
effect: "Allow",
81+
actions: ["sts:AssumeRole"],
82+
principals: [
83+
{
84+
type: "Service",
85+
identifiers: ["firehose.amazonaws.com"],
86+
},
87+
],
88+
},
89+
],
90+
}).json,
91+
})
92+
93+
const firehosePolicy = new aws.iam.RolePolicy("LakeFirehosePolicy", {
94+
role: firehoseRole.id,
95+
policy: aws.iam.getPolicyDocumentOutput({
96+
statements: [
97+
{
98+
effect: "Allow",
99+
actions: [
100+
"s3tables:ListTableBuckets",
101+
"s3tables:GetTableBucket",
102+
"s3tables:GetNamespace",
103+
"s3tables:GetTable",
104+
"s3tables:GetTableData",
105+
"s3tables:GetTableMetadataLocation",
106+
"s3tables:ListNamespaces",
107+
"s3tables:ListTables",
108+
"s3tables:PutTableData",
109+
"s3tables:UpdateTableMetadataLocation",
110+
],
111+
resources: ["*"],
112+
},
113+
{
114+
effect: "Allow",
115+
actions: [
116+
"glue:GetCatalog",
117+
"glue:GetCatalogs",
118+
"glue:GetDatabase",
119+
"glue:GetDatabases",
120+
"glue:GetTable",
121+
"glue:GetTables",
122+
"glue:UpdateTable",
123+
],
124+
resources: [
125+
glueCatalogArn,
126+
glueS3TablesCatalogArn,
127+
$interpolate`${glueS3TablesCatalogArn}/*`,
128+
glueS3TablesDatabaseWildcardArn,
129+
glueS3TablesTableWildcardArn,
130+
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/*`,
131+
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/*/*`,
132+
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/*`,
133+
],
134+
},
135+
{
136+
effect: "Allow",
137+
actions: [
138+
"s3:AbortMultipartUpload",
139+
"s3:GetBucketLocation",
140+
"s3:GetObject",
141+
"s3:ListBucket",
142+
"s3:ListBucketMultipartUploads",
143+
"s3:PutObject",
144+
],
145+
resources: [firehoseErrorBucket.arn, $interpolate`${firehoseErrorBucket.arn}/*`],
146+
},
147+
{
148+
effect: "Allow",
149+
actions: ["lakeformation:GetDataAccess"],
150+
resources: ["*"],
151+
},
152+
],
153+
}).json,
154+
})
155+
156+
const firehose = new aws.kinesis.FirehoseDeliveryStream(
157+
"LakeFirehose",
158+
{
159+
name: `opencode-${$app.stage}-lake-ingest`,
160+
destination: "iceberg",
161+
icebergConfiguration: {
162+
appendOnly: true,
163+
bufferingInterval: 60,
164+
bufferingSize: 1,
165+
catalogArn: glueS3TablesChildCatalogArn,
166+
processingConfiguration: {
167+
enabled: true,
168+
processors: [
169+
{
170+
type: "MetadataExtraction",
171+
parameters: [
172+
{ parameterName: "JsonParsingEngine", parameterValue: "JQ-1.6" },
173+
{
174+
parameterName: "MetadataExtractionQuery",
175+
parameterValue:
176+
'{destinationDatabaseName:._lake_database,destinationTableName:._lake_table,operation:(._lake_operation // "insert")}',
177+
},
178+
],
179+
},
180+
],
181+
},
182+
roleArn: firehoseRole.arn,
183+
s3BackupMode: "FailedDataOnly",
184+
s3Configuration: {
185+
roleArn: firehoseRole.arn,
186+
bucketArn: firehoseErrorBucket.arn,
187+
errorOutputPrefix: "errors/!{firehose:error-output-type}/",
188+
},
189+
},
190+
},
191+
{ dependsOn: [s3TablesCatalog, firehosePolicy] },
192+
)
193+
194+
export const lakeVpc = new sst.aws.Vpc("LakeVpc")
195+
export const lakeCluster = new sst.aws.Cluster("LakeCluster", { vpc: lakeVpc })
196+
export const lakeRegion = region.region
197+
export const lakeCatalog = $interpolate`${glueCatalogName}/${tableBucket.name}`
198+
export const lakeAthenaWorkgroup = athenaWorkgroup
199+
200+
const ingestSecret = new random.RandomPassword("LakeIngestSecret", { length: 32 })
201+
202+
const ingestConfig = new sst.Linkable("LakeIngestConfig", {
203+
properties: {
204+
streamName: firehose.name,
205+
secret: ingestSecret.result,
206+
},
207+
})
208+
209+
const ingestService = new sst.aws.Service("LakeIngestService", {
210+
cluster: lakeCluster,
211+
architecture: "arm64",
212+
cpu: "0.5 vCPU",
213+
memory: "1 GB",
214+
image: {
215+
context: ".",
216+
dockerfile: "packages/stats/server/Dockerfile",
217+
},
218+
link: [ingestConfig],
219+
permissions: [
220+
{
221+
actions: ["firehose:PutRecord", "firehose:PutRecordBatch"],
222+
resources: [firehose.arn],
223+
},
224+
],
225+
scaling: {
226+
min: $app.stage === "production" ? 2 : 1,
227+
max: $app.stage === "production" ? 32 : 4,
228+
cpuUtilization: 60,
229+
memoryUtilization: 70,
230+
},
231+
loadBalancer: {
232+
domain: {
233+
name: `lake.${domain}`,
234+
dns: sst.cloudflare.dns(),
235+
},
236+
rules: [
237+
{ listen: "80/http", redirect: "443/https" },
238+
{ listen: "443/https", forward: "3000/http" },
239+
],
240+
health: {
241+
"3000/http": {
242+
path: "/ready",
243+
successCodes: "200-299",
244+
},
245+
},
246+
},
247+
health: {
248+
command: [
249+
"CMD-SHELL",
250+
"bun --eval \"fetch('http://localhost:3000/health').then((r) => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))\"",
251+
],
252+
interval: "30 seconds",
253+
retries: 3,
254+
startPeriod: "30 seconds",
255+
timeout: "5 seconds",
256+
},
257+
dev: {
258+
command: "bun run start",
259+
directory: "packages/stats/server",
260+
url: "http://localhost:3000",
261+
},
262+
wait: $app.stage === "production",
263+
})
264+
265+
export const lakeIngest = new sst.Linkable("LakeIngest", {
266+
properties: {
267+
url: ingestService.url,
268+
secret: ingestSecret.result,
269+
},
270+
})
271+
272+
export const lakeQueryPermissions = [
273+
{
274+
actions: ["athena:StartQueryExecution", "athena:GetQueryExecution", "athena:GetQueryResults"],
275+
resources: [athenaWorkgroup.arn],
276+
},
277+
{
278+
actions: [
279+
"glue:GetCatalog",
280+
"glue:GetCatalogs",
281+
"glue:GetDatabase",
282+
"glue:GetDatabases",
283+
"glue:GetTable",
284+
"glue:GetTables",
285+
"glue:GetPartitions",
286+
],
287+
resources: [
288+
glueCatalogArn,
289+
glueS3TablesCatalogArn,
290+
$interpolate`${glueS3TablesCatalogArn}/*`,
291+
glueS3TablesDatabaseWildcardArn,
292+
glueS3TablesTableWildcardArn,
293+
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/*`,
294+
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/*/*`,
295+
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/*`,
296+
],
297+
},
298+
{
299+
actions: ["s3:GetBucketLocation", "s3:ListBucket"],
300+
resources: [athenaResultsBucket.arn],
301+
},
302+
{
303+
actions: ["s3:GetObject", "s3:PutObject", "s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads"],
304+
resources: [$interpolate`${athenaResultsBucket.arn}/*`],
305+
},
306+
{
307+
actions: [
308+
"s3tables:GetTableBucket",
309+
"s3tables:GetNamespace",
310+
"s3tables:GetTable",
311+
"s3tables:GetTableData",
312+
"s3tables:GetTableMetadataLocation",
313+
"s3tables:ListNamespaces",
314+
"s3tables:ListTables",
315+
],
316+
resources: ["*"],
317+
},
318+
{
319+
actions: ["lakeformation:GetDataAccess"],
320+
resources: ["*"],
321+
},
322+
]

0 commit comments

Comments
 (0)