Skip to content
This repository was archived by the owner on May 30, 2023. It is now read-only.

Commit 6e8f932

Browse files
ttattlttattl
committed
feat: init create project from prebuild dataset
1 parent c6f0d63 commit 6e8f932

12 files changed

Lines changed: 649 additions & 5 deletions

File tree

‎daita-app/core-service/api-defs/daita_http_api.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -289,6 +289,20 @@ paths:
289289
type: "aws_proxy"
290290
payloadFormatVersion: "2.0"
291291

292+
/projects/create_project_from_prebuild:
293+
post:
294+
responses:
295+
default:
296+
description: ""
297+
x-amazon-apigateway-integration:
298+
credentials:
299+
Fn::GetAtt: [ApiGatewayCallLambdaRole, Arn]
300+
uri:
301+
Fn::Sub: arn:${AWS::Partition}:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${FunctionCreateProjectFromPrebuild.Arn}/invocations
302+
httpMethod: "POST"
303+
type: "aws_proxy"
304+
payloadFormatVersion: "2.0"
305+
292306
# /projects/apply_expert_mode_param:
293307
# post:
294308
# responses:

‎daita-app/core-service/core_service_template.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,9 @@ Parameters:
132132
Type: String
133133
CognitoUserPoolClient:
134134
Type: String
135+
CreateProjectPrebuildSMArn:
136+
Type: String
137+
135138
Resources:
136139
#================ ROLES =====================================================
137140

@@ -411,6 +414,16 @@ Resources:
411414
Variables:
412415
T_CONST_PREBUILD_DATASET: !Ref TableConstPrebuildDatasetName
413416

417+
FunctionCreateProjectFromPrebuild:
418+
Type: AWS::Serverless::Function
419+
Properties:
420+
CodeUri: functions/handlers/project/create_prj_fr_prebuild
421+
Role: !GetAtt LambdaExecutionRole.Arn
422+
Environment:
423+
Variables:
424+
T_CONST_PREBUILD_DATASET: !Ref TableConstPrebuildDatasetName
425+
SM_CREATE_PRJ_PREBUILD: !Ref CreateProjectPrebuildSMArn
426+
414427
### for health check functions
415428
HealthCheckFunction:
416429
Type: AWS::Serverless::Function

‎daita-app/core-service/functions/handlers/project/create_prj_fr_prebuild/app.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import boto3
2+
import json
3+
import os
4+
import uuid
5+
6+
from config import *
7+
from response import *
8+
from error_messages import *
9+
from identity_check import *
10+
11+
from system_parameter_store import SystemParameterStore
12+
from lambda_base_class import LambdaBaseClass
13+
from models.prebuild_dataset_model import PrebuildDatasetModel
14+
15+
16+
class CreatePrebuildDatasetClass(LambdaBaseClass):
17+
18+
def __init__(self) -> None:
19+
super().__init__()
20+
self.client_events = boto3.client('events')
21+
self.client_step_func = boto3.client('stepfunctions')
22+
self.prebuild_dataset_model = PrebuildDatasetModel(os.environ["T_CONST_PREBUILD_DATASET"])
23+
self.sm_create_prj_prebuild = os.environ["SM_CREATE_PRJ_PREBUILD"]
24+
25+
@LambdaBaseClass.parse_body
26+
def parser(self, body):
27+
self.logger.debug(f"body in main_parser: {body}")
28+
29+
self.id_token = body[KEY_NAME_ID_TOKEN]
30+
self.name_id_prebuild_dataset = body["name_id_prebuild"]
31+
self.number_random = body["number_random"]
32+
33+
def _check_input_value(self):
34+
prebuild_dataset = self.prebuild_dataset_model.get_prebuild_dataset(self.name_id_prebuild_dataset)
35+
if prebuild_dataset is None:
36+
raise Exception(MESS_ERR_INVALID_PREBUILD_DATASET_NAME.format(self.name_id_prebuild_dataset))
37+
38+
### check number max
39+
if self.number_random <= 0 or self.number_random >= prebuild_dataset[PrebuildDatasetModel.FIELD_TOTAL_IMAGES]:
40+
self.number_random = prebuild_dataset[PrebuildDatasetModel.FIELD_TOTAL_IMAGES]
41+
42+
### udpate the link to s3
43+
self.s3_key = prebuild_dataset[PrebuildDatasetModel.FIELD_S3_KEY]
44+
self.visual_name = prebuild_dataset[PrebuildDatasetModel.FIELD_VISUAL_NAME]
45+
46+
return
47+
48+
def handle(self, event, context):
49+
50+
### parse body
51+
self.parser(event)
52+
53+
### check identity
54+
identity_id = self.get_identity(self.id_token)
55+
56+
### create project on DB
57+
_uuid = uuid.uuid4().hex
58+
project_id = f'{self.visual_name}_{_uuid}'
59+
s3_prefix = f'{os.environ["BUCKET_NAME"]}/{identity_id}/{project_id}'
60+
db_client = boto3.client('dynamodb')
61+
db_resource = boto3.resource('dynamodb')
62+
try:
63+
is_sample = True
64+
gen_status = "GENERATING"
65+
table_prj = db_resource.Table(os.environ["T_PROJECT"])
66+
table_prj.put_item(
67+
Item = {
68+
'ID': _uuid,
69+
'project_id': project_id,
70+
'identity_id': identity_id,
71+
'project_name': project_name,
72+
's3_prefix': s3_prefix,
73+
'project_info': project_info,
74+
# 'sub': sub,
75+
'created_date': convert_current_date_to_iso8601(),
76+
'is_sample': is_sample,
77+
'gen_status': gen_status
78+
},
79+
ConditionExpression = Attr('project_name').not_exists() & Attr('identity_id').not_exists()
80+
)
81+
82+
except db_resource.meta.client.exceptions.ConditionalCheckFailedException as e:
83+
print('Error condition: ', e)
84+
err_mess = const.MES_DUPLICATE_PROJECT_NAME.format(project_name)
85+
return convert_response({"error": True,
86+
"success": False,
87+
"message": err_mess,
88+
"data": None})
89+
except Exception as e:
90+
print('Error: ', repr(e))
91+
return convert_response({"error": True,
92+
"success": False,
93+
"message": repr(e),
94+
"data": None})
95+
96+
### call async step function
97+
stepfunction_input = {
98+
"identity_id": identity_id,
99+
"id_token": self.id_token,
100+
}
101+
response = self.client_step_func.start_execution(
102+
stateMachineArn=self.sm_create_prj_prebuild,
103+
input=json.dumps(stepfunction_input)
104+
)
105+
106+
return generate_response(
107+
message="OK",
108+
status_code=HTTPStatus.OK,
109+
data={},
110+
)
111+
112+
@error_response
113+
def lambda_handler(event, context):
114+
115+
return CreatePrebuildDatasetClass().handle(event, context)
116+
117+

‎daita-app/core-service/functions/handlers/project/list_prebuild_dataset/app.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,15 +35,16 @@ def handle(self, event, context):
3535
self.parser(event)
3636

3737
### check identity
38-
identity_id = self.get_identity(self.id_token)
38+
# identity_id = self.get_identity(self.id_token)
3939

4040
### get list info
41-
items = self.prebuild_dataset_model.get_list_prebuild_dataset()
41+
items = self.prebuild_dataset_model.get_list_prebuild_dataset()
42+
ls_item_convert = [self.prebuild_dataset_model.convert_item_to_json(item) for item in items]
4243

4344
return generate_response(
4445
message="OK",
4546
status_code=HTTPStatus.OK,
46-
data=items,
47+
data=ls_item_convert,
4748
)
4849

4950
@error_response

‎daita-app/project_service/functions/hdler_get_data.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import boto3
2+
import json
3+
import os
4+
5+
from config import *
6+
from response import *
7+
from error_messages import *
8+
from identity_check import *
9+
10+
from system_parameter_store import SystemParameterStore
11+
from lambda_base_class import LambdaBaseClass
12+
from models.data_model import DataModel, DataItem
13+
from models.task_model import TaskModel
14+
from utils import get_bucket_key_from_s3_uri, split_ls_into_batch
15+
16+
17+
class GetDataClass(LambdaBaseClass):
18+
19+
def __init__(self) -> None:
20+
super().__init__()
21+
self.client_events = boto3.client('events')
22+
self.const = SystemParameterStore()
23+
self.s3 = boto3.client('s3')
24+
self.task_model = TaskModel(os.environ["TABLE_REFERENCE_IMAGE_TASK"])
25+
26+
@LambdaBaseClass.parse_body
27+
def parser(self, body):
28+
self.logger.debug(f"body in main_parser: {body}")
29+
self.execution_id = body["id"]
30+
body = body["detail"]
31+
self.identity_id = body[KEY_NAME_IDENTITY_ID]
32+
self.project_id = body[KEY_NAME_PROJECT_ID]
33+
self.task_id = body[KEY_NAME_TASK_ID]
34+
self.ls_method_id = body[KEY_NAME_LS_METHOD_ID]
35+
36+
def _check_input_value(self):
37+
pass
38+
39+
def handle(self, event, context):
40+
41+
### parse body
42+
self.parser(event)
43+
44+
### update task execution arn
45+
self.task_model.update_attribute(self.task_id, self.identity_id,
46+
[(TaskModel.FIELD_EXECUTION_SM_ID, self.execution_id)])
47+
48+
### get data DB corresponding
49+
data_table_name = os.environ["TABLE_DATA_ORIGINAL"]
50+
self.data_model = DataModel(data_table_name)
51+
52+
### get all data from original to calculate reference image
53+
ls_info = self.data_model.get_all_data_in_project(self.project_id)
54+
55+
### split into batch
56+
batch_size = int(self.const.get_param(os.environ["BATCHSIZE_REF_IMG"]))
57+
print("Batch size value: ", batch_size)
58+
ls_batchs = split_ls_into_batch(ls_info, batch_size=batch_size)
59+
print("ls_batchs: ", ls_batchs)
60+
61+
if len(ls_info)>0:
62+
bucket, folder = get_bucket_key_from_s3_uri(ls_info[0][DataItem.FIELD_S3_KEY])
63+
folder = "/".join(folder.split("/")[:-1])
64+
s3_key_path = os.path.join(folder, f"reference_image_step_output/{self.task_id}/RI_getdata.json")
65+
self.s3.put_object(
66+
Body=json.dumps(ls_batchs),
67+
Bucket= bucket,
68+
Key= s3_key_path
69+
)
70+
else:
71+
bucket = None
72+
s3_key_path = None
73+
74+
return generate_response(
75+
message="OK",
76+
status_code=HTTPStatus.OK,
77+
data={
78+
"bucket": bucket,
79+
"s3_key_path": s3_key_path,
80+
"ls_length_batch": list(range(len(ls_batchs)))
81+
},
82+
is_in_stepfunction=True
83+
)
84+
85+
def lambda_handler(event, context):
86+
87+
return GetDataClass().handle(event, context)
88+
89+

‎daita-app/project_service/functions/hdler_move_update_data.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import boto3
2+
import json
3+
import os
4+
5+
from config import *
6+
from response import *
7+
from error_messages import *
8+
from identity_check import *
9+
10+
from system_parameter_store import SystemParameterStore
11+
from lambda_base_class import LambdaBaseClass
12+
from models.data_model import DataModel, DataItem
13+
from models.task_model import TaskModel
14+
from utils import get_bucket_key_from_s3_uri, split_ls_into_batch
15+
16+
17+
class MoveUpdateDataClass(LambdaBaseClass):
18+
19+
def __init__(self) -> None:
20+
super().__init__()
21+
self.client_events = boto3.client('events')
22+
self.const = SystemParameterStore()
23+
self.s3 = boto3.client('s3')
24+
self.task_model = TaskModel(os.environ["TABLE_REFERENCE_IMAGE_TASK"])
25+
26+
@LambdaBaseClass.parse_body
27+
def parser(self, body):
28+
self.logger.debug(f"body in main_parser: {body}")
29+
self.execution_id = body["id"]
30+
body = body["detail"]
31+
self.identity_id = body[KEY_NAME_IDENTITY_ID]
32+
self.project_id = body[KEY_NAME_PROJECT_ID]
33+
self.task_id = body[KEY_NAME_TASK_ID]
34+
self.ls_method_id = body[KEY_NAME_LS_METHOD_ID]
35+
36+
def _check_input_value(self):
37+
pass
38+
39+
def handle(self, event, context):
40+
41+
### parse body
42+
self.parser(event)
43+
44+
### update task execution arn
45+
self.task_model.update_attribute(self.task_id, self.identity_id,
46+
[(TaskModel.FIELD_EXECUTION_SM_ID, self.execution_id)])
47+
48+
### get data DB corresponding
49+
data_table_name = os.environ["TABLE_DATA_ORIGINAL"]
50+
self.data_model = DataModel(data_table_name)
51+
52+
### get all data from original to calculate reference image
53+
ls_info = self.data_model.get_all_data_in_project(self.project_id)
54+
55+
### split into batch
56+
batch_size = int(self.const.get_param(os.environ["BATCHSIZE_REF_IMG"]))
57+
print("Batch size value: ", batch_size)
58+
ls_batchs = split_ls_into_batch(ls_info, batch_size=batch_size)
59+
print("ls_batchs: ", ls_batchs)
60+
61+
if len(ls_info)>0:
62+
bucket, folder = get_bucket_key_from_s3_uri(ls_info[0][DataItem.FIELD_S3_KEY])
63+
folder = "/".join(folder.split("/")[:-1])
64+
s3_key_path = os.path.join(folder, f"reference_image_step_output/{self.task_id}/RI_getdata.json")
65+
self.s3.put_object(
66+
Body=json.dumps(ls_batchs),
67+
Bucket= bucket,
68+
Key= s3_key_path
69+
)
70+
else:
71+
bucket = None
72+
s3_key_path = None
73+
74+
return generate_response(
75+
message="OK",
76+
status_code=HTTPStatus.OK,
77+
data={
78+
"bucket": bucket,
79+
"s3_key_path": s3_key_path,
80+
"ls_length_batch": list(range(len(ls_batchs)))
81+
},
82+
is_in_stepfunction=True
83+
)
84+
85+
def lambda_handler(event, context):
86+
87+
return MoveUpdateDataClass().handle(event, context)
88+
89+

0 commit comments

Comments
 (0)