Skip to content

Commit 9c27b12

Browse files
MmahdiM79MmahdiM79
authored
Add rpc stubs (#18)
* email_service for stubs added * stubs configs json added to protos folder * generate_python_stubs_file python script added * generate_python_stubs command added to makefile * check_stubs_configs_file command added to makefile * stubs_configs_check CI added * generate_python_stubs command added to build_and_push_python_codes CI * new stub config trigger added for set_new_version_tag CI * stubs_py_template.txt file renamed to python_stubs_py_template.txt * README file updated --------- Co-authored-by: Mohammad Mahdi Malmasi
1 parent 22a0809 commit 9c27b12

9 files changed

Lines changed: 308 additions & 7 deletions

File tree

‎.github/workflows/build_and_push_python_codes.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ jobs:
2929
- name: Build
3030
run: |
3131
make generate_python_protos
32+
make generate_python_stubs
3233
3334
- name: Checkout destination repo
3435
uses: actions/checkout@v2

‎.github/workflows/set_new_version_tag.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ on:
88
paths:
99
- 'protos/**'
1010
- '**/*python*'
11+
- '**/python/**'
12+
- 'stubs/**'
1113

1214
jobs:
1315
set-version-tag:

‎.github/workflows/stubs_configs_check.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
name: Check stubs configs json file
2+
3+
on:
4+
push:
5+
branches:
6+
- '*'
7+
- '!master'
8+
9+
jobs:
10+
build:
11+
runs-on: ubuntu-latest
12+
steps:
13+
- name: Checkout source code
14+
uses: actions/checkout@v2
15+
with:
16+
ref: ${{ github.ref }}
17+
18+
- name: Check stubs configs json file
19+
run: |
20+
make check_stubs_configs_file

‎Makefile‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,49 @@
11
PROTOS_DIR=./protos
2+
STUBS_CONFIGS_FILE=$(PROTOS_DIR)/stubs/stubs_configs.json
3+
SCRIPTS_DIR=./scripts
4+
TEXT_FILES_DIR=$(SCRIPTS_DIR)/text_files
5+
PYTHON_SCRIPTS_DIR=$(SCRIPTS_DIR)/python
6+
27
PYTHON_PROTOS_OUTPUT_DIR=./AIaaS_interface
38
PYTHON_REQUIREMENTS_FILE_PATH=./requirements.txt
9+
PYTHON_STUBS_OUTPUT_DIR=$(PYTHON_PROTOS_OUTPUT_DIR)
10+
PYTHON_STUBS_CONFIGS_CHECKER_SCRIPT=$(PYTHON_SCRIPTS_DIR)/check_stubs_configs.py
11+
PYTHON_STUBS_GENERATOR_SCRIPT=$(PYTHON_SCRIPTS_DIR)/generate_python_stubs_file.py
12+
STUBS_OUTPUT_FILE_PATH=$(PYTHON_STUBS_OUTPUT_DIR)/stubs.py
13+
TEMPLATE_OUTPUT_FILE_PATH=$(TEXT_FILES_DIR)/python_stubs_py_template.txt
14+
415

516

617
clean:
7-
rm -rf $(PYTHON_PROTOS_OUTPUT_DIR)
18+
@rm -rf $(PYTHON_PROTOS_OUTPUT_DIR)
819

920
set_new_version_tag:
10-
bash ./scripts/generate_new_version_tag.sh
21+
@bash ./scripts/generate_new_version_tag.sh
1122

1223
install_python_requirements:
1324
pip install -r $(PYTHON_REQUIREMENTS_FILE_PATH)
1425

26+
check_stubs_configs_file:
27+
python \
28+
$(PYTHON_STUBS_CONFIGS_CHECKER_SCRIPT) \
29+
$(STUBS_CONFIGS_FILE) \
30+
$(PYTHON_PROTOS_OUTPUT_DIR) \
31+
$(PROTOS_DIR)
32+
1533
generate_python_protos: clean install_python_requirements
16-
mkdir $(PYTHON_PROTOS_OUTPUT_DIR)
17-
find $(PROTOS_DIR) -name "*.proto" \
34+
@mkdir $(PYTHON_PROTOS_OUTPUT_DIR)
35+
@find $(PROTOS_DIR) -name "*.proto" \
1836
-exec python -m grpc_tools.protoc \
1937
-I$(PROTOS_DIR) \
2038
--python_out=$(PYTHON_PROTOS_OUTPUT_DIR) \
2139
--pyi_out=$(PYTHON_PROTOS_OUTPUT_DIR) \
2240
--grpc_python_out=$(PYTHON_PROTOS_OUTPUT_DIR) {} \;
2341
bash ./scripts/add_init_files.sh $(PYTHON_PROTOS_OUTPUT_DIR)
42+
43+
generate_python_stubs: generate_python_protos
44+
python \
45+
$(PYTHON_STUBS_GENERATOR_SCRIPT) \
46+
$(STUBS_CONFIGS_FILE) \
47+
$(PYTHON_STUBS_OUTPUT_DIR) \
48+
$(STUBS_OUTPUT_FILE_PATH) \
49+
$(TEMPLATE_OUTPUT_FILE_PATH)

‎README.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ Read more about gRPC [here](https://grpc.io/docs/what-is-grpc/introduction/).
99
1. checkout a new branch from `master` branch
1010
2. add a folder for your microservice in `protos` directory if it doesn't exist
1111
3. add your proto file in the folder you created in step 2
12-
4. make sure your proto file is valid
13-
5. commit and push your changes
14-
6. create a pull request to merge your branch into `master` branch
12+
4. if you added a new `service`, update `stubs_configs.json` file
13+
5. make sure your proto file is valid
14+
6. commit and push your changes
15+
7. create a pull request to merge your branch into `master` branch
1516

1617
## How to use the gRPC my microservice project?
1718
After you add your proto file to this repository and your pull request is merged into `master` branch, you can have your

‎protos/stubs/stubs_configs.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"_comment": {
3+
"notice": "Allowed ports for services are 50051-50999. update next_port if you want to add more services",
4+
"next_port": 50052
5+
},
6+
7+
"services": [
8+
{
9+
"name": "NotificationEmail",
10+
"proto_file_path": "protos/platform_management/notification_server/email.proto",
11+
"host": "0.0.0.0",
12+
"port": 50051
13+
}
14+
]
15+
}

‎scripts/python/check_stubs_configs.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import sys
2+
import json
3+
import re
4+
5+
6+
def check_config_keys_and_values(service_config: dict) -> None:
7+
global protos_folder
8+
9+
required_keys = [
10+
'name',
11+
'proto_file_path',
12+
'host',
13+
'port',
14+
]
15+
for required_key in required_keys:
16+
if required_key not in service_config:
17+
raise Exception(f'{required_key} is required in service config!')
18+
if len(service_config.keys()) > len(required_keys):
19+
raise Exception('service config must have only required keys!')
20+
21+
if type(service_config['name']) != str:
22+
raise Exception('name must be a string!')
23+
if type(service_config['proto_file_path']) != str:
24+
raise Exception('proto_file_path must be a string!')
25+
if service_config['proto_file_path'].split('/')[0] != protos_folder:
26+
raise Exception('proto_file_path must be in the protos folder!')
27+
28+
try:
29+
proto_file = open(service_config['proto_file_path'], 'r')
30+
except:
31+
raise Exception('proto_file_path must be a valid path to a proto file!')
32+
33+
proto_file_text = proto_file.read()
34+
if re.search(r'service\s+', proto_file_text) is None:
35+
raise Exception('proto file must have a service!')
36+
if re.search(r'service\s+' + service_config['name'] + r'\s*{', proto_file_text) is None:
37+
raise Exception('proto file must have a service with the same name in service config!')
38+
proto_file.close()
39+
40+
if type(service_config['host']) != str:
41+
raise Exception('host must be a string!')
42+
if type(service_config['port']) != int:
43+
raise Exception('port must be an integer!')
44+
45+
46+
47+
48+
if __name__ == '__main__':
49+
# read arguments
50+
stubs_configs_file_path = sys.argv[1]
51+
protos_output_folder = sys.argv[2].split('/')[1]
52+
protos_folder = sys.argv[3].split('/')[1]
53+
54+
# read stubs configs file
55+
stubs_json_dict = json.loads(open(stubs_configs_file_path, 'r').read())
56+
57+
58+
services_configs = stubs_json_dict['services']
59+
if type(services_configs) != list:
60+
raise Exception('services must be a list of configs!')
61+
62+
next_port = stubs_json_dict['_comment']['next_port']
63+
if type(next_port) != int:
64+
raise Exception('next_port must be an integer!')
65+
66+
67+
used_ports, implemented_services = [], []
68+
for service_config in services_configs:
69+
check_config_keys_and_values(service_config)
70+
71+
if service_config['port'] < 50051:
72+
raise Exception('port must be greater than 50051!')
73+
if service_config['port'] > 50999:
74+
raise Exception('port must be less than 50999!')
75+
if service_config['port'] in used_ports:
76+
raise Exception('port must be unique!')
77+
if service_config['port'] >= next_port:
78+
raise Exception('next_port must be updated!')
79+
used_ports.append(service_config['port'])
80+
81+
if service_config['name'] in implemented_services:
82+
raise Exception('service name must be unique!')
83+
implemented_services.append(service_config['name'])
84+
85+
86+
print('\n\nstubs configs file is valid ✅')
87+
exit(0)

‎scripts/python/generate_python_stubs_file.py‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
import json
2+
import sys
3+
import re
4+
5+
6+
# global variables
7+
DICT_HOST_KEY = 'host'
8+
DICT_PORT_KEY = 'port'
9+
DICT_STUB_CLASS_KEY = 'stub_class'
10+
11+
12+
13+
class Service:
14+
def __init__(
15+
self,
16+
host: str,
17+
port: int,
18+
name: str,
19+
proto_file_path: str,
20+
protos_output_folder: str,
21+
):
22+
self.host = host
23+
self.port = port
24+
self.name = name
25+
self.__proto_file_path = proto_file_path
26+
self.__protos_output_folder = protos_output_folder
27+
28+
self.module_path: str = None
29+
self.__generate_module_import_code_str()
30+
31+
def __generate_module_import_code_str(self) -> None:
32+
module_path = self.__proto_file_path.replace('/', '.')
33+
module_path = module_path.replace('.proto', '_pb2_grpc')
34+
module_path = module_path.replace('protos', self.__protos_output_folder)
35+
self.module_path = 'from ' + module_path + ' import ' + self.name + 'Stub'
36+
37+
38+
39+
def convert_to_snake_case(string):
40+
pattern = r'([A-Z])'
41+
split_string = re.sub(pattern, r'_\1', string).strip('_')
42+
return split_string.upper()
43+
44+
45+
46+
def generate_stubs_file_code(
47+
services: list,
48+
stubs_output_file_path: str,
49+
template_output_file_path: str,
50+
) -> None:
51+
52+
# read template file
53+
stubs_output_template_file = open(template_output_file_path, 'r')
54+
stubs_output_file_text = stubs_output_template_file.read()
55+
stubs_output_template_file.close()
56+
57+
# replace dict keys in get_stub function
58+
stubs_output_file_text = stubs_output_file_text.replace(
59+
'$HOST_KEY',
60+
DICT_HOST_KEY,
61+
)
62+
stubs_output_file_text = stubs_output_file_text.replace(
63+
'$PORT_KEY',
64+
DICT_PORT_KEY,
65+
)
66+
stubs_output_file_text = stubs_output_file_text.replace(
67+
'$STUB_CLASS_KEY',
68+
DICT_STUB_CLASS_KEY,
69+
)
70+
71+
# import libraries code generation
72+
default_imports = [
73+
'grpc',
74+
]
75+
imports_code = ''
76+
for default_import in default_imports:
77+
imports_code += f'import {default_import}\n'
78+
imports_code += '\n'
79+
80+
for service in services:
81+
imports_code += service.module_path + '\n'
82+
83+
stubs_output_file_text = stubs_output_file_text.replace(
84+
'{{ import libraries code }}',
85+
imports_code,
86+
)
87+
88+
# services code generation
89+
services_code = 'class Services:\n'
90+
for service in services:
91+
services_code += f'\t{convert_to_snake_case(service.name)} = {{\n'
92+
services_code += f'\t\t"{DICT_HOST_KEY}": "{service.host}",\n'
93+
services_code += f'\t\t"{DICT_PORT_KEY}": {service.port},\n'
94+
services_code += f'\t\t"{DICT_STUB_CLASS_KEY}": {service.name}Stub,\n'
95+
services_code += '\t}\n'
96+
stubs_output_file_text = stubs_output_file_text.replace(
97+
'{{ Services class code }}',
98+
services_code,
99+
)
100+
101+
# write stubs output file
102+
stubs_output_file = open(stubs_output_file_path, 'w')
103+
stubs_output_file.write(stubs_output_file_text)
104+
stubs_output_file.close()
105+
106+
107+
108+
109+
if __name__ == '__main__':
110+
# read arguments
111+
stubs_configs_file_path = sys.argv[1]
112+
stubs_output_folder = sys.argv[2]
113+
protos_output_folder = stubs_output_folder.split('/')[1]
114+
stubs_output_file_path = sys.argv[3]
115+
template_output_file_path = sys.argv[4]
116+
117+
118+
# read stubs configs file
119+
stubs_json_dict = json.loads(open(stubs_configs_file_path, 'r').read())
120+
services_configs = stubs_json_dict['services']
121+
122+
123+
# create services objects
124+
services = []
125+
for service in services_configs:
126+
services.append(
127+
Service(
128+
host=service['host'],
129+
port=service['port'],
130+
name=service['name'],
131+
proto_file_path=service['proto_file_path'],
132+
protos_output_folder=protos_output_folder,
133+
)
134+
)
135+
136+
generate_stubs_file_code(
137+
services=services,
138+
stubs_output_file_path=stubs_output_file_path,
139+
template_output_file_path=template_output_file_path,
140+
)

‎scripts/text_files/python_stubs_py_template.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{{ import libraries code }}
2+
3+
4+
{{ Services class code }}
5+
6+
7+
def get_stub(config: dict):
8+
channel = grpc.insecure_channel(config['$HOST_KEY'] + ':' + str(config['$PORT_KEY']))
9+
return config['$STUB_CLASS_KEY'](channel)

0 commit comments

Comments
 (0)