From 2ea73340f10a3c78a283fa2f2c219d083d8312ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=CC=80ng=20Phu=CC=9Bo=CC=9Bng=20Ngo=CC=82?= Date: Fri, 26 Feb 2021 10:42:07 +0700 Subject: [PATCH 01/31] dev/support_multi_modes_feature: add filter mode, add method Post in route GET/service/check --- server/app/routes.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/server/app/routes.py b/server/app/routes.py index 0b4d5e97..c8ec63e0 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -345,6 +345,22 @@ def func_wrapper(*args, **kwargs): return wrapper +def filter_mode(required_mode=['lite']): + def wrapper(func): + """generic request filter system for customization""" + + @wraps(func) + def func_wrapper(*args, **kwargs): + required_mode.append('admin') + if g.session['mode'] in required_mode: + return func(*args, **kwargs) + abort(make_response(jsonify(message="your account does not allow to access this page"), 403)) + + return func_wrapper + + return wrapper + + def has_ability(flask_global, ability, entity): for f in has_ability_funcs: if not f(flask_global, ability, entity): @@ -500,7 +516,7 @@ def server_disable(service, resource): return flask.jsonify("ok") -@app.route("/service/check/", methods=["GET"]) +@app.route("/service/check/", methods=["GET", "POST"]) @filter_request("GET/service/check") def check(service): service_options = flask.request.get_json() if flask.request.is_json else None From e8cd46bcc070ecfc297d513e93da5a9913947bc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=CC=80ng=20Phu=CC=9Bo=CC=9Bng=20Ngo=CC=82?= Date: Wed, 3 Mar 2021 14:32:49 +0700 Subject: [PATCH 02/31] dev/support_multi_modes_feature: change response task log, add request json config when update service config --- server/app/routes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/app/routes.py b/server/app/routes.py index c8ec63e0..c21beb8f 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -453,10 +453,10 @@ def post_admin_request(app, service, action, value="1"): @app.route("/service/configs/", methods=["POST"]) -@filter_request("GET/service/configs", "edit_config") +@filter_request("POST/service/configs", "edit_config") def set_service_config(service): check_permission(service, "edit_config") - request_body = flask.request.form.get('config') + request_body = flask.request.form.get('config') or flask.request.json.get('config') try: update_config = json.loads(request_body) update_config["updated_at"] = time.time() @@ -2020,7 +2020,7 @@ def get_log(task_id): if content is None: abort(flask.make_response( flask.jsonify(message="cannot find log for task %s" % task_id), 404)) - response = flask.make_response(content) + response = flask.jsonify(content) return response From 15123b9a374c5d14be3d384f59f0e1aae5421b21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=CC=80ng=20Phu=CC=9Bo=CC=9Bng=20Ngo=CC=82?= Date: Wed, 3 Mar 2021 18:30:52 +0700 Subject: [PATCH 03/31] dev/support_multi_modes_feature: revert response get task log --- server/app/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app/routes.py b/server/app/routes.py index c21beb8f..f9694ce4 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -2020,7 +2020,7 @@ def get_log(task_id): if content is None: abort(flask.make_response( flask.jsonify(message="cannot find log for task %s" % task_id), 404)) - response = flask.jsonify(content) + response = flask.make_response(content) return response From 9e5ece24022fe844bd78ae2aa55390aad387d9eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=CC=80ng=20Phu=CC=9Bo=CC=9Bng=20Ngo=CC=82?= Date: Fri, 5 Mar 2021 11:16:34 +0700 Subject: [PATCH 04/31] dev/support_multi_modes_feature: redirect if user does not login --- server/app/routes.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/app/routes.py b/server/app/routes.py index f9694ce4..ba23ba94 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -352,6 +352,15 @@ def wrapper(func): @wraps(func) def func_wrapper(*args, **kwargs): required_mode.append('admin') + if g.get('session') is None: + mode = app.get_other_config(["authentication", "mode"]) + redirect_url = '/signin/local' + + if mode == 'systran': + redirect_url = '/signin' + + return flask.redirect(redirect_url) + if g.session['mode'] in required_mode: return func(*args, **kwargs) abort(make_response(jsonify(message="your account does not allow to access this page"), 403)) From 6682a0ec297fc8a32ccb106a0b30bc8d69768c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=CC=80ng=20Phu=CC=9Bo=CC=9Bng=20Ngo=CC=82?= Date: Sat, 6 Mar 2021 13:54:56 +0700 Subject: [PATCH 05/31] dev/support_multi_modes_feature: remove default required_mode --- server/app/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app/routes.py b/server/app/routes.py index ba23ba94..90abd07d 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -345,7 +345,7 @@ def func_wrapper(*args, **kwargs): return wrapper -def filter_mode(required_mode=['lite']): +def filter_mode(required_mode=[]): def wrapper(func): """generic request filter system for customization""" From e9e687f4e70fa4f78e8d4bb562b5f91506ec1d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=CC=80ng=20Phu=CC=9Bo=CC=9Bng=20Ngo=CC=82?= Date: Tue, 9 Mar 2021 14:59:32 +0700 Subject: [PATCH 06/31] dev/support_multi_modes_feature: remove get config data in json --- server/app/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app/routes.py b/server/app/routes.py index 90abd07d..1a211b9d 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -465,7 +465,7 @@ def post_admin_request(app, service, action, value="1"): @filter_request("POST/service/configs", "edit_config") def set_service_config(service): check_permission(service, "edit_config") - request_body = flask.request.form.get('config') or flask.request.json.get('config') + request_body = flask.request.form.get('config') try: update_config = json.loads(request_body) update_config["updated_at"] = time.time() From 55a5cdf0f65126d219896103c24bb88f3de8f102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=CC=80ng=20Phu=CC=9Bo=CC=9Bng=20Ngo=CC=82?= Date: Tue, 9 Mar 2021 15:10:25 +0700 Subject: [PATCH 07/31] dev/support_multi_modes_feature: remove check session in filter_mode --- server/app/routes.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/server/app/routes.py b/server/app/routes.py index 1a211b9d..84beada3 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -352,16 +352,8 @@ def wrapper(func): @wraps(func) def func_wrapper(*args, **kwargs): required_mode.append('admin') - if g.get('session') is None: - mode = app.get_other_config(["authentication", "mode"]) - redirect_url = '/signin/local' - if mode == 'systran': - redirect_url = '/signin' - - return flask.redirect(redirect_url) - - if g.session['mode'] in required_mode: + if g.get('session') is None or g.session['mode'] in required_mode: return func(*args, **kwargs) abort(make_response(jsonify(message="your account does not allow to access this page"), 403)) From 8ce69d673132b7022cce53ac2c5f0efe26027d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=CC=80ng=20Phu=CC=9Bo=CC=9Bng=20Ngo=CC=82?= Date: Wed, 17 Mar 2021 09:17:53 +0700 Subject: [PATCH 08/31] dev/support_multi_modes_feature: fix default ngpu when release task --- server/app/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app/routes.py b/server/app/routes.py index 4f14d994..3e784559 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1533,7 +1533,7 @@ def launch(service): if (task_type != "train" and iterations != 1) or iterations < 1: abort(flask.make_response(flask.jsonify(message="invalid value for iterations"), 400)) - ngpus = 1 + ngpus = 0 if task_type == TASK_RELEASE_TYPE else 1 if "ngpus" in content: ngpus = content["ngpus"] ncpus = content.get("ncpus") From 849ae300c11ec7f5c5aafca808efba632cb1092d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=CC=80ng=20Phu=CC=9Bo=CC=9Bng=20Ngo=CC=82?= Date: Tue, 23 Mar 2021 11:15:47 +0700 Subject: [PATCH 09/31] refs #56253: handle train restricted folder permission --- server/app/routes.py | 41 ++++++++++++++++++++++++++++++++++++ server/utils/common_utils.py | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 server/utils/common_utils.py diff --git a/server/app/routes.py b/server/app/routes.py index 3e784559..dd1addb1 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -28,6 +28,7 @@ # only for launch() maybe deprecated from nmtwizard.task import TaskBase from utils.storage_utils import StorageUtils +from utils.common_utils import is_resource_train_restricted, check_permission_access_train_restricted GLOBAL_POOL_NAME = "global_pool" SYSTRAN_BASE_STORAGE = "shared_testdata" @@ -1454,6 +1455,39 @@ def get_evaluations(): return cust_jsonify(evaluation_catalogs) +def get_json_config(command): + idx = 0 + while idx < len(command): + if (command[idx] == '-c' or command[idx] == '--config'): + return idx + 1, command[idx + 1] + idx += 1 + return None, '' + + +def add_train_restricted_config(json_config, parent_task_id): + config = json.loads(json_config) + + if not config.get("data") or not config["data"]["sample_dist"]: + return json_config + + ok, parent_config = builtins.pn9model_db.catalog_get_info(parent_task_id, False) + + if not ok: + return json_config + + sample_dist = config["data"]["sample_dist"] + parent_sample_dist = parent_config["data"]["sample_dist"] + + for item in parent_sample_dist: + # hide train_restricted path + if not is_resource_train_restricted(item['path']): + continue + sample_dist.append(item) + + config["data"]["sample_dist"] = sample_dist + return json.dumps(config) + + @app.route("/task/launch/", methods=["POST"]) @filter_request("POST/task/launch", "train") def launch(service): @@ -1599,6 +1633,13 @@ def launch(service): (task_type == "prepr" and parent_task_type != "train" and parent_task_type != "vocab")): abort(flask.make_response(flask.jsonify(message="invalid parent task type: %s" % parent_task_type), 400)) + if task_type == 'train' and not check_permission_access_train_restricted('read'): + json_idx, json_config = get_json_config(content["docker"]["command"]) + + if json_idx is not None: + configuration = add_train_restricted_config(json_config, parent_task_id) + content["docker"]["command"][json_idx] = configuration + task_ids = [] task_create = [] diff --git a/server/utils/common_utils.py b/server/utils/common_utils.py new file mode 100644 index 00000000..f9b24b5f --- /dev/null +++ b/server/utils/common_utils.py @@ -0,0 +1,38 @@ +import re +from flask import g + +from app import app + + +def is_allow_access_train_restricted(path, permission): + if not is_resource_train_restricted(path): + return True + + return check_permission_access_train_restricted(permission) + + +def check_permission_access_train_restricted(permission): + # get entity code of user + entity_code = None + if g.get('user'): + entity_code = g.user.entity.entity_code + + owner_code = app.get_other_config(['train_restricted', 'owner_code']) + partner_codes = app.get_other_config(['train_restricted', 'partner_codes']) + + # user is OWNER, set full permission + if owner_code is None or entity_code == owner_code: + return True + + # user is PARTNER partners, set allowed permission + if partner_codes is None or any(entity_code in partner['codes'] and permission in partner['permissions'] for partner in partner_codes): + return True + + return False + + +def is_resource_train_restricted(path): + regex_resource = '^shared_data:[a-z]{2}_[a-z]{2}\/train_restricted(\/.*)*$' + regex_config = '^\$\{SHARED_DATA_TRAIN_DIR\}/[a-z]{2}_[a-z]{2}\/train_restricted(\/.*)*$' + + return re.search(regex_resource, path) or re.search(regex_config, path) From 2d6d49c121699707f3c8794570715924fe95d25e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=CC=80ng=20Phu=CC=9Bo=CC=9Bng=20Ngo=CC=82?= Date: Wed, 24 Mar 2021 08:50:17 +0700 Subject: [PATCH 10/31] dev/56253_handler_train_restricted_folder_permission: change function name, regex pattern --- server/utils/common_utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/utils/common_utils.py b/server/utils/common_utils.py index f9b24b5f..0b327cf6 100644 --- a/server/utils/common_utils.py +++ b/server/utils/common_utils.py @@ -4,7 +4,7 @@ from app import app -def is_allow_access_train_restricted(path, permission): +def verify_resource_train_restricted(path, permission): if not is_resource_train_restricted(path): return True @@ -32,7 +32,6 @@ def check_permission_access_train_restricted(permission): def is_resource_train_restricted(path): - regex_resource = '^shared_data:[a-z]{2}_[a-z]{2}\/train_restricted(\/.*)*$' - regex_config = '^\$\{SHARED_DATA_TRAIN_DIR\}/[a-z]{2}_[a-z]{2}\/train_restricted(\/.*)*$' + regex_pattern = '\/train_restricted(\/.*)*$' - return re.search(regex_resource, path) or re.search(regex_config, path) + return re.search(regex_pattern, path) From ef2f765ea3c9e8747bbdf0bee701fafe3cb8d256 Mon Sep 17 00:00:00 2001 From: touffet Date: Thu, 25 Mar 2021 20:31:09 +0100 Subject: [PATCH 11/31] refs #56292: auto partition corpus on new model training --- server/app/routes.py | 56 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/server/app/routes.py b/server/app/routes.py index dd1addb1..be495997 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -693,7 +693,7 @@ def launch_v2(): exists_dataset = get_dataset_by_name(entity_code, dataset_name) if exists_dataset: - return make_response(jsonify(message=f"Dataset \"{dataset_name}\" is already existed"), 400) + return make_response(jsonify(message=f"Dataset \"{dataset_name}\" already exists"), 400) data_file_info = get_data_file_info(request_data, routes_config) @@ -747,15 +747,21 @@ def parse_request_data(current_request): training_data = request_files.getlist("training_data") testing_data = request_files.getlist("testing_data") + model_data = request_files.getlist("model_data") dataset = request_data.getlist("dataset") corpus_type = int(request_data.get("corpus_type")) dataset_name = request_data.get("dataset_name") + testing_percent = request_data.get("testing_percent") + if testing_percent is not None: + testing_percent = int(testing_percent) return {**request_data, **{"tags": json.loads(tags)}, **{ "training_data": training_data, "testing_data": testing_data, + "model_data": model_data, "dataset": dataset, "corpus_type": corpus_type, + "testing_percent": testing_percent, "dataset_name": dataset_name }} @@ -775,8 +781,9 @@ def validate_request_data(current_request): validate_priority(request_data.get("priority")) validate_iteration(request_data.get("num_of_iteration")) - validate_file(request_data.get("corpus_type"), corpus_config, request_files.getlist("training_data"), - request_files.getlist("testing_data"), request_data.getlist("dataset")) + validate_file(request_data.get("corpus_type"), request_data.get("testing_percent"), corpus_config, + request_files.getlist("training_data"), request_files.getlist("testing_data"), + request_files.getlist("model_data"), request_data.getlist("dataset")) def validate_tags(tags): @@ -879,12 +886,34 @@ def upload_user_files(routes_config, path, files): return push_infos_list -def validate_file(corpus_type, corpus_config, training_data, testing_data, dataset): +def partition_and_upload_user_files(routes_config, training_path, testing_path, files, testing_percent): + training_push_infos_list = [] + testing_push_infos_list = [] + for file in files: + push_infos = routes_config.storage_client.partition_auto(file, + training_path, + testing_path, + remote_path=training_path, + storage_id=routes_config.global_storage_name, + percent=testing_percent) + + assert push_infos and push_infos['files'] and len(push_infos['files']) == 2 + training_file_info = push_infos['files'][0] + testing_file_info = push_infos['files'][1] + assert training_file_info['nbSegments'] and testing_file_info['nbSegments'] + training_push_infos_list.append(training_file_info) + testing_push_infos_list.append(testing_file_info) + return training_push_infos_list, testing_push_infos_list + + +def validate_file(corpus_type, testing_percent, corpus_config, training_data, testing_data, model_data, dataset): if not corpus_type or not corpus_type.isnumeric() or int(corpus_type) not in CORPUS_TYPE.values(): raise Exception('Invalid corpus_type') - if int(corpus_type) == CORPUS_TYPE["USER_UPLOAD"]: + if int(corpus_type) == CORPUS_TYPE["USER_UPLOAD"] and testing_percent is None: validate_training_data(training_data, corpus_config) validate_testing_data(testing_data, corpus_config) + elif int(corpus_type) == CORPUS_TYPE["USER_UPLOAD"] and testing_percent is not None: + validate_training_data(model_data, corpus_config) else: if len(dataset) == 0: raise Exception('Num of dataset must greater than 0') @@ -899,9 +928,13 @@ def get_dataset_by_name(entity, dataset_name): def get_data_file_info(request_data, routes_config): corpus_type = request_data.get("corpus_type") + testing_percent = request_data.get("testing_percent") if corpus_type == CORPUS_TYPE["USER_UPLOAD"]: - training_data = request_data.get("training_data") + if testing_percent is not None: + training_data = request_data.get("model_data") + else: + training_data = request_data.get("training_data") testing_data = request_data.get("testing_data") return get_user_upload_file_info(routes_config, request_data, training_data, testing_data) @@ -915,13 +948,18 @@ def get_data_file_info(request_data, routes_config): def get_user_upload_file_info(routes_config, request_data, training_data, testing_data): entity_code = routes_config.creator['entity_code'] dataset_name = request_data.get('dataset_name') + testing_percent = request_data.get("testing_percent") training_data_path = os.path.join(entity_code, dataset_name, "train") + os.path.sep testing_data_path = os.path.join(entity_code, dataset_name, "test") + os.path.sep - data_training = upload_user_files(routes_config, training_data_path, training_data) - data_testing = upload_user_files(routes_config, testing_data_path, testing_data) - + if (testing_percent is None): + data_training = upload_user_files(routes_config, training_data_path, training_data) + data_testing = upload_user_files(routes_config, testing_data_path, testing_data) + else: + training_data_path = "/" + os.path.join(entity_code, dataset_name, "train") + os.path.sep + testing_data_path = "/" + os.path.join(entity_code, dataset_name, "test") + os.path.sep + data_training, data_testing = partition_and_upload_user_files(routes_config, training_data_path, testing_data_path, training_data, testing_percent) create_model_dataset(routes_config, request_data, GLOBAL_POOL_NAME) dataset = get_dataset_by_name(entity_code, dataset_name) From 45ce023591f9c9531b7d57552020f4085b7ba984 Mon Sep 17 00:00:00 2001 From: touffet Date: Fri, 26 Mar 2021 13:11:53 +0100 Subject: [PATCH 12/31] refs #56292: only allow testing_percent to be an int > 0 --- server/app/routes.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/app/routes.py b/server/app/routes.py index be495997..b953ef3b 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -752,7 +752,7 @@ def parse_request_data(current_request): corpus_type = int(request_data.get("corpus_type")) dataset_name = request_data.get("dataset_name") testing_percent = request_data.get("testing_percent") - if testing_percent is not None: + if testing_percent: testing_percent = int(testing_percent) return {**request_data, **{"tags": json.loads(tags)}, **{ @@ -912,7 +912,7 @@ def validate_file(corpus_type, testing_percent, corpus_config, training_data, te if int(corpus_type) == CORPUS_TYPE["USER_UPLOAD"] and testing_percent is None: validate_training_data(training_data, corpus_config) validate_testing_data(testing_data, corpus_config) - elif int(corpus_type) == CORPUS_TYPE["USER_UPLOAD"] and testing_percent is not None: + elif int(corpus_type) == CORPUS_TYPE["USER_UPLOAD"] and testing_percent: validate_training_data(model_data, corpus_config) else: if len(dataset) == 0: @@ -931,7 +931,7 @@ def get_data_file_info(request_data, routes_config): testing_percent = request_data.get("testing_percent") if corpus_type == CORPUS_TYPE["USER_UPLOAD"]: - if testing_percent is not None: + if testing_percent: training_data = request_data.get("model_data") else: training_data = request_data.get("training_data") @@ -953,13 +953,13 @@ def get_user_upload_file_info(routes_config, request_data, training_data, testin training_data_path = os.path.join(entity_code, dataset_name, "train") + os.path.sep testing_data_path = os.path.join(entity_code, dataset_name, "test") + os.path.sep - if (testing_percent is None): - data_training = upload_user_files(routes_config, training_data_path, training_data) - data_testing = upload_user_files(routes_config, testing_data_path, testing_data) - else: + if testing_percent: training_data_path = "/" + os.path.join(entity_code, dataset_name, "train") + os.path.sep testing_data_path = "/" + os.path.join(entity_code, dataset_name, "test") + os.path.sep data_training, data_testing = partition_and_upload_user_files(routes_config, training_data_path, testing_data_path, training_data, testing_percent) + else: + data_training = upload_user_files(routes_config, training_data_path, training_data) + data_testing = upload_user_files(routes_config, testing_data_path, testing_data) create_model_dataset(routes_config, request_data, GLOBAL_POOL_NAME) dataset = get_dataset_by_name(entity_code, dataset_name) From be3fafbe1cf560842cf07c17a4cbb19d6d798c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=CC=80ng=20Phu=CC=9Bo=CC=9Bng=20Ngo=CC=82?= Date: Mon, 29 Mar 2021 16:02:15 +0700 Subject: [PATCH 13/31] refs #55454: Add argument entity_owner --- client/launcher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/launcher.py b/client/launcher.py index 3c4dd037..61fbd597 100644 --- a/client/launcher.py +++ b/client/launcher.py @@ -187,6 +187,7 @@ def confirm(prompt=None, resp=False): for arg in exec_arguments: parser_exec.add_argument(*arg[:-1], **arg[-1]) parser_exec.add_argument('docker_command', type=str, nargs='*', help='Docker command') +parser_exec.add_argument('-e', '--entity_owner', help='entity owner') parser_launch = subparsers.add_parser('launch', help='launch a task on the service associated' From 0d138337c667f277a3372cb06a48ddc77cf109a5 Mon Sep 17 00:00:00 2001 From: Dang Chuan Nguyen Date: Fri, 2 Apr 2021 22:29:32 +0200 Subject: [PATCH 14/31] refs #56292: [*] use tmp file to partition call --- server/app/routes.py | 5 ++++- server/nmtwizard/task.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/server/app/routes.py b/server/app/routes.py index b953ef3b..468d2390 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -889,8 +889,11 @@ def upload_user_files(routes_config, path, files): def partition_and_upload_user_files(routes_config, training_path, testing_path, files, testing_percent): training_push_infos_list = [] testing_push_infos_list = [] + temp_files = tempfile.mkdtemp() for file in files: - push_infos = routes_config.storage_client.partition_auto(file, + tmp_file = os.path.join(temp_files, file.filename) + file.save(tmp_file) + push_infos = routes_config.storage_client.partition_auto(tmp_file, training_path, testing_path, remote_path=training_path, diff --git a/server/nmtwizard/task.py b/server/nmtwizard/task.py index e7c263a7..e3618801 100644 --- a/server/nmtwizard/task.py +++ b/server/nmtwizard/task.py @@ -128,7 +128,7 @@ def get_docker_image_info(routes_config, docker_image, mongo_client): def get_docker_image_from_db(service_module, mongo_client): image = "systran/pn9_tf" registry = get_registry(service_module, image) - tag = "v1.46.0-beta1" + tag = "v1.49.0" result = { "image": image, @@ -258,7 +258,7 @@ def __init__(self, task_infos, parent_task_id, model, to_score): task_infos.content["docker"] = { "image": image_score, "registry": get_registry(task_infos.routes_configuration.service_module, image_score), - "tag": "2.1.0-beta1", + "tag": "latest", "command": [] } From 48df6301e91a6d607f934f1c106aae8a6dd1169e Mon Sep 17 00:00:00 2001 From: Pierre-jean Liard Date: Thu, 8 Apr 2021 11:25:54 +0200 Subject: [PATCH 15/31] refs #54931: Fix tag format --- server/app/routes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/app/routes.py b/server/app/routes.py index dd1addb1..f1852af3 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1227,6 +1227,7 @@ def create_model_catalog(training_task_id, input_name, request_data, image_tag, source = request_data.get("source") target = request_data.get("target") parent_model = request_data.get("parent_model") + tags = [{'entity': tag.get('entity'), 'tag': tag.get('tag')} for tag in tags] config = { "source": source, "target": target, From 8ac6979f292aa8707886574b166683d0ac6e1720 Mon Sep 17 00:00:00 2001 From: Pham Duy Hieu Date: Tue, 13 Apr 2021 09:21:30 +0700 Subject: [PATCH 16/31] refs #56045: allow GPU trans in as_release mode --- client/launcher.py | 9 +++++++-- server/app/routes.py | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/client/launcher.py b/client/launcher.py index 61fbd597..0a1f2ad7 100644 --- a/client/launcher.py +++ b/client/launcher.py @@ -209,8 +209,7 @@ def confirm(prompt=None, resp=False): help='options selected to run the service') parser_launch.add_argument('-r', '--resource', help="alternatively to `options`, resource name to use") -parser_launch.add_argument('-g', '--gpus', type=int, default=1, - help='number of gpus') +parser_launch.add_argument('-g', '--gpus', type=int, help='number of gpus') parser_launch.add_argument('-c', '--cpus', type=int, help='number of cpus - if not provided, ' 'will be obtained from pool config') parser_launch.add_argument('-w', '--wait_after_launch', default=2, type=int, @@ -466,6 +465,12 @@ def process_request(service_list, cmd, subcmd, is_json, args, auth=None): if args.service not in service_list: raise ValueError("ERROR: service '%s' not defined" % args.service) + if args.gpus is None: + if "trans" in args.docker_command: + args.gpus = 0 + else: + args.gpus = 1 + if args.gpus < 0: raise ValueError("ERROR: ngpus must be >= 0") diff --git a/server/app/routes.py b/server/app/routes.py index 468d2390..55dbd40d 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1748,7 +1748,6 @@ def launch(service): if task_type == "trans" and can_trans_as_release: if "--as_release" not in content["docker"]["command"] and trans_as_release: content["docker"]["command"].append("--as_release") - content["ngpus"] = ngpus = 0 task_resource = service_module.select_resource_from_capacity( resource, Capacity(content["ngpus"], From a04ba9a90baf49793838ad7133c5abffe5e59031 Mon Sep 17 00:00:00 2001 From: Pierre-jean Liard Date: Tue, 13 Apr 2021 18:11:47 +0200 Subject: [PATCH 17/31] refs #56551: Fix client sample size --- server/app/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app/routes.py b/server/app/routes.py index 55dbd40d..c83504e2 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1156,12 +1156,12 @@ def get_sample_data(current_data, parent_data, sample_by_path): for current_sample_dist in current_sample_dists: duplicate = list(filter(lambda sample_dist: (current_sample_dist['path'] == sample_dist['path']), sample_dists)) + client_sample += int(sample_by_path[current_sample_dist['path']]) if len(duplicate) > 0: continue new_sample_dists.append(current_sample_dist) - client_sample += int(sample_by_path[current_sample_dist['path']]) new_sample_size = app.get_other_config(['training_options', 'sample_size'], fallback=10000000) client_weight = get_client_weight(new_sample_size, client_ratio, client_sample) From 68a9aa73461ad800070532601a3a2c128e3a2689 Mon Sep 17 00:00:00 2001 From: Pierre-jean Liard Date: Wed, 14 Apr 2021 16:11:30 +0200 Subject: [PATCH 18/31] refs #54931: Add tag option in snw client to launch training --- client/launcher.py | 15 ++++++++++ server/app/routes.py | 65 ++++++++++++++++++++++++++++++-------------- 2 files changed, 60 insertions(+), 20 deletions(-) diff --git a/client/launcher.py b/client/launcher.py index 61fbd597..9f8e4310 100644 --- a/client/launcher.py +++ b/client/launcher.py @@ -237,6 +237,7 @@ def confirm(prompt=None, resp=False): parser_launch.add_argument('--notransasrelease', action='store_true', help='don\'t run translate as release (image >= 1.8.0)') parser_launch.add_argument('docker_command', type=str, nargs='*', help='Docker command') +parser_launch.add_argument('--tag', nargs='*', help='model tags') parser_list_tasks = subparsers_tasks.add_parser('list', help='{lt} list tasks matching prefix pattern') @@ -517,6 +518,20 @@ def process_request(service_list, cmd, subcmd, is_json, args, auth=None): content["totuminer"] = [(_parse_local_filename(i, files), o) for (i, o) in ARGS.totuminer] + if args.tag: + content['tags'] = [] + for tag in args.tag: + tmp = {} + entity = '' + if ':' in tag: + entity, tag_name = tag.split(':') + else: + tag_name = tag + tmp['tag'] = tag_name + if entity: + tmp['entity'] = entity + content['tags'].append(tmp) + LOGGER.debug("sending request: %s", json.dumps(content)) launch_url = os.path.join(args.url, "task/launch", args.service) diff --git a/server/app/routes.py b/server/app/routes.py index f1852af3..cb5942d3 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -581,15 +581,17 @@ def create_tasks_for_launch_v2(creation_infos): "tasks_id": tasks_id, "train_task_id": train_task_id } - + # Create tasks for parent model - to_translate_corpus_for_parent_model = [corpus for corpus in creation_infos.to_translate_corpus if SYSTRAN_BASE_STORAGE not in corpus[0]] - to_score_corpus_for_parent_model = [corpus for corpus in creation_infos.to_score_corpus if SYSTRAN_BASE_STORAGE not in corpus[0]] - + to_translate_corpus_for_parent_model = [corpus for corpus in creation_infos.to_translate_corpus if + SYSTRAN_BASE_STORAGE not in corpus[0]] + to_score_corpus_for_parent_model = [corpus for corpus in creation_infos.to_score_corpus if + SYSTRAN_BASE_STORAGE not in corpus[0]] + create_trans_score_tasks_for_model(creation_infos.task_infos.request_data.get("parent_model"), - to_translate_corpus_for_parent_model, - to_score_corpus_for_parent_model, - creation_infos.task_infos.request_data) + to_translate_corpus_for_parent_model, + to_score_corpus_for_parent_model, + creation_infos.task_infos.request_data) return creation_output @@ -621,7 +623,8 @@ def create_trans_score_tasks(creation_infos, model, docker_content, parent_model to_translate_corpus, to_score_corpus = creation_infos.to_translate_corpus, creation_infos.to_score_corpus if model_info.get('tests'): - to_translate_corpus, to_score_corpus = get_only_new_test_corpus(model_info.get('tests'), to_translate_corpus, to_score_corpus) + to_translate_corpus, to_score_corpus = get_only_new_test_corpus(model_info.get('tests'), to_translate_corpus, + to_score_corpus) if not to_translate_corpus: return @@ -686,7 +689,7 @@ def launch_v2(): dataset_name = request_data.get("dataset_name") if dataset_name is None: - abort(make_response(jsonify(message="unknown dataset"), 400)) + abort(make_response(jsonify(message="unknown dataset"), 400)) entity_code = routes_config.creator['entity_code'] @@ -927,8 +930,8 @@ def get_user_upload_file_info(routes_config, request_data, training_data, testin dataset = get_dataset_by_name(entity_code, dataset_name) return { - 'training': list(map(lambda ele: { **ele, 'dataset_id': str(dataset["_id"]) }, data_training)), - 'testing': list(map(lambda ele: { **ele, 'dataset_id': str(dataset["_id"]) }, data_testing)) + 'training': list(map(lambda ele: {**ele, 'dataset_id': str(dataset["_id"])}, data_training)), + 'testing': list(map(lambda ele: {**ele, 'dataset_id': str(dataset["_id"])}, data_testing)) } @@ -1033,7 +1036,7 @@ def get_final_training_config(request_data, training_corpus_infos): boolean_param(request.args.get('short'))) if ok: # Change batch size to settings value if specified - if "config" in parent_config["options"] and "train" in parent_config["options"]["config"]\ + if "config" in parent_config["options"] and "train" in parent_config["options"]["config"] \ and "batch_size" in parent_config["options"]["config"]["train"]: batch_size = app.get_other_config(['training_options', 'batch_size'], fallback=None) if batch_size: @@ -1084,7 +1087,7 @@ def apply(sampling_rule): for storage_block in distribution: if storage_block.get('distribution'): - storage_block['distribution'] = list(map(apply, storage_block.get('distribution'))) + storage_block['distribution'] = list(map(apply, storage_block.get('distribution'))) return distribution @@ -1124,8 +1127,10 @@ def get_sample_data(current_data, parent_data, sample_by_path): new_sample_size = app.get_other_config(['training_options', 'sample_size'], fallback=10000000) client_weight = get_client_weight(new_sample_size, client_ratio, client_sample) - sample_dists = adapt_distribution_proportions(sample_dists, get_parent_formula_distribution_proportions, client_ratio) - new_sample_dists = adapt_distribution_proportions(new_sample_dists, get_client_formula_distribution_proportions, client_weight, is_parent=False) + sample_dists = adapt_distribution_proportions(sample_dists, get_parent_formula_distribution_proportions, + client_ratio) + new_sample_dists = adapt_distribution_proportions(new_sample_dists, get_client_formula_distribution_proportions, + client_weight, is_parent=False) new_sample_dists.extend(sample_dists) return new_sample_size, new_sample_dists @@ -1223,7 +1228,8 @@ def create_model_dataset(routes_config, request_data, service): return builtins.pn9model_db.insert_dataset(item) -def create_model_catalog(training_task_id, input_name, request_data, image_tag, creator, tasks, tags, domain, user_corpus, state="creating"): +def create_model_catalog(training_task_id, input_name, request_data, image_tag, creator, tasks, tags, domain, + user_corpus, state="creating"): source = request_data.get("source") target = request_data.get("target") parent_model = request_data.get("parent_model") @@ -1314,7 +1320,8 @@ def create_evaluation(): models = request_data.get("models") testing_info = upload_user_files(routes_config, f"{upload_path}/test/", request_data.get('corpus')) - to_translate_corpus, to_score_corpus = get_translate_score_corpus(testing_info, request_data, routes_config, False, output_path) + to_translate_corpus, to_score_corpus = get_translate_score_corpus(testing_info, request_data, routes_config, False, + output_path) docker_image_info = TaskBase.get_docker_image_info(routes_config, request_data.get("docker_image"), mongo_client) docker_content = {**docker_image_info, **{"command": []}} @@ -1489,6 +1496,19 @@ def add_train_restricted_config(json_config, parent_task_id): return json.dumps(config) +def parse_tags(tags): + result = {'existed': []} + for tag in tags: + tag_name = tag.get('tag') + entity = tag.get('entity', '') + if not entity: + entity = flask.g.user.entity.entity_code + info_tag = builtins.pn9model_db.tag_get(entity, tag_name) + if info_tag: + result['existed'].append(str(info_tag.get('_id'))) + return result + + @app.route("/task/launch/", methods=["POST"]) @filter_request("POST/task/launch", "train") def launch(service): @@ -1503,6 +1523,10 @@ def launch(service): else: abort(flask.make_response(flask.jsonify(message="missing content in request"), 400)) + if content.get('tags') and isinstance(content.get('tags'), list): + parsed_tags = parse_tags(content.get('tags')) + content['tags'] = parsed_tags + files = {} for k in flask.request.files: files[k] = flask.request.files[k].read() @@ -1813,7 +1837,7 @@ def launch(service): "registry": get_registry(service_module, image_score), "tag": "latest", "command": ["score", "-o"] + oref["output"] + ["-r"] + oref["ref"] + - option_lang + ['-f', "launcher:scores"] + option_lang + ['-f', "launcher:scores"] } score_task_id, explicit_name = build_task_id(content_score, xxyy, "score", parent_task_id) @@ -1868,7 +1892,7 @@ def launch(service): "registry": get_registry(service_module, image_score), "tag": "latest", "command": ["tuminer", "--tumode", "score", "--srcfile"] + in_out["infile"] + ["--tgtfile"] + - in_out["outfile"] + ["--output"] + in_out["scorefile"] + in_out["outfile"] + ["--output"] + in_out["scorefile"] } tuminer_task_id, explicit_name = build_task_id(content_tuminer, xxyy, "tuminer", parent_task_id) @@ -2216,6 +2240,7 @@ def get_all_files_of_dataset(dataset_path, global_storage_name, storage_client): continue directories = storage_client.list(data_path, storage_id=global_storage_name) for k, v in directories.items(): - result[key].append({**v, **{"filename": k if k.startswith('/') else '/' + k, "nbSegments": v.get("entries")}}) + result[key].append( + {**v, **{"filename": k if k.startswith('/') else '/' + k, "nbSegments": v.get("entries")}}) return result From 71af679db1daf4039b5e2fb6591a19b13ae4d306 Mon Sep 17 00:00:00 2001 From: Pierre-jean Liard Date: Thu, 15 Apr 2021 16:49:27 +0200 Subject: [PATCH 19/31] refs #54931: Change snw tags option : --tag -> --tags --- client/launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/launcher.py b/client/launcher.py index 9f8e4310..af2d7fcc 100644 --- a/client/launcher.py +++ b/client/launcher.py @@ -237,7 +237,7 @@ def confirm(prompt=None, resp=False): parser_launch.add_argument('--notransasrelease', action='store_true', help='don\'t run translate as release (image >= 1.8.0)') parser_launch.add_argument('docker_command', type=str, nargs='*', help='Docker command') -parser_launch.add_argument('--tag', nargs='*', help='model tags') +parser_launch.add_argument('--tags', nargs='*', help='model tags') parser_list_tasks = subparsers_tasks.add_parser('list', help='{lt} list tasks matching prefix pattern') From 476625c23e56cab46e7b0d08c54158bbd37198ea Mon Sep 17 00:00:00 2001 From: Pierre-jean Liard Date: Thu, 15 Apr 2021 16:49:54 +0200 Subject: [PATCH 20/31] refs #54931: Add comment for snw --- server/app/routes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/app/routes.py b/server/app/routes.py index cb5942d3..32d73be3 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1523,6 +1523,7 @@ def launch(service): else: abort(flask.make_response(flask.jsonify(message="missing content in request"), 400)) + # Parse tags from snw client if content.get('tags') and isinstance(content.get('tags'), list): parsed_tags = parse_tags(content.get('tags')) content['tags'] = parsed_tags From 8f99e3690e6f99f155c80a22a092c7b300ecc1da Mon Sep 17 00:00:00 2001 From: Pierre-jean Liard Date: Thu, 15 Apr 2021 16:58:55 +0200 Subject: [PATCH 21/31] refs #54931: Code refactoring for tags option --- client/launcher.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/client/launcher.py b/client/launcher.py index af2d7fcc..ed629147 100644 --- a/client/launcher.py +++ b/client/launcher.py @@ -518,18 +518,13 @@ def process_request(service_list, cmd, subcmd, is_json, args, auth=None): content["totuminer"] = [(_parse_local_filename(i, files), o) for (i, o) in ARGS.totuminer] - if args.tag: + if args.tags: content['tags'] = [] - for tag in args.tag: - tmp = {} - entity = '' + for tag in args.tags: + tmp = {'tag': tag} if ':' in tag: entity, tag_name = tag.split(':') - else: - tag_name = tag - tmp['tag'] = tag_name - if entity: - tmp['entity'] = entity + tmp = {'entity': entity, 'tag': tag_name} content['tags'].append(tmp) LOGGER.debug("sending request: %s", json.dumps(content)) From 41add4f7f2b66acf3cae167bfba91e51e0077dd3 Mon Sep 17 00:00:00 2001 From: Pierre-jean Liard Date: Thu, 15 Apr 2021 17:05:23 +0200 Subject: [PATCH 22/31] refs #54931: Return 400 status when tag is invalid --- server/app/routes.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/app/routes.py b/server/app/routes.py index 32d73be3..ef5e8b9e 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1506,7 +1506,9 @@ def parse_tags(tags): info_tag = builtins.pn9model_db.tag_get(entity, tag_name) if info_tag: result['existed'].append(str(info_tag.get('_id'))) - return result + else: + return False, result + return True, result @app.route("/task/launch/", methods=["POST"]) @@ -1525,7 +1527,10 @@ def launch(service): # Parse tags from snw client if content.get('tags') and isinstance(content.get('tags'), list): - parsed_tags = parse_tags(content.get('tags')) + res, parsed_tags = parse_tags(content.get('tags')) + if not res: + abort(flask.make_response(flask.jsonify(message="Invalid tags"), 400)) + content['tags'] = parsed_tags files = {} From 1cc03124e8756acdc9df1a4c1a12a0c679ed8cf7 Mon Sep 17 00:00:00 2001 From: knguyen Date: Thu, 15 Apr 2021 16:17:33 +0700 Subject: [PATCH 23/31] refs #56433: Allow evaluation with Google model in mode lite --- server/app/routes.py | 41 ++++++++++++++++++++++++++++++++++++---- server/nmtwizard/task.py | 21 ++++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/server/app/routes.py b/server/app/routes.py index 55dbd40d..dbfa0433 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1288,6 +1288,12 @@ def create_tasks_for_evaluation(creation_infos, models, evaluation_id, docker_co model_task_map = {} tasks_to_create = [] models_info = [] + google_docker_content = {} + if 'google_image_info' in docker_content: + google_docker_content = docker_content['google_image_info'] + docker_content.pop('google_image_info') + google_docker_content['command'] = [] + for model in models: tasks_id_per_model = [] tasks_to_create_per_model = [] @@ -1300,10 +1306,23 @@ def create_tasks_for_evaluation(creation_infos, models, evaluation_id, docker_co "evaluation_id": str(evaluation_id), "eval_model": model } - creation_infos.task_infos.content["docker"] = docker_content - task_translate = TaskTranslate(task_infos=creation_infos.task_infos, - parent_task_id=model, - to_translate=creation_infos.to_translate_corpus) + if check_google_model(model): + creation_infos.task_infos.content["docker"] = google_docker_content + task_translate = TaskTranslate(task_infos=creation_infos.task_infos, + parent_task_id=model, + to_translate=creation_infos.to_translate_corpus) + lang_config = { + "source": creation_infos.task_infos.request_data['source'], + "target": creation_infos.task_infos.request_data['target'] + } + config = ['-c', json.dumps(lang_config)] + task_translate.update_content_docker_command(config) + else: + creation_infos.task_infos.content["docker"] = docker_content + task_translate = TaskTranslate(task_infos=creation_infos.task_infos, + parent_task_id=model, + to_translate=creation_infos.to_translate_corpus) + tasks_to_create_per_model.append(task_translate) tasks_id_per_model.append(task_translate.task_id) @@ -1357,6 +1376,13 @@ def create_evaluation(): to_translate_corpus, to_score_corpus = get_translate_score_corpus(testing_info, request_data, routes_config, False, output_path) docker_image_info = TaskBase.get_docker_image_info(routes_config, request_data.get("docker_image"), mongo_client) + for model in models: + if check_google_model(model): + google_docker_image_info = TaskBase.get_google_docker_image_from_db(routes_config.service_module, + mongo_client) + docker_image_info['google_image_info'] = google_docker_image_info + break + docker_content = {**docker_image_info, **{"command": []}} content = { "docker": {}, @@ -2258,3 +2284,10 @@ def get_all_files_of_dataset(dataset_path, global_storage_name, storage_client): result[key].append({**v, **{"filename": k if k.startswith('/') else '/' + k, "nbSegments": v.get("entries")}}) return result + + +def check_google_model(model): + if model.split('_')[0].lower() == 'google': + return True + else: + return False diff --git a/server/nmtwizard/task.py b/server/nmtwizard/task.py index e3618801..01e89ae2 100644 --- a/server/nmtwizard/task.py +++ b/server/nmtwizard/task.py @@ -78,6 +78,9 @@ def __init__(self, task_infos, must_patch_config_name=False): def update_other_infos(self, other_infos): self.other_task_info.update(other_infos) + def update_content_docker_command(self, infos): + self._content['docker']['command'] = infos + self._content['docker']['command'] + def create(self, redis_db, taskfile_dir): create_internal(redis_db, taskfile_dir, @@ -124,6 +127,24 @@ def get_docker_image_info(routes_config, docker_image, mongo_client): return TaskBase.get_docker_image_from_request(routes_config.service_module, routes_config.entity_owner, docker_image) + @staticmethod + def get_google_docker_image_from_db(service_module, mongo_client): + image = "nmtwizard/google-translate" + registry = get_registry(service_module, image) + tag = "2.9.4" + + result = { + "image": image, + "tag": tag, + "registry": registry + } + + latest_docker_image_tag = TaskBase.get_latest_docker_image_tag(image, mongo_client) + + if not latest_docker_image_tag: + return result + return {**result, **{"tag": f'{latest_docker_image_tag}'}} + @staticmethod def get_docker_image_from_db(service_module, mongo_client): image = "systran/pn9_tf" From a5ce3e46498f757d7ae54ac7a8eb412eadd26e0f Mon Sep 17 00:00:00 2001 From: Pierre-jean Liard Date: Fri, 16 Apr 2021 13:36:52 +0200 Subject: [PATCH 24/31] refs #54931: Fix parse tag function --- server/app/routes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/app/routes.py b/server/app/routes.py index ef5e8b9e..f5c63a4b 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1497,7 +1497,7 @@ def add_train_restricted_config(json_config, parent_task_id): def parse_tags(tags): - result = {'existed': []} + result = [] for tag in tags: tag_name = tag.get('tag') entity = tag.get('entity', '') @@ -1505,7 +1505,7 @@ def parse_tags(tags): entity = flask.g.user.entity.entity_code info_tag = builtins.pn9model_db.tag_get(entity, tag_name) if info_tag: - result['existed'].append(str(info_tag.get('_id'))) + result.append({'tag': tag_name, 'entity': entity}) else: return False, result return True, result @@ -1525,7 +1525,7 @@ def launch(service): else: abort(flask.make_response(flask.jsonify(message="missing content in request"), 400)) - # Parse tags from snw client + # Parse tags if content.get('tags') and isinstance(content.get('tags'), list): res, parsed_tags = parse_tags(content.get('tags')) if not res: From 05cb87bd5e0266ea607bb5ae8b4d5428374190b2 Mon Sep 17 00:00:00 2001 From: touffet Date: Wed, 28 Apr 2021 15:38:37 +0200 Subject: [PATCH 25/31] refs #56646: add parent_task option to snw launch --- client/launcher.py | 4 ++++ server/app/routes.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/client/launcher.py b/client/launcher.py index 9d57e901..ab863051 100644 --- a/client/launcher.py +++ b/client/launcher.py @@ -237,6 +237,7 @@ def confirm(prompt=None, resp=False): help='don\'t run translate as release (image >= 1.8.0)') parser_launch.add_argument('docker_command', type=str, nargs='*', help='Docker command') parser_launch.add_argument('--tags', nargs='*', help='model tags') +parser_launch.add_argument('--parent_task', help='parent task ID') parser_list_tasks = subparsers_tasks.add_parser('list', help='{lt} list tasks matching prefix pattern') @@ -532,6 +533,9 @@ def process_request(service_list, cmd, subcmd, is_json, args, auth=None): tmp = {'entity': entity, 'tag': tag_name} content['tags'].append(tmp) + if args.parent_task: + content['parent_task'] = args.parent_task + LOGGER.debug("sending request: %s", json.dumps(content)) launch_url = os.path.join(args.url, "task/launch", args.service) diff --git a/server/app/routes.py b/server/app/routes.py index 16db45c6..58be0762 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1720,13 +1720,15 @@ def launch(service): priority = content.get("priority", 0) (xxyy, parent_task_id) = shallow_command_analysis(content["docker"]["command"]) + if "parent_task" in content: + parent_task_id = content.get("parent_task") parent_struct = None parent_task_type = None if not exec_mode and parent_task_id: (parent_struct, parent_task_type) = model_name_analysis(parent_task_id) # check that parent model type matches current command - if parent_task_type: + if parent_task_type and "parent_task" not in content: if (parent_task_type == "trans" or parent_task_type == "relea" or (task_type == "prepr" and parent_task_type != "train" and parent_task_type != "vocab")): abort(flask.make_response(flask.jsonify(message="invalid parent task type: %s" % parent_task_type), 400)) From 0e44ee2f33efce28d3e00bbcc056c963e7285aa1 Mon Sep 17 00:00:00 2001 From: touffet Date: Fri, 30 Apr 2021 14:48:19 +0200 Subject: [PATCH 26/31] refs #56678: ngpu is 1 for train or 0 otherwise --- server/app/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app/routes.py b/server/app/routes.py index 16db45c6..22ed66e8 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1665,7 +1665,7 @@ def launch(service): if (task_type != "train" and iterations != 1) or iterations < 1: abort(flask.make_response(flask.jsonify(message="invalid value for iterations"), 400)) - ngpus = 0 if task_type == TASK_RELEASE_TYPE else 1 + ngpus = 1 if task_type == "train" else 0 if "ngpus" in content: ngpus = content["ngpus"] ncpus = content.get("ncpus") From c39761b615c40ff8ef9b4b977a1ed1dd539e725d Mon Sep 17 00:00:00 2001 From: touffet Date: Mon, 3 May 2021 16:46:43 +0200 Subject: [PATCH 27/31] refs #56365: change parent_task to "dependency" --- client/launcher.py | 8 ++++---- server/app/routes.py | 14 ++++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/client/launcher.py b/client/launcher.py index ab863051..bef5e7bf 100644 --- a/client/launcher.py +++ b/client/launcher.py @@ -188,6 +188,7 @@ def confirm(prompt=None, resp=False): parser_exec.add_argument(*arg[:-1], **arg[-1]) parser_exec.add_argument('docker_command', type=str, nargs='*', help='Docker command') parser_exec.add_argument('-e', '--entity_owner', help='entity owner') +parser_exec.add_argument('--dependency', help='dependency task (will launch after this task is terminated)') parser_launch = subparsers.add_parser('launch', help='launch a task on the service associated' @@ -237,7 +238,7 @@ def confirm(prompt=None, resp=False): help='don\'t run translate as release (image >= 1.8.0)') parser_launch.add_argument('docker_command', type=str, nargs='*', help='Docker command') parser_launch.add_argument('--tags', nargs='*', help='model tags') -parser_launch.add_argument('--parent_task', help='parent task ID') +parser_launch.add_argument('--dependency', help='dependency task (will launch after this task is terminated)') parser_list_tasks = subparsers_tasks.add_parser('list', help='{lt} list tasks matching prefix pattern') @@ -501,6 +502,8 @@ def process_request(service_list, cmd, subcmd, is_json, args, auth=None): content["name"] = args.name if args.priority: content["priority"] = args.priority + if args.dependency: + content["dependency"] = args.dependency if cmd == "task" and subcmd == "launch": if args.iterations: @@ -533,9 +536,6 @@ def process_request(service_list, cmd, subcmd, is_json, args, auth=None): tmp = {'entity': entity, 'tag': tag_name} content['tags'].append(tmp) - if args.parent_task: - content['parent_task'] = args.parent_task - LOGGER.debug("sending request: %s", json.dumps(content)) launch_url = os.path.join(args.url, "task/launch", args.service) diff --git a/server/app/routes.py b/server/app/routes.py index 58be0762..ac3eef88 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1720,15 +1720,13 @@ def launch(service): priority = content.get("priority", 0) (xxyy, parent_task_id) = shallow_command_analysis(content["docker"]["command"]) - if "parent_task" in content: - parent_task_id = content.get("parent_task") parent_struct = None parent_task_type = None if not exec_mode and parent_task_id: (parent_struct, parent_task_type) = model_name_analysis(parent_task_id) # check that parent model type matches current command - if parent_task_type and "parent_task" not in content: + if parent_task_type: if (parent_task_type == "trans" or parent_task_type == "relea" or (task_type == "prepr" and parent_task_type != "train" and parent_task_type != "vocab")): abort(flask.make_response(flask.jsonify(message="invalid parent task type: %s" % parent_task_type), 400)) @@ -1742,11 +1740,15 @@ def launch(service): task_ids = [] task_create = [] - + first_of_chain = True while iterations > 0: if (chain_prepr_train and parent_task_type != "prepr") or task_type == "prepr": prepr_task_id, explicit_name = build_task_id(content, xxyy, "prepr", parent_task_id) + if "dependency" in content and first_of_chain: + parent_task_id = content["dependency"] + first_of_chain = False + if explicit_name: TaskBase.patch_config_explicit_name(content, explicit_name) @@ -1788,6 +1790,10 @@ def launch(service): task_id, explicit_name = build_task_id(content, xxyy, task_suffix, parent_task_id) + if "dependency" in content and first_of_chain: + parent_task_id = content["dependency"] + first_of_chain = False + if explicit_name: TaskBase.patch_config_explicit_name(content, explicit_name) From f3a82ca54db5e7cd8e15a2c46d9afbd790dc4192 Mon Sep 17 00:00:00 2001 From: touffet Date: Tue, 4 May 2021 17:29:53 +0200 Subject: [PATCH 28/31] refs #56678: 0 gpus by default for score --- client/launcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/launcher.py b/client/launcher.py index 9d57e901..63b05358 100644 --- a/client/launcher.py +++ b/client/launcher.py @@ -159,7 +159,7 @@ def confirm(prompt=None, resp=False): "help": 'options selected to run the service'}], ['-r', '--resource', {"help": "alternatively to `options`, resource name to use"}], - ['-g', '--gpus', {"type": int, "default": 1, "help": 'number of gpus'}], + ['-g', '--gpus', {"type": int, "help": 'number of gpus'}], ['-c', '--cpus', {"type": int, "help": 'number of cpus - if not provided, will be obtained ' 'from pool config'}], @@ -467,7 +467,7 @@ def process_request(service_list, cmd, subcmd, is_json, args, auth=None): raise ValueError("ERROR: service '%s' not defined" % args.service) if args.gpus is None: - if "trans" in args.docker_command: + if "trans" in args.docker_command or "score" in args.docker_command: args.gpus = 0 else: args.gpus = 1 From dfa85638ed794d88cd58645fea7edbabf33b3b61 Mon Sep 17 00:00:00 2001 From: trungkienbkhn Date: Mon, 10 May 2021 17:28:07 +0700 Subject: [PATCH 29/31] refs #56381: Fix token expire after 1 day between: the end of preprocess and the beginning of train task --- server/app/routes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/app/routes.py b/server/app/routes.py index 55dbd40d..4ad489c9 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1717,7 +1717,7 @@ def launch(service): (redis_db, taskfile_dir, prepr_task_id, "prepr", parent_task_id, preprocess_resource, service, _duplicate_adapt(service_module, content), - files, priority, 0, content["ncpus"], other_task_info)) + files, priority, 0, content["ncpus"], deepcopy(other_task_info))) task_ids.append( "%s\t%s\tngpus: %d, ncpus: %d" % ("prepr", prepr_task_id, 0, content["ncpus"])) remove_config_option(train_command) @@ -1759,7 +1759,7 @@ def launch(service): _duplicate_adapt(service_module, content), files, priority, content["ngpus"], content["ncpus"], - other_task_info)) + deepcopy(other_task_info))) task_ids.append("%s\t%s\tngpus: %d, ncpus: %d" % ( task_type, task_id, content["ngpus"], content["ncpus"])) @@ -1812,7 +1812,7 @@ def launch(service): _duplicate_adapt(service_module, content_translate), (), content_translate["priority"], content_translate["ngpus"], content_translate["ncpus"], - other_task_info)) + deepcopy(other_task_info))) task_ids.append("%s\t%s\tngpus: %d, ncpus: %d" % ( "trans", trans_task_id, content_translate["ngpus"], content_translate["ncpus"])) @@ -1862,7 +1862,7 @@ def launch(service): content_score, files, priority + 2, 0, 1, - other_task_info)) + deepcopy(other_task_info))) task_ids.append("%s\t%s\tngpus: %d, ncpus: %d" % ( "score", score_task_id, 0, 1)) @@ -1917,7 +1917,7 @@ def launch(service): content_tuminer, (), priority + 2, ngpus_recommend, ncpus_recommend, - other_task_info)) + deepcopy(other_task_info))) task_ids.append("%s\t%s\tngpus: %d, ncpus: %d" % ( "tuminer", tuminer_task_id, ngpus_recommend, ncpus_recommend)) From 633e53ba266f94da38cea862288b09ac20d89df0 Mon Sep 17 00:00:00 2001 From: touffet Date: Tue, 11 May 2021 15:57:59 +0200 Subject: [PATCH 30/31] refs #56678: 1 gpu to train 0 otherwise --- client/launcher.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/launcher.py b/client/launcher.py index 63b05358..004993a8 100644 --- a/client/launcher.py +++ b/client/launcher.py @@ -467,10 +467,10 @@ def process_request(service_list, cmd, subcmd, is_json, args, auth=None): raise ValueError("ERROR: service '%s' not defined" % args.service) if args.gpus is None: - if "trans" in args.docker_command or "score" in args.docker_command: - args.gpus = 0 - else: + if "train" in args.docker_command: args.gpus = 1 + else: + args.gpus = 0 if args.gpus < 0: raise ValueError("ERROR: ngpus must be >= 0") From ef77c24d5ef08e201dc39dafe1be794476e1efbf Mon Sep 17 00:00:00 2001 From: Pierre-jean Liard Date: Tue, 11 May 2021 16:11:49 +0200 Subject: [PATCH 31/31] refs #56757: remove build object from parent config --- server/app/routes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/app/routes.py b/server/app/routes.py index a1c8ca69..245a98dc 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -1076,6 +1076,8 @@ def get_final_training_config(request_data, training_corpus_infos): ok, parent_config = builtins.pn9model_db.catalog_get_info(request_data["parent_model"], boolean_param(request.args.get('short'))) if ok: + # Remove build object from parent config + parent_config.pop('build', None) # Change batch size to settings value if specified if "config" in parent_config["options"] and "train" in parent_config["options"]["config"] \ and "batch_size" in parent_config["options"]["config"]["train"]: