Skip to content

Commit c1db964

Browse files
yzbxyzbx
authored
Merge pull request #12 from modelai/ymir-dev
update doc
2 parents 3f0847b + 4520d90 commit c1db964

17 files changed

Lines changed: 230 additions & 218 deletions

‎det-mmdetection-tmi/mmdet/core/evaluation/eval_hooks.py‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from mmdet.utils.util_ymir import write_ymir_training_result
1010
from torch.nn.modules.batchnorm import _BatchNorm
1111
from ymir_exc import monitor
12-
from ymir_exc.util import YmirStage, get_ymir_process
12+
from ymir_exc.util import YmirStage, get_merged_config, write_ymir_monitor_process
1313

1414

1515
def _calc_dynamic_intervals(start_interval, dynamic_interval_list):
@@ -28,6 +28,7 @@ class EvalHook(BaseEvalHook):
2828

2929
def __init__(self, *args, dynamic_intervals=None, **kwargs):
3030
super(EvalHook, self).__init__(*args, **kwargs)
31+
self.ymir_cfg = get_merged_config()
3132

3233
self.use_dynamic_intervals = dynamic_intervals is not None
3334
if self.use_dynamic_intervals:
@@ -51,9 +52,7 @@ def after_train_epoch(self, runner):
5152
if self.by_epoch:
5253
monitor_interval = max(1, runner.max_epochs // 1000)
5354
if runner.epoch % monitor_interval == 0:
54-
percent = get_ymir_process(
55-
stage=YmirStage.TASK, p=runner.epoch / runner.max_epochs)
56-
monitor.write_monitor_logger(percent=percent)
55+
write_ymir_monitor_process(self.ymir_cfg, task='training', naive_stage_percent=runner.epoch / runner.max_epochs, stage=YmirStage.TASK)
5756
super().after_train_epoch(runner)
5857

5958
def before_train_iter(self, runner):
@@ -64,9 +63,7 @@ def after_train_iter(self, runner):
6463
if not self.by_epoch:
6564
monitor_interval = max(1, runner.max_iters // 1000)
6665
if runner.iter % monitor_interval == 0:
67-
percent = get_ymir_process(
68-
stage=YmirStage.TASK, p=runner.iter / runner.max_iters)
69-
monitor.write_monitor_logger(percent=percent)
66+
write_ymir_monitor_process(self.ymir_cfg, task='training', naive_stage_percent=runner.ite / runner.max_iters, stage=YmirStage.TASK)
7067
super().after_train_iter(runner)
7168

7269
def _do_evaluate(self, runner):
@@ -98,6 +95,7 @@ class DistEvalHook(BaseDistEvalHook):
9895

9996
def __init__(self, *args, dynamic_intervals=None, **kwargs):
10097
super(DistEvalHook, self).__init__(*args, **kwargs)
98+
self.ymir_cfg = get_merged_config()
10199

102100
self.use_dynamic_intervals = dynamic_intervals is not None
103101
if self.use_dynamic_intervals:
@@ -121,9 +119,7 @@ def after_train_epoch(self, runner):
121119
if self.by_epoch and runner.rank == 0:
122120
monitor_interval = max(1, runner.max_epochs // 1000)
123121
if runner.epoch % monitor_interval == 0:
124-
percent = get_ymir_process(
125-
stage=YmirStage.TASK, p=runner.epoch / runner.max_epochs)
126-
monitor.write_monitor_logger(percent=percent)
122+
write_ymir_monitor_process(self.ymir_cfg, task='training', naive_stage_percent=runner.epoch / runner.max_epochs, stage=YmirStage.TASK)
127123
super().after_train_epoch(runner)
128124

129125
def before_train_iter(self, runner):
@@ -134,9 +130,7 @@ def after_train_iter(self, runner):
134130
if not self.by_epoch and runner.rank == 0:
135131
monitor_interval = max(1, runner.max_iters // 1000)
136132
if runner.iter % monitor_interval == 0:
137-
percent = get_ymir_process(
138-
stage=YmirStage.TASK, p=runner.iter / runner.max_iters)
139-
monitor.write_monitor_logger(percent=percent)
133+
write_ymir_monitor_process(self.ymir_cfg, task='training', naive_stage_percent=runner.iter / runner.max_iters, stage=YmirStage.TASK)
140134
super().after_train_iter(runner)
141135

142136
def _do_evaluate(self, runner):

‎det-mmdetection-tmi/ymir_infer.py‎

Lines changed: 3 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from ymir_exc import dataset_reader as dr
1515
from ymir_exc import env, monitor
1616
from ymir_exc import result_writer as rw
17-
from ymir_exc.util import YmirStage, get_merged_config, get_ymir_process
17+
from ymir_exc.util import YmirStage, get_merged_config, write_ymir_monitor_process
1818

1919

2020
def parse_option(cfg_options: str) -> dict:
@@ -73,18 +73,6 @@ class YmirModel:
7373
def __init__(self, cfg: edict):
7474
self.cfg = cfg
7575

76-
if cfg.ymir.run_mining and cfg.ymir.run_infer:
77-
# mining_task_idx = 0
78-
infer_task_idx = 1
79-
task_num = 2
80-
else:
81-
# mining_task_idx = 0
82-
infer_task_idx = 0
83-
task_num = 1
84-
85-
self.task_idx = infer_task_idx
86-
self.task_num = task_num
87-
8876
# Specify the path to model config and checkpoint file
8977
config_file = get_config_file(cfg)
9078
checkpoint_file = get_best_weight_file(cfg)
@@ -121,15 +109,10 @@ def main():
121109
idx += 1
122110

123111
if idx % monitor_gap == 0:
124-
percent = get_ymir_process(stage=YmirStage.TASK,
125-
p=idx / N,
126-
task_idx=model.task_idx,
127-
task_num=model.task_num)
128-
monitor.write_monitor_logger(percent=percent)
112+
write_ymir_monitor_process(cfg, task='infer', naive_stage_percent=idx / N, stage = YmirStage.TASK)
129113

130114
rw.write_infer_result(infer_result=infer_result)
131-
percent = get_ymir_process(stage=YmirStage.POSTPROCESS, p=1, task_idx=model.task_idx, task_num=model.task_num)
132-
monitor.write_monitor_logger(percent=percent)
115+
write_ymir_monitor_process(cfg, task='infer', naive_stage_percent=1.0, stage=YmirStage.POSTPROCESS)
133116
return 0
134117

135118

‎det-mmdetection-tmi/ymir_mining_cald.py‎

Lines changed: 4 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
from tqdm import tqdm
2121
from ymir_exc import monitor
2222
from ymir_exc import result_writer as rw
23-
from ymir_exc.util import YmirStage, get_merged_config, get_ymir_process
23+
from ymir_exc.util import YmirStage, get_merged_config, write_ymir_monitor_process
2424
from ymir_infer import YmirModel
2525

2626
LOCAL_RANK = int(os.getenv('LOCAL_RANK', -1)) # https://pytorch.org/docs/stable/elastic/run.html
@@ -254,16 +254,6 @@ class YmirMining(YmirModel):
254254

255255
def __init__(self, cfg: edict):
256256
super().__init__(cfg)
257-
if cfg.ymir.run_mining and cfg.ymir.run_infer:
258-
mining_task_idx = 0
259-
# infer_task_idx = 1
260-
task_num = 2
261-
else:
262-
mining_task_idx = 0
263-
# infer_task_idx = 0
264-
task_num = 1
265-
self.task_idx = mining_task_idx
266-
self.task_num = task_num
267257

268258
def mining(self):
269259
with open(self.cfg.ymir.input.candidate_index_file, 'r') as f:
@@ -287,11 +277,8 @@ def mining(self):
287277
mining_result = []
288278
for idx, asset_path in enumerate(tbar):
289279
if idx % monitor_gap == 0:
290-
percent = get_ymir_process(stage=YmirStage.TASK,
291-
p=idx / N,
292-
task_idx=self.task_idx,
293-
task_num=self.task_num)
294-
monitor.write_monitor_logger(percent=percent)
280+
write_ymir_monitor_process(self.cfg, task='mining', naive_stage_percent=idx / N, stage=YmirStage.TASK)
281+
295282
# batch-level sync, avoid 30min time-out error
296283
if WORLD_SIZE > 1 and idx < max_barrier_times:
297284
dist.barrier()
@@ -401,9 +388,7 @@ def main():
401388

402389
if RANK in [0, -1]:
403390
rw.write_mining_result(mining_result=mining_result)
404-
405-
percent = get_ymir_process(stage=YmirStage.POSTPROCESS, p=1, task_idx=miner.task_idx, task_num=miner.task_num)
406-
monitor.write_monitor_logger(percent=percent)
391+
write_ymir_monitor_process(cfg, task='mining', naive_stage_percent=1, stage=YmirStage.POSTPROCESS)
407392

408393
return 0
409394

‎det-mmdetection-tmi/ymir_train.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from easydict import EasyDict as edict
88
from mmdet.utils.util_ymir import get_best_weight_file, write_ymir_training_result
99
from ymir_exc import monitor
10-
from ymir_exc.util import YmirStage, find_free_port, get_merged_config, get_ymir_process
10+
from ymir_exc.util import YmirStage, find_free_port, get_merged_config, write_ymir_monitor_process
1111

1212

1313
def main(cfg: edict) -> int:
@@ -41,8 +41,7 @@ def main(cfg: edict) -> int:
4141
else:
4242
logging.warning('no weight file used for training!')
4343

44-
monitor.write_monitor_logger(
45-
percent=get_ymir_process(YmirStage.PREPROCESS, p=0.2))
44+
write_ymir_monitor_process(cfg, task='training', naive_stage_percent=0.2, stage=YmirStage.POSTPROCESS)
4645

4746
work_dir = cfg.ymir.output.models_dir
4847
if num_gpus == 0:

0 commit comments

Comments
 (0)