Skip to content

Commit 76d6076

Browse files
yzbxyzbx
authored
Merge pull request #14 from modelai/ymir-dev
update for ymir2.0.0
2 parents c1db964 + 2ff6f57 commit 76d6076

14 files changed

Lines changed: 287 additions & 25 deletions

‎README.MD‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ docker build -t ymir-executor/mmdet:cu111-tmi -f docker/Dockerfile.cuda111 .
9393

9494
- [ymir-executor-sdk](https://github.com/modelai/ymir-executor-sdk) ymir-executor development SDK.
9595

96+
- [dataset convert](https://github.com/modelai/ymir-executor-sdk/blob/master/docs/dataset_convert.md)
97+
9698
- [ymir-executor-verifer](https://github.com/modelai/ymir-executor-verifier) debug and check your ymir-executor
9799

98100
## how to import pretrained model weights

‎README_zh-CN.MD‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,8 @@ docker build -t ymir-executor/live-code:mxnet-tmi -f mxnet.dockerfile
109109

110110
- [ymir-executor-sdk](https://github.com/modelai/ymir-executor-sdk) ymir镜像开发辅助库
111111

112+
- [数据集转换](https://github.com/modelai/ymir-executor-sdk/blob/master/docs/dataset_convert.md)
113+
112114
- [ymir-executor-verifer](https://github.com/modelai/ymir-executor-verifier) 调试与检测 ymir-executor
113115

114116
## 如何导入预训练模型

‎det-mmdetection-tmi/mining_base.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
from typing import List
33

44
import torch
5-
import torch.nn.functional as F
5+
import torch.nn.functional as F # noqa
66
from easydict import EasyDict as edict
77

88

‎det-mmdetection-tmi/start.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ def _run_mining(cfg: edict) -> None:
3939
gpu_count = len(gpu_id.split(','))
4040
mining_algorithm: str = cfg.param.get('mining_algorithm', 'aldd')
4141

42-
supported_mining_algorithm = ['cald', 'aldd', 'random']
43-
assert mining_algorithm in supported_mining_algorithm, f'unknown mining_algorithm {mining_algorithm}, not in {supported_mining_algorithm}'
42+
supported_miner = ['cald', 'aldd', 'random', 'entropy']
43+
assert mining_algorithm in supported_miner, f'unknown mining_algorithm {mining_algorithm}, not in {supported_miner}'
4444
if gpu_count <= 1:
4545
command = f'python3 ymir_mining_{mining_algorithm}.py'
4646
else:
@@ -67,7 +67,6 @@ def _run_infer() -> None:
6767

6868
cfg = get_merged_config()
6969
os.environ.setdefault('YMIR_MODELS_DIR', cfg.ymir.output.models_dir)
70-
os.environ.setdefault('COCO_EVAL_TMP_FILE', os.path.join(
71-
cfg.ymir.output.root_dir, 'eval_tmp.json'))
70+
os.environ.setdefault('COCO_EVAL_TMP_FILE', os.path.join(cfg.ymir.output.root_dir, 'eval_tmp.json'))
7271
os.environ.setdefault('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION', 'python')
7372
sys.exit(start(cfg))

‎det-mmdetection-tmi/ymir_infer.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from mmdet.utils.util_ymir import get_best_weight_file
1313
from tqdm import tqdm
1414
from ymir_exc import dataset_reader as dr
15-
from ymir_exc import env, monitor
15+
from ymir_exc import env
1616
from ymir_exc import result_writer as rw
1717
from ymir_exc.util import YmirStage, get_merged_config, write_ymir_monitor_process
1818

‎det-mmdetection-tmi/ymir_mining_cald.py‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,12 @@
1111
import numpy as np
1212
import torch
1313
import torch.distributed as dist
14-
from easydict import EasyDict as edict
1514
from mmcv.runner import init_dist
1615
from mmdet.apis.test import collect_results_gpu
1716
from mmdet.utils.util_ymir import BBOX, CV_IMAGE
1817
from nptyping import NDArray
1918
from scipy.stats import entropy
2019
from tqdm import tqdm
21-
from ymir_exc import monitor
2220
from ymir_exc import result_writer as rw
2321
from ymir_exc.util import YmirStage, get_merged_config, write_ymir_monitor_process
2422
from ymir_infer import YmirModel
@@ -250,11 +248,7 @@ def split_result(result: NDArray) -> Tuple[BBOX, NDArray, NDArray]:
250248
return bboxes, conf, class_id
251249

252250

253-
class YmirMining(YmirModel):
254-
255-
def __init__(self, cfg: edict):
256-
super().__init__(cfg)
257-
251+
class CALDMiner(YmirModel):
258252
def mining(self):
259253
with open(self.cfg.ymir.input.candidate_index_file, 'r') as f:
260254
images = [line.strip() for line in f.readlines()]
@@ -276,7 +270,7 @@ def mining(self):
276270
beta = 1.3
277271
mining_result = []
278272
for idx, asset_path in enumerate(tbar):
279-
if idx % monitor_gap == 0:
273+
if idx % monitor_gap == 0 and RANK in [0, -1]:
280274
write_ymir_monitor_process(self.cfg, task='mining', naive_stage_percent=idx / N, stage=YmirStage.TASK)
281275

282276
# batch-level sync, avoid 30min time-out error
@@ -380,7 +374,7 @@ def main():
380374
init_dist(launcher='pytorch', backend="nccl" if dist.is_nccl_available() else "gloo")
381375

382376
cfg = get_merged_config()
383-
miner = YmirMining(cfg)
377+
miner = CALDMiner(cfg)
384378
gpu = max(0, LOCAL_RANK)
385379
device = torch.device('cuda', gpu)
386380
miner.model.to(device)

‎det-mmdetection-tmi/ymir_mining_entropy.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
"""
2+
entropy mining
3+
"""
4+
import os
5+
import sys
6+
7+
import cv2
8+
import numpy as np
9+
import torch
10+
import torch.distributed as dist
11+
from mmcv.runner import init_dist
12+
from mmdet.apis.test import collect_results_gpu
13+
from tqdm import tqdm
14+
from ymir_exc import result_writer as rw
15+
from ymir_exc.util import YmirStage, get_merged_config, write_ymir_monitor_process
16+
from ymir_mining_cald import split_result, CALDMiner
17+
18+
LOCAL_RANK = int(os.getenv('LOCAL_RANK', -1)) # https://pytorch.org/docs/stable/elastic/run.html
19+
RANK = int(os.getenv('RANK', -1))
20+
WORLD_SIZE = int(os.getenv('WORLD_SIZE', 1))
21+
22+
23+
class EntropyMiner(CALDMiner):
24+
25+
def mining(self):
26+
with open(self.cfg.ymir.input.candidate_index_file, 'r') as f:
27+
images = [line.strip() for line in f.readlines()]
28+
29+
max_barrier_times = len(images) // WORLD_SIZE
30+
if RANK == -1:
31+
N = len(images)
32+
tbar = tqdm(images)
33+
else:
34+
images_rank = images[RANK::WORLD_SIZE]
35+
N = len(images_rank)
36+
if RANK == 0:
37+
tbar = tqdm(images_rank)
38+
else:
39+
tbar = images_rank
40+
41+
monitor_gap = max(1, N // 100)
42+
mining_result = []
43+
for idx, asset_path in enumerate(tbar):
44+
if idx % monitor_gap == 0 and RANK in [0, -1]:
45+
write_ymir_monitor_process(self.cfg, task='mining', naive_stage_percent=idx / N, stage=YmirStage.TASK)
46+
# batch-level sync, avoid 30min time-out error
47+
if WORLD_SIZE > 1 and idx < max_barrier_times:
48+
dist.barrier()
49+
50+
img = cv2.imread(asset_path)
51+
# xyxy,conf,cls
52+
result = self.predict(img)
53+
bboxes, conf, _ = split_result(result)
54+
if len(result) == 0:
55+
# no result for the image without augmentation
56+
mining_result.append((asset_path, -10))
57+
continue
58+
conf = conf.data.cpu().numpy()
59+
mining_result.append((asset_path, -np.sum(conf * np.log2(conf))))
60+
61+
if WORLD_SIZE > 1:
62+
mining_result = collect_results_gpu(mining_result, len(images))
63+
64+
return mining_result
65+
66+
67+
def main():
68+
if LOCAL_RANK != -1:
69+
init_dist(launcher='pytorch', backend="nccl" if dist.is_nccl_available() else "gloo")
70+
71+
cfg = get_merged_config()
72+
miner = EntropyMiner(cfg)
73+
gpu = max(0, LOCAL_RANK)
74+
device = torch.device('cuda', gpu)
75+
miner.model.to(device)
76+
mining_result = miner.mining()
77+
78+
if RANK in [0, -1]:
79+
rw.write_mining_result(mining_result=mining_result)
80+
81+
write_ymir_monitor_process(cfg, task='mining', naive_stage_percent=1, stage=YmirStage.POSTPROCESS)
82+
83+
return 0
84+
85+
86+
if __name__ == "__main__":
87+
sys.exit(main())

‎det-yolov5-tmi/ymir/docker/cuda102.dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ RUN apt-get update && apt-get install -y gnupg2 git libglib2.0-0 \
2121
&& rm -rf /var/lib/apt/lists/*
2222

2323
# install ymir-exc sdk
24-
RUN pip install "git+https://github.com/modelai/ymir-executor-sdk.git@ymir1.0.0"
24+
RUN pip install "git+https://github.com/modelai/ymir-executor-sdk.git@ymir1.3.0"
2525

2626
# Copy file from host to docker and install requirements
2727
COPY . /app

‎det-yolov5-tmi/ymir/docker/cuda111.dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ RUN apt-get update && apt-get install -y gnupg2 git libglib2.0-0 \
2424

2525
COPY ./requirements.txt /workspace/
2626
# install ymir-exc sdk and requirements
27-
RUN pip install "git+https://github.com/modelai/ymir-executor-sdk.git@ymir1.0.0" \
27+
RUN pip install "git+https://github.com/modelai/ymir-executor-sdk.git@ymir1.3.0" \
2828
&& pip install -r /workspace/requirements.txt
2929

3030
# Copy file from host to docker and install requirements

‎det-yolov5-tmi/ymir/mining/data_augment.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import cv2
99
import numpy as np
1010
from nptyping import NDArray
11+
1112
from ymir.ymir_yolov5 import BBOX, CV_IMAGE
1213

1314

0 commit comments

Comments
 (0)