diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e62151..8e076b22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,25 @@ No changes to highlight. ## Other Changes: +No changes to highlight. + +# v1.4.1 + +## New Features: + +- Add yolov9 pretrained weights by `@illian01` in [PR 631](https://github.com/Nota-NetsPresso/netspresso-trainer/pull/631) +- Add EXIR exporting feature by `@illian01` in [PR 632](https://github.com/Nota-NetsPresso/netspresso-trainer/pull/632) + +## Bug Fixes: + +No changes to highlight. + +## Breaking Changes: + +No changes to highlight. + +## Other Changes: + Fix/add data params mlflow by `@hglee98` in [PR 629](https://github.com/Nota-NetsPresso/netspresso-trainer/pull/629) # 1.4.0 diff --git a/Dockerfile b/Dockerfile index 8c37353f..66d5dd5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.16 +FROM python:3.10 ARG TORCH_VERSION="2.0.1" ARG TORCHVISION_VERSION="0.15.2" diff --git a/README.md b/README.md index aa8c7c8c..20160d20 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ _____ ### Prerequisites -- Python `3.8` | `3.9` | `3.10` -- PyTorch `2.0.1` (recommended) (compatible with: `1.11.x` - `2.0.1`) +- Python `>=3.10` +- PyTorch `>=2.0.1` ### Install with pypi diff --git a/config/benchmark_examples/detection-coco2017-yolov9_tiny/augmentation.yaml b/config/benchmark_examples/detection-coco2017-yolov9_tiny/augmentation.yaml new file mode 100644 index 00000000..145ada0b --- /dev/null +++ b/config/benchmark_examples/detection-coco2017-yolov9_tiny/augmentation.yaml @@ -0,0 +1,56 @@ +augmentation: + train: + - + name: mosaicdetection + size: [640, 640] + mosaic_prob: 1.0 + affine_scale: [0.1, 1.9] + degrees: 0.0 + translate: 0.1 + shear: 0.0 + enable_mixup: True + mixup_prob: 0.15 + mixup_scale: [0.1, 2.0] + fill: 0 + mosaic_off_duration: 15 + - + name: hsvjitter + h_mag: 5 + s_mag: 30 + v_mag: 30 + - + name: randomhorizontalflip + p: 0.5 + - + name: resize + size: 640 + interpolation: bilinear + max_size: ~ + resize_criteria: long + - + name: pad + size: 640 + fill: 0 + - + name: randomresize + base_size: [640, 640] + stride: 32 + random_range: 5 + interpolation: bilinear + - + name: totensor + pixel_range: 1.0 + inference: + - + name: resize + size: 640 + interpolation: bilinear + max_size: ~ + resize_criteria: long + - + name: pad + size: 640 + fill: 0 + - + name: totensor + pixel_range: 1.0 diff --git a/config/benchmark_examples/detection-coco2017-yolov9_tiny/data.yaml b/config/benchmark_examples/detection-coco2017-yolov9_tiny/data.yaml new file mode 100644 index 00000000..86801f45 --- /dev/null +++ b/config/benchmark_examples/detection-coco2017-yolov9_tiny/data.yaml @@ -0,0 +1,21 @@ +data: + name: coco2017 + task: detection + format: local # local, huggingface + path: + root: ./data/coco2017 # dataset root + train: + image: images/train # directory for training images + label: labels/train # directory for training labels + valid: + image: images/valid # directory for valid images + label: labels/valid # directory for valid labels + test: + image: ~ + label: ~ + pattern: + image: ~ + label: ~ + id_mapping: id_mapping.json + # id_mapping: ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'] + pallete: ~ diff --git a/config/benchmark_examples/detection-coco2017-yolov9_tiny/environment.yaml b/config/benchmark_examples/detection-coco2017-yolov9_tiny/environment.yaml new file mode 100644 index 00000000..a5b5650d --- /dev/null +++ b/config/benchmark_examples/detection-coco2017-yolov9_tiny/environment.yaml @@ -0,0 +1,6 @@ +environment: + seed: 1 + num_workers: 4 + gpus: 0, 1, 2, 3, 4, 5, 6, 7 + batch_size: 16 # Batch size per gpu + cache_data: False \ No newline at end of file diff --git a/config/benchmark_examples/detection-coco2017-yolov9_tiny/logging.yaml b/config/benchmark_examples/detection-coco2017-yolov9_tiny/logging.yaml new file mode 100644 index 00000000..d352ee29 --- /dev/null +++ b/config/benchmark_examples/detection-coco2017-yolov9_tiny/logging.yaml @@ -0,0 +1,18 @@ +logging: + project_id: ~ + output_dir: ./outputs + tensorboard: true + mlflow: false + stdout: true + num_save_samples: 16 # num_save_samples should be >= 0 or None + model_save_options: + save_optimizer_state: true + save_best_only: false + best_model_criterion: loss # metric + sample_input_size: [640, 640] # Used for flops and onnx export + onnx_export_opset: 13 # Recommend in range [13, 17] + validation_epoch: &validation_epoch 10 + save_checkpoint_epoch: *validation_epoch # Multiplier of `validation_epoch`. + metrics: + classwise_analysis: False + metric_names: ~ # None for default settings \ No newline at end of file diff --git a/config/benchmark_examples/detection-coco2017-yolov9_tiny/model.yaml b/config/benchmark_examples/detection-coco2017-yolov9_tiny/model.yaml new file mode 100644 index 00000000..0030cb91 --- /dev/null +++ b/config/benchmark_examples/detection-coco2017-yolov9_tiny/model.yaml @@ -0,0 +1,72 @@ +model: + task: detection + name: yolov9_tiny + checkpoint: + use_pretrained: false + load_head: false + path: ~ + optimizer_path: ~ + freeze_backbone: false + architecture: + full: ~ # auto + backbone: + name: gelan + params: + stem_out_channels: 16 + stem_kernel_size: 3 + stem_stride: 2 + return_stage_idx: [1, 2, 3] + act_type: &act_type silu + stage_params: + # Conv2D: ['conv', out_channels, kernel_size, stride] + # ELAN: ['elan', out_channels, part_channels, use_identity] + # RepNCSPELAN: ['repncspelan', out_channels, part_channels, use_identity, depth] + # AConv: ['aconv', out_channels] + # ADown: ['adown', out_channels] + - + - ['conv', 32, 3, 2] + - ['elan', 32, 32, false] + - + - ['aconv', 64] + - ['repncspelan', 64, 64, false, 3] + - + - ['aconv', 96] + - ['repncspelan', 96, 96, false, 3] + - + - ['aconv', 128] + - ['repncspelan', 128, 128, false, 3] + neck: + name: yolov9fpn + params: + repeat_num: 3 + act_type: *act_type + use_aux_loss: &use_aux_loss false + bu_type: aconv + spp_channels: 128 + n4_channels: 96 + p3_channels: 64 + p3_to_p4_channels: 48 + p4_channels: 96 + p4_to_p5_channels: 64 + p5_channels: 128 + head: + name: yolo_detection_head + params: + version: v9 + num_anchors: ~ + use_group: true + reg_max: ®_max 16 + act_type: *act_type + use_aux_loss: *use_aux_loss + postprocessor: + params: + # postprocessor - decode + reg_max: *reg_max + score_thresh: 0.01 + # postprocessor - nms + nms_thresh: 0.65 + class_agnostic: false + losses: + - criterion: yolov9_loss + weight: ~ + l1_activate_epoch: ~ diff --git a/config/benchmark_examples/detection-coco2017-yolov9_tiny/training.yaml b/config/benchmark_examples/detection-coco2017-yolov9_tiny/training.yaml new file mode 100644 index 00000000..2721976c --- /dev/null +++ b/config/benchmark_examples/detection-coco2017-yolov9_tiny/training.yaml @@ -0,0 +1,23 @@ +training: + epochs: 500 + mixed_precision: True + max_norm: ~ + ema: + name: exp_decay + decay: 0.9999 + beta: 2000 + optimizer: + name: sgd + lr: 0.01 + momentum: 0.937 + weight_decay: 0.0005 # No bias and norm decay + nesterov: True + no_bias_decay: True + no_norm_weight_decay: True + overwrite: ~ + scheduler: + name: cosine_no_sgdr + warmup_epochs: 3 + warmup_bias_lr: 0.001 + min_lr: 0.0001 + end_epoch: 485 diff --git a/config/logging.yaml b/config/logging.yaml index a49bb2ca..ce2be760 100644 --- a/config/logging.yaml +++ b/config/logging.yaml @@ -15,4 +15,5 @@ logging: save_checkpoint_epoch: *validation_epoch # Multiplier of `validation_epoch`. metrics: classwise_analysis: False - metric_names: ~ # None for default settings \ No newline at end of file + metric_names: ~ # None for default settings + class_groups: ~ # Example: {vehicle: [car, truck, bus], person: [person, rider]} \ No newline at end of file diff --git a/config/model/mobilenetv4/mobilenetv4-conv-medium-classification.yaml b/config/model/mobilenetv4/mobilenetv4-conv-medium-classification.yaml index 5301ebe9..6a896c19 100644 --- a/config/model/mobilenetv4/mobilenetv4-conv-medium-classification.yaml +++ b/config/model/mobilenetv4/mobilenetv4-conv-medium-classification.yaml @@ -21,7 +21,7 @@ model: norm_type: batch_norm act_type: relu return_stage_idx: ~ - layer_scale: 0.1 + layer_scale: ~ stage_params: # Conv2D: ['conv', out_channels, kernel_size, stride] # FusedIB: ['fi', out_channels, hidden_channels, kernel_size, stride] diff --git a/config/model/mobilenetv4/mobilenetv4-conv-small-classification.yaml b/config/model/mobilenetv4/mobilenetv4-conv-small-classification.yaml index a71d02a3..5114fd33 100644 --- a/config/model/mobilenetv4/mobilenetv4-conv-small-classification.yaml +++ b/config/model/mobilenetv4/mobilenetv4-conv-small-classification.yaml @@ -21,7 +21,7 @@ model: norm_type: batch_norm act_type: relu return_stage_idx: ~ - layer_scale: 0.1 + layer_scale: ~ stage_params: # Conv2D: ['conv', out_channels, kernel_size, stride] # FusedIB: ['fi', out_channels, hidden_channels, kernel_size, stride] diff --git a/config/model/mobilenetv4/mobilenetv4-hybrid-large-classification.yaml b/config/model/mobilenetv4/mobilenetv4-hybrid-large-classification.yaml index c5736965..13d2a0aa 100644 --- a/config/model/mobilenetv4/mobilenetv4-hybrid-large-classification.yaml +++ b/config/model/mobilenetv4/mobilenetv4-hybrid-large-classification.yaml @@ -21,7 +21,7 @@ model: norm_type: batch_norm act_type: gelu return_stage_idx: ~ - layer_scale: ~ + layer_scale: 0.1 stage_params: # Conv2D: ['conv', out_channels, kernel_size, stride] # FusedIB: ['fi', out_channels, hidden_channels, kernel_size, stride] diff --git a/config/model/yolo/yolov9_c-detection.yaml b/config/model/yolov9/yolov9_c-detection.yaml similarity index 98% rename from config/model/yolo/yolov9_c-detection.yaml rename to config/model/yolov9/yolov9_c-detection.yaml index 56893e9f..50cfb0dd 100644 --- a/config/model/yolo/yolov9_c-detection.yaml +++ b/config/model/yolov9/yolov9_c-detection.yaml @@ -2,7 +2,7 @@ model: task: detection name: yolov9_c checkpoint: - use_pretrained: false + use_pretrained: true load_head: false path: ~ optimizer_path: ~ diff --git a/config/model/yolo/yolov9_m-detection.yaml b/config/model/yolov9/yolov9_m-detection.yaml similarity index 98% rename from config/model/yolo/yolov9_m-detection.yaml rename to config/model/yolov9/yolov9_m-detection.yaml index dee08ea8..d8fa0088 100644 --- a/config/model/yolo/yolov9_m-detection.yaml +++ b/config/model/yolov9/yolov9_m-detection.yaml @@ -2,7 +2,7 @@ model: task: detection name: yolov9_m checkpoint: - use_pretrained: false + use_pretrained: true load_head: false path: ~ optimizer_path: ~ diff --git a/config/model/yolo/yolov9_s-detection.yaml b/config/model/yolov9/yolov9_s-detection.yaml similarity index 98% rename from config/model/yolo/yolov9_s-detection.yaml rename to config/model/yolov9/yolov9_s-detection.yaml index b041c1c6..6beafd85 100644 --- a/config/model/yolo/yolov9_s-detection.yaml +++ b/config/model/yolov9/yolov9_s-detection.yaml @@ -2,7 +2,7 @@ model: task: detection name: yolov9_s checkpoint: - use_pretrained: false + use_pretrained: true load_head: false path: ~ optimizer_path: ~ diff --git a/config/model/yolov9/yolov9_tiny-detection.yaml b/config/model/yolov9/yolov9_tiny-detection.yaml new file mode 100644 index 00000000..65bbd170 --- /dev/null +++ b/config/model/yolov9/yolov9_tiny-detection.yaml @@ -0,0 +1,72 @@ +model: + task: detection + name: yolov9_tiny + checkpoint: + use_pretrained: true + load_head: false + path: ~ + optimizer_path: ~ + freeze_backbone: false + architecture: + full: ~ # auto + backbone: + name: gelan + params: + stem_out_channels: 16 + stem_kernel_size: 3 + stem_stride: 2 + return_stage_idx: [1, 2, 3] + act_type: &act_type silu + stage_params: + # Conv2D: ['conv', out_channels, kernel_size, stride] + # ELAN: ['elan', out_channels, part_channels, use_identity] + # RepNCSPELAN: ['repncspelan', out_channels, part_channels, use_identity, depth] + # AConv: ['aconv', out_channels] + # ADown: ['adown', out_channels] + - + - ['conv', 32, 3, 2] + - ['elan', 32, 32, false] + - + - ['aconv', 64] + - ['repncspelan', 64, 64, false, 3] + - + - ['aconv', 96] + - ['repncspelan', 96, 96, false, 3] + - + - ['aconv', 128] + - ['repncspelan', 128, 128, false, 3] + neck: + name: yolov9fpn + params: + repeat_num: 3 + act_type: *act_type + use_aux_loss: &use_aux_loss false + bu_type: aconv + spp_channels: 128 + n4_channels: 96 + p3_channels: 64 + p3_to_p4_channels: 48 + p4_channels: 96 + p4_to_p5_channels: 64 + p5_channels: 128 + head: + name: yolo_detection_head + params: + version: v9 + num_anchors: ~ + use_group: true + reg_max: ®_max 16 + act_type: *act_type + use_aux_loss: *use_aux_loss + postprocessor: + params: + # postprocessor - decode + reg_max: *reg_max + score_thresh: 0.01 + # postprocessor - nms + nms_thresh: 0.65 + class_agnostic: false + losses: + - criterion: yolov9_loss + weight: ~ + l1_activate_epoch: ~ diff --git a/docs/benchmarks/benchmarks.md b/docs/benchmarks/benchmarks.md index 0fec534b..e3f7e0f6 100644 --- a/docs/benchmarks/benchmarks.md +++ b/docs/benchmarks/benchmarks.md @@ -40,6 +40,10 @@ If you have a better recipe, please share with us anytime. We appreciate all eff |---|---|---|---|---|---|---|---|---|---|---| | COCO-val | [RT-DETR_res18*](https://github.com/Nota-NetsPresso/netspresso-trainer/blob/master/config/model/rtdetr/rtdetr-res18-detection.yaml) | [download](https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/rtdetr/rtdetr_res18_coco.safetensors?versionId=uu9v49NI6rQx8wOY6bJbEXUFOG_R9xqH) | (640, 640) | 65.77 | 52.75 | 48.49 | 20.18M | 40.36G | Supported | No input z-norm, [lyuwenyu/RT-DETR](https://github.com/lyuwenyu/RT-DETR/tree/main/rtdetr_pytorch) | | COCO-val | [RT-DETR_res50*](https://github.com/Nota-NetsPresso/netspresso-trainer/blob/master/config/model/rtdetr/rtdetr-res50-detection.yaml) | [download](https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/rtdetr/rtdetr_res50_coco.safetensors?versionId=JHmnjY13BEflpnDCYPFJ1c17UwpqDrLQ) | (640, 640) | 72.64 | 59.50 | 54.73 | 42.94M | 138.36G | Supported | No input z-norm, [lyuwenyu/RT-DETR](https://github.com/lyuwenyu/RT-DETR/tree/main/rtdetr_pytorch) | +| COCO-val | [yolov9-tiny](https://github.com/Nota-NetsPresso/netspresso-trainer/blob/master/config/model/yolov9/yolov9_tiny-detection.yaml) | [download](https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolov9/yolov9_tiny_coco.safetensors) | (640, 640) | 50.03 | 38.63 | 36.02 | 2.44M | 9.99G | Supported | No input z-norm | +| COCO-val | [yolov9-s*](https://github.com/Nota-NetsPresso/netspresso-trainer/blob/master/config/model/yolov9/yolov9_s-detection.yaml) | [download](https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolov9/yolov9_s_coco.safetensors) | (640, 640) | 62.63 | 51.13 | 47.13 | 7.23M | 26.87G | Supported | No input z-norm, [YOLO](https://yolo-docs.readthedocs.io/en/latest/2_model_zoo/0_object_detection.html) | +| COCO-val | [yolov9-m*](https://github.com/Nota-NetsPresso/netspresso-trainer/blob/master/config/model/yolov9/yolov9_m-detection.yaml) | [download](https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolov9/yolov9_m_coco.safetensors) | (640, 640) | 67.43 | 56.13 | 51.72 | 20.12M | 77.08G | Supported | No input z-norm, [YOLO](https://yolo-docs.readthedocs.io/en/latest/2_model_zoo/0_object_detection.html) | +| COCO-val | [yolov9-c*](https://github.com/Nota-NetsPresso/netspresso-trainer/blob/master/config/model/yolov9/yolov9_c-detection.yaml) | [download](https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolov9/yolov9_c_coco.safetensors) | (640, 640) | 69.16 | 57.90 | 53.28 | 25.50M | 103.17G | Supported | No input z-norm, [YOLO](https://yolo-docs.readthedocs.io/en/latest/2_model_zoo/0_object_detection.html) | | COCO-val | [YOLOX-nano*](https://github.com/Nota-NetsPresso/netspresso-trainer/blob/master/config/model/yolox/yolox-nano-detection.yaml) | [download](https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolox/yolox_nano_coco.safetensors?versionId=JCXugDTwGegx9Kl6Jc5AMJpIkA.WlNVP) | (416, 416) | 41.30 | 27.90 | 26.33 | 0.91M | 1.08G | Supported | [Megvii-BaseDetection/YOLOX](https://github.com/Megvii-BaseDetection/YOLOX?tab=readme-ov-file#benchmark), conf_thresh=0.01, nms_thresh=0.65 | | COCO-val | [YOLOX-tiny*](https://github.com/Nota-NetsPresso/netspresso-trainer/blob/master/config/model/yolox/yolox-tiny-detection.yaml) | [download](https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolox/yolox_tiny_coco.safetensors?versionId=lJp1bCEToD_6IaL9kRCqcYIwVZ.QQ.1P) | (416, 416) | 50.69 | 36.18 | 34.00 | 5.06M | 6.45G | Supported | [Megvii-BaseDetection/YOLOX](https://github.com/Megvii-BaseDetection/YOLOX?tab=readme-ov-file#benchmark), conf_thresh=0.01, nms_thresh=0.65 | | COCO-val | [YOLOX-s](https://github.com/Nota-NetsPresso/netspresso-trainer/blob/master/config/model/yolox/yolox-s-detection.yaml) | [download](https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolox/yolox_s_coco.safetensors?versionId=QRLqHKqhv8TSYBrmsQ3M8lCR8w7HEZyA) | (640, 640) | 58.56 | 44.10 | 40.63 | 8.97M | 26.81G | Supported | conf_thresh=0.01, nms_thresh=0.65 | diff --git a/docs/getting_started/installation/installation.md b/docs/getting_started/installation/installation.md index b8c42fdf..e0b88d05 100644 --- a/docs/getting_started/installation/installation.md +++ b/docs/getting_started/installation/installation.md @@ -2,8 +2,8 @@ ### Prerequisites -- Python `3.8` | `3.9` | `3.10` -- PyTorch `2.0.1` (recommended) (compatible with: `1.11.x` - `2.0.1`) +- Python `>=3.10` +- PyTorch `>=2.0.1` ### Install with pypi diff --git a/requirements.txt b/requirements.txt index 5ffe68ff..e00673bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -torch>=1.11.0,<=2.0.1 -torchvision>=0.12.0,<=0.15.2 +torch>=2.0.1 +torchvision onnx onnxruntime numpy diff --git a/src/netspresso_trainer/VERSION b/src/netspresso_trainer/VERSION index e21e727f..13175fdc 100644 --- a/src/netspresso_trainer/VERSION +++ b/src/netspresso_trainer/VERSION @@ -1 +1 @@ -1.4.0 \ No newline at end of file +1.4.1 \ No newline at end of file diff --git a/src/netspresso_trainer/loggers/base.py b/src/netspresso_trainer/loggers/base.py index 9398ddd9..5f14695d 100644 --- a/src/netspresso_trainer/loggers/base.py +++ b/src/netspresso_trainer/loggers/base.py @@ -126,6 +126,7 @@ def _convert_images_as_readable(self, samples: Union[Dict, List]): images = [magic_image_handler(image) for image in images] sample_readable = {} + sample_readable['name'] = [Path(n).stem for n in samples['name'][:self.num_sample_images]] sample_readable['images'] = images # TODO: pred and target can be more complex data structure later. sample_readable['pred'] = samples['pred'][:self.num_sample_images] diff --git a/src/netspresso_trainer/loggers/image.py b/src/netspresso_trainer/loggers/image.py index 1c42f789..3aef639c 100644 --- a/src/netspresso_trainer/loggers/image.py +++ b/src/netspresso_trainer/loggers/image.py @@ -43,15 +43,19 @@ def save_result(self, image_dict: Dict, prefix, epoch): prefix_dir: Path = self.save_dir / prefix prefix_dir.mkdir(exist_ok=True) + names = image_dict.get('name', []) for k, v_list in image_dict.items(): + if k == 'name': + continue for idx, v in enumerate(v_list): assert isinstance(v, np.ndarray) + stem = names[idx] if idx < len(names) else f"{idx:03d}" if epoch is None: - self.save_ndarray_as_image(v, f"{prefix_dir}/{idx:03d}_{k}.png", dataformats='HWC') + self.save_ndarray_as_image(v, f"{prefix_dir}/{stem}_{k}.png", dataformats='HWC') elif self.save_best_only: - self.save_ndarray_as_image(v, f"{prefix_dir}/best_{idx:03d}_{k}.png", dataformats='HWC') + self.save_ndarray_as_image(v, f"{prefix_dir}/best_{stem}_{k}.png", dataformats='HWC') else: - self.save_ndarray_as_image(v, f"{prefix_dir}/{epoch:04d}_{idx:03d}_{k}.png", dataformats='HWC') + self.save_ndarray_as_image(v, f"{prefix_dir}/{epoch:04d}_{stem}_{k}.png", dataformats='HWC') def __call__( self, diff --git a/src/netspresso_trainer/loggers/stdout.py b/src/netspresso_trainer/loggers/stdout.py index 8586573a..a9c24c40 100644 --- a/src/netspresso_trainer/loggers/stdout.py +++ b/src/netspresso_trainer/loggers/stdout.py @@ -65,12 +65,73 @@ def __call__( else: rows += [class_info.split('_', 1) for class_info in list(metrics[headers[-1]]['classwise'].keys())] rows += [['-', 'All', ]] if not data_stats else [['-', 'All', data_stats['total_instances']]] + all_row_idx = len(rows) - 1 + + has_weighted = any('weighted_mean' in v for v in metrics.values()) + if has_weighted: + rows += [['-', 'All (weighted)', data_stats['total_instances']]] if data_stats else [['-', 'All (weighted)']] + weighted_row_idx = len(rows) - 1 for _metric_name, score_dict in metrics.items(): if 'classwise' in score_dict: # If classwise analysis is activated for cls_num, item in enumerate(score_dict['classwise']): rows[cls_num].append(score_dict['classwise'][item]) - rows[-1].append(score_dict['mean']) + rows[all_row_idx].append(score_dict['mean']) + if has_weighted: + rows[weighted_row_idx].append(score_dict.get('weighted_mean', score_dict['mean'])) metric_std_log += tabulate(rows, headers=headers, tablefmt='grid', numalign='left', stralign='left') - logger.info(metric_std_log) # tabulaate is already contained as pandas dependency + + # Group-aware tables (one per group config, intra-group confusion = TP) + has_group = any('group_results' in v for v in metrics.values()) + if has_group: + # Determine number of group configs from the first metric that has group_results + first_with_gr = next(v for v in metrics.values() if 'group_results' in v) + num_configs = len(first_with_gr['group_results']) + instances_per_group_list = data_stats.get('instances_per_group_list') if data_stats else None + + for ci in range(num_configs): + config_name = first_with_gr['group_results'][ci].get('config_name', str(ci)) + first_gc = first_with_gr['group_results'][ci].get('group_classwise', {}) + instances_per_group = instances_per_group_list[ci] if instances_per_group_list and ci < len(instances_per_group_list) else None + + if instances_per_group is not None: + group_headers = ['Group ID', 'Group name', '# of Instances', *list(metrics.keys())] + group_rows = [ + grp_info.split('_', 1) + [instances_per_group.get(int(grp_info.split('_', 1)[0]), '-')] + for grp_info in list(first_gc.keys()) + ] + group_rows += [['-', 'All (group)', sum(instances_per_group.values())]] + else: + group_headers = ['Group ID', 'Group name', *list(metrics.keys())] + group_rows = [grp_info.split('_', 1) for grp_info in list(first_gc.keys())] + group_rows += [['-', 'All (group)']] + + grp_all_idx = len(group_rows) - 1 + has_group_weighted = any( + 'group_results' in v and ci < len(v['group_results']) and 'group_weighted_mean' in v['group_results'][ci] + for v in metrics.values() + ) + if has_group_weighted: + grp_wt_row = ['-', 'All (group/weighted)', sum(instances_per_group.values())] if instances_per_group is not None else ['-', 'All (group/weighted)'] + group_rows += [grp_wt_row] + grp_wt_idx = len(group_rows) - 1 + + for _metric_name, score_dict in metrics.items(): + if 'group_results' in score_dict and ci < len(score_dict['group_results']): + gr = score_dict['group_results'][ci] + gc_data = gr.get('group_classwise', {}) + for grp_num, item in enumerate(gc_data): + group_rows[grp_num].append(gc_data[item]) + group_rows[grp_all_idx].append(gr.get('group_mean', '-')) + if has_group_weighted: + group_rows[grp_wt_idx].append(gr.get('group_weighted_mean', '-')) + else: + group_rows[grp_all_idx].append('-') + if has_group_weighted: + group_rows[grp_wt_idx].append('-') + + metric_std_log += f'\nGroup-aware metric [{config_name}] (intra-group confusion = TP):\n' + metric_std_log += tabulate(group_rows, headers=group_headers, tablefmt='grid', numalign='left', stralign='left') + + logger.info(metric_std_log) # tabulate is already contained as pandas dependency diff --git a/src/netspresso_trainer/metrics/base.py b/src/netspresso_trainer/metrics/base.py index 2dc3e37b..24bbb5fa 100644 --- a/src/netspresso_trainer/metrics/base.py +++ b/src/netspresso_trainer/metrics/base.py @@ -14,7 +14,7 @@ # # ---------------------------------------------------------------------------- -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import torch @@ -22,46 +22,83 @@ class BaseMetric: - def __init__(self, metric_name, num_classes, classwise_analysis, **kwargs): + def __init__(self, metric_name, num_classes, classwise_analysis, + group_configs=None, **kwargs): self.metric_name = metric_name self.num_classes = num_classes self.classwise_analysis = classwise_analysis + self.group_configs = group_configs or [] if self.classwise_analysis: self.classwise_metric_meters = [MetricMeter(f'{metric_name}_{i}', ':6.2f') for i in range(num_classes)] self.metric_meter = MetricMeter(metric_name, ':6.2f') + self.weighted_metric_meter = None # Optionally set by subclasses + # Per-group-config meter sets — set by subclasses when group_configs is provided + self.group_classwise_meter_sets = None # List[List[MetricMeter]] + self.group_metric_meters = None # List[MetricMeter] + self.group_weighted_metric_meters = None # List[MetricMeter] def calibrate(self, pred, target, **kwargs): raise NotImplementedError class MetricFactory: - def __init__(self, task, metrics, metric_adaptor, classwise_analysis) -> None: + def __init__(self, task, metrics, metric_adaptor, classwise_analysis, + group_configs=None) -> None: self.task = task self.metrics = metrics self.metric_adaptor = metric_adaptor self.classwise_analysis = classwise_analysis + self.group_configs = group_configs or [] # List of group config dicts def reset_values(self): for phase in self.metrics: - [metric.metric_meter.reset() for metric in self.metrics[phase]] + for metric in self.metrics[phase]: + metric.metric_meter.reset() + if metric.weighted_metric_meter is not None: + metric.weighted_metric_meter.reset() + if metric.classwise_analysis: + for meter in metric.classwise_metric_meters: + meter.reset() + if metric.group_metric_meters is not None: + for ci in range(len(metric.group_metric_meters)): + metric.group_metric_meters[ci].reset() + metric.group_weighted_metric_meters[ci].reset() + for meter in metric.group_classwise_meter_sets[ci]: + meter.reset() def update(self, pred: torch.Tensor, target: torch.Tensor, phase: str, **kwargs: Any) -> None: - if len(pred) == 0: # Removed dummy batch has 0 len + if len(pred) == 0: # Removed dummy batch has 0 len return - kwargs.update(self.metric_adaptor(pred, target)) + adaptor_kwargs = {} + if self.group_configs: + adaptor_kwargs['group_map_arrays'] = [gc['group_map_array'] for gc in self.group_configs] + kwargs.update(self.metric_adaptor(pred, target, **adaptor_kwargs)) for metric in self.metrics[phase.lower()]: metric.calibrate(pred, target, **kwargs) def result(self, phase='train'): - ret = {metric.metric_name: {} for metric in self.metrics[phase]} # Initialize with empty dict + ret = {metric.metric_name: {} for metric in self.metrics[phase]} - if phase == 'valid' and self.classwise_analysis: # Add classwise results only for valid phase + if self.classwise_analysis: for metric in self.metrics[phase]: - classwise_result_dict = {i:classwise_meter.avg for i, classwise_meter in enumerate(metric.classwise_metric_meters)} - ret[metric.metric_name] = {'classwise': classwise_result_dict} + if metric.classwise_analysis: + classwise_result_dict = {i: m.avg for i, m in enumerate(metric.classwise_metric_meters)} + ret[metric.metric_name]['classwise'] = classwise_result_dict for metric in self.metrics[phase]: - ret[metric.metric_name]['mean'] = metric.metric_meter.avg # Add mean score + ret[metric.metric_name]['mean'] = metric.metric_meter.avg + if metric.weighted_metric_meter is not None: + ret[metric.metric_name]['weighted_mean'] = metric.weighted_metric_meter.avg + if metric.group_metric_meters is not None: + group_results = [] + for ci, gc in enumerate(self.group_configs): + group_results.append({ + 'config_name': gc['name'], + 'group_classwise': {i: m.avg for i, m in enumerate(metric.group_classwise_meter_sets[ci])}, + 'group_mean': metric.group_metric_meters[ci].avg, + 'group_weighted_mean': metric.group_weighted_metric_meters[ci].avg, + }) + ret[metric.metric_name]['group_results'] = group_results return ret diff --git a/src/netspresso_trainer/metrics/builder.py b/src/netspresso_trainer/metrics/builder.py index f7e81e3e..6fc45563 100644 --- a/src/netspresso_trainer/metrics/builder.py +++ b/src/netspresso_trainer/metrics/builder.py @@ -14,15 +14,92 @@ # # ---------------------------------------------------------------------------- -from typing import Any, Dict +from typing import Any, Dict, List, Optional + +import numpy as np from .base import MetricFactory from .registry import METRIC_ADAPTORS, METRIC_LIST, PHASE_LIST, TASK_AVAILABLE_METRICS, TASK_DEFAULT_METRICS -def build_metrics(task: str, model_conf, metrics_conf, num_classes, **kwargs) -> MetricFactory: +def _build_single_group_config(config_name: str, groups_conf, class_map: Dict[int, str], num_classes: int) -> dict: + """Build one group config dict from a {group_name: [class_name, ...]} mapping.""" + name_to_id = {name: id_ for id_, name in class_map.items()} + + group_map: Dict[int, int] = {} + group_names: Dict[int, str] = {} + + group_id = 0 + for group_name, class_names in groups_conf.items(): + group_names[group_id] = str(group_name) + for class_name in class_names: + cid = name_to_id.get(str(class_name)) + if cid is not None: + group_map[cid] = group_id + group_id += 1 + + # Ungrouped classes each get a solo group + for cid in sorted(class_map.keys()): + if cid not in group_map: + group_names[group_id] = class_map[cid] + group_map[cid] = group_id + group_id += 1 + + num_groups = group_id + group_map_array = np.array([group_map.get(i, i) for i in range(num_classes)], dtype=np.int64) + return { + 'name': config_name, + 'group_map': group_map, + 'group_names': group_names, + 'num_groups': num_groups, + 'group_map_array': group_map_array, + } + + +def _parse_class_groups(class_groups_conf, class_map: Dict[int, str], num_classes: int) -> List[dict]: + """Parse class_groups config into a list of group config dicts. + + Supports two YAML formats: + Single config (flat dict): + class_groups: + vehicle: [car, truck] + person: [person, rider] + + Multiple configs (nested dict): + class_groups: + by_type: + vehicle: [car, truck] + by_size: + large: [bus, truck] + + Returns: + List of group config dicts, each with keys: + name, group_map, group_names, num_groups, group_map_array + """ + if class_groups_conf is None or not class_map: + return [] + + conf_dict = dict(class_groups_conf) + if not conf_dict: + return [] + + # Detect format: if any value is a list → single config, if any value is a dict → multi config + first_val = next(iter(conf_dict.values())) + is_single = not hasattr(first_val, 'items') # list or ListConfig → single + + if is_single: + return [_build_single_group_config('default', class_groups_conf, class_map, num_classes)] + else: + return [ + _build_single_group_config(str(config_name), config_groups, class_map, num_classes) + for config_name, config_groups in class_groups_conf.items() + ] + + +def build_metrics(task: str, model_conf, metrics_conf, num_classes, class_map: Optional[Dict] = None, **kwargs) -> MetricFactory: metric_names = metrics_conf.metric_names classwise_analysis = metrics_conf.classwise_analysis + class_groups_conf = getattr(metrics_conf, 'class_groups', None) if metric_names is None: metric_names = TASK_DEFAULT_METRICS[task] @@ -34,14 +111,21 @@ def build_metrics(task: str, model_conf, metrics_conf, num_classes, **kwargs) -> if hasattr(model_conf.losses[0], 'ignore_index'): kwargs['ignore_index'] = model_conf.losses[0].ignore_index + # Parse class groups — returns a list of group config dicts (empty if not configured) + group_configs = _parse_class_groups(class_groups_conf, class_map or {}, num_classes) + metrics = {} for phase in PHASE_LIST: - if phase == 'valid': # classwise_analysis is only available in valid phase - metrics[phase] = [METRIC_LIST[name](num_classes=num_classes, classwise_analysis=classwise_analysis, **kwargs) for name in metric_names] - else: - metrics[phase] = [METRIC_LIST[name](num_classes=num_classes, classwise_analysis=False, **kwargs) for name in metric_names] + metrics[phase] = [ + METRIC_LIST[name]( + num_classes=num_classes, + classwise_analysis=classwise_analysis, + group_configs=group_configs, + **kwargs, + ) + for name in metric_names + ] metric_adaptor = METRIC_ADAPTORS[task](metric_names) - - metric_handler = MetricFactory(task, metrics, metric_adaptor, classwise_analysis) + metric_handler = MetricFactory(task, metrics, metric_adaptor, classwise_analysis, group_configs=group_configs) return metric_handler diff --git a/src/netspresso_trainer/metrics/detection/metric.py b/src/netspresso_trainer/metrics/detection/metric.py index 296d96d8..538c0922 100644 --- a/src/netspresso_trainer/metrics/detection/metric.py +++ b/src/netspresso_trainer/metrics/detection/metric.py @@ -26,6 +26,7 @@ import numpy as np from ..base import BaseMetric +from ...utils.record import MetricMeter def box_iou_batch(boxes_true: np.ndarray, boxes_detection: np.ndarray) -> np.ndarray: @@ -260,6 +261,31 @@ def recall_per_class( return recalls +def _collect_image_stats(predicted_objs, true_objs, iou_thresholds): + """Collect match stats for a single image. Returns a tuple or None.""" + if predicted_objs.shape[0] == 0 and true_objs.shape[0] == 0: + return None + elif predicted_objs.shape[0] == 0: + # GT exists, no predictions: all GT are FN + return ( + np.zeros((0, iou_thresholds.size), dtype=bool), + np.zeros(0), + np.zeros(0), + true_objs[:, 4], + ) + elif true_objs.shape[0] == 0: + # Predictions exist, no GT: all predictions are FP + return ( + np.zeros((predicted_objs.shape[0], iou_thresholds.size), dtype=bool), + predicted_objs[:, 5], + predicted_objs[:, 4], + np.zeros(0), + ) + else: + matches = match_detection_batch(predicted_objs, true_objs, iou_thresholds) + return (matches, predicted_objs[:, 5], predicted_objs[:, 4], true_objs[:, 4]) + + class DetectionMetricAdaptor: ''' Adapter to process redundant operations for the metrics. @@ -267,11 +293,11 @@ class DetectionMetricAdaptor: def __init__(self, metric_names) -> None: self.metric_names = metric_names - def __call__(self, predictions: List[dict], targets: List[dict]): + def __call__(self, predictions: List[dict], targets: List[dict], group_map_arrays=None): iou_thresholds = np.linspace(0.5, 0.95, 10) stats = [] + group_stats_list = [[] for _ in (group_map_arrays or [])] - # Gather matching stats for predictions and targets for pred, target in zip(predictions, targets): predicted_objs_bbox, predicted_objs_class, predicted_objs_confidence = pred['boxes'], pred['labels'], pred['scores'] true_objs_bbox, true_objs_class = target['boxes'], target['labels'] @@ -279,38 +305,104 @@ def __call__(self, predictions: List[dict], targets: List[dict]): true_objs = np.concatenate((true_objs_bbox, true_objs_class[..., np.newaxis]), axis=-1) predicted_objs = np.concatenate((predicted_objs_bbox, predicted_objs_class[..., np.newaxis], predicted_objs_confidence[..., np.newaxis]), axis=-1) - if predicted_objs.shape[0] == 0 and true_objs.shape[0]: - stats.append( - ( - np.zeros((0, iou_thresholds.size), dtype=bool), - *np.zeros((2, 0)), - true_objs[:, 4], - ) - ) - - if true_objs.shape[0]: - matches = match_detection_batch(predicted_objs, true_objs, iou_thresholds) - stats.append( - ( - matches, - predicted_objs[:, 5], - predicted_objs[:, 4], - true_objs[:, 4], - ) - ) - - return {'stats': stats} + entry = _collect_image_stats(predicted_objs, true_objs, iou_thresholds) + if entry is not None: + stats.append(entry) + + # Group-aware stats: one set per group config + for ci, gma in enumerate(group_map_arrays or []): + g_true_class = gma[true_objs[:, 4].astype(int)] if true_objs.shape[0] > 0 else true_objs[:, 4] + g_true_objs = np.column_stack([true_objs[:, :4], g_true_class]) if true_objs.shape[0] > 0 else true_objs + g_pred_class = gma[predicted_objs[:, 4].astype(int)] if predicted_objs.shape[0] > 0 else predicted_objs[:, 4] + g_predicted_objs = np.column_stack([predicted_objs[:, :4], g_pred_class, predicted_objs[:, 5]]) if predicted_objs.shape[0] > 0 else predicted_objs + + g_entry = _collect_image_stats(g_predicted_objs, g_true_objs, iou_thresholds) + if g_entry is not None: + group_stats_list[ci].append(g_entry) + + return {'stats': stats, 'group_stats_list': group_stats_list} + + +def _init_group_meter_sets(metric, metric_name, group_configs): + """Initialize list-based group-level metric meters on a detection metric instance.""" + metric.group_classwise_meter_sets = [ + [MetricMeter(f'{metric_name}_cfg{ci}_group_{i}', ':6.2f') for i in range(gc['num_groups'])] + for ci, gc in enumerate(group_configs) + ] + metric.group_metric_meters = [ + MetricMeter(f'{metric_name}_cfg{ci}_mean', ':6.2f') for ci in range(len(group_configs)) + ] + metric.group_weighted_metric_meters = [ + MetricMeter(f'{metric_name}_cfg{ci}_weighted', ':6.2f') for ci in range(len(group_configs)) + ] + + +def _update_group_meter_set(group_stats, num_groups, iou_idx, ap_fn, + group_classwise_meters, group_metric_meter, group_weighted_meter, + mean_over_iou=False): + """Compute and update group-level metrics from group_stats. + + Args: + group_stats: list of stat tuples with group-remapped class IDs + num_groups: number of groups (= num_classes for group AP computation) + iou_idx: IoU threshold index (0 = IoU@0.5, 5 = IoU@0.75) + ap_fn: function to compute per-class APs (average_precisions_per_class, etc.) + mean_over_iou: if True, average AP over all IoU levels (for mAP50_95) + """ + if not group_stats: + group_metric_meter.update(0) + group_weighted_meter.update(0) + for m in group_classwise_meters: + m.update(0) + return + + concatenated = [np.concatenate(items, 0) for items in zip(*group_stats)] + group_aps = ap_fn(*concatenated, num_classes=num_groups) + + for i, m in enumerate(group_classwise_meters): + score = float(np.nanmean(group_aps[i, :])) if mean_over_iou else float(group_aps[i, iou_idx]) + m.update(score) + + mean_val = float(np.nanmean(group_aps)) if mean_over_iou else float(np.nanmean(group_aps[:, iou_idx])) + group_metric_meter.update(mean_val) + + true_group_ids = concatenated[3] + unique_groups, group_counts = np.unique(true_group_ids[true_group_ids >= 0], return_counts=True) + if mean_over_iou: + g_scores = np.nanmean(group_aps[unique_groups.astype(int), :], axis=1) + else: + g_scores = group_aps[unique_groups.astype(int), iou_idx] + valid = ~np.isnan(g_scores) + weighted = float(np.sum(g_scores[valid] * group_counts[valid]) / np.sum(group_counts[valid])) if valid.sum() > 0 else 0.0 + group_weighted_meter.update(weighted) + + +def _apply_group_updates(metric, group_stats_list, iou_idx, ap_fn, mean_over_iou=False): + """Iterate over all group configs and update their meter sets.""" + if metric.group_metric_meters is None: + return + for ci, (gc, gstats) in enumerate(zip(metric.group_configs, group_stats_list)): + _update_group_meter_set( + gstats, gc['num_groups'], iou_idx=iou_idx, ap_fn=ap_fn, + group_classwise_meters=metric.group_classwise_meter_sets[ci], + group_metric_meter=metric.group_metric_meters[ci], + group_weighted_meter=metric.group_weighted_metric_meters[ci], + mean_over_iou=mean_over_iou, + ) class mAP50(BaseMetric): - def __init__(self, num_classes, classwise_analysis, **kwargs): + def __init__(self, num_classes, classwise_analysis, group_configs=None, **kwargs): metric_name = 'mAP50' # Name for logging - super().__init__(metric_name=metric_name, num_classes=num_classes, classwise_analysis=classwise_analysis) + super().__init__(metric_name=metric_name, num_classes=num_classes, classwise_analysis=classwise_analysis, group_configs=group_configs or []) + self.weighted_metric_meter = MetricMeter(f'{metric_name}_weighted', ':6.2f') + if self.group_configs: + _init_group_meter_sets(self, metric_name, self.group_configs) def calibrate(self, predictions, targets, **kwargs): - stats = kwargs['stats'] # Get from DetectionMetricAdapter + stats = kwargs['stats'] + group_stats_list = kwargs.get('group_stats_list', []) - # Compute average precisions if any matches exist if stats: concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)] average_precisions = average_precisions_per_class(*concatenated_stats, num_classes=self.num_classes) @@ -319,20 +411,33 @@ def calibrate(self, predictions, targets, **kwargs): for i, classwise_meter in enumerate(self.classwise_metric_meters): classwise_meter.update(average_precisions[i, 0]) self.metric_meter.update(np.nanmean(average_precisions[:, 0])) + + true_class_ids = concatenated_stats[3] + unique_classes, class_counts = np.unique(true_class_ids[true_class_ids >= 0], return_counts=True) + ap_at_iou = average_precisions[unique_classes.astype(int), 0] + valid = ~np.isnan(ap_at_iou) + weighted = float(np.sum(ap_at_iou[valid] * class_counts[valid]) / np.sum(class_counts[valid])) if valid.sum() > 0 else 0.0 + self.weighted_metric_meter.update(weighted) else: self.metric_meter.update(0) + self.weighted_metric_meter.update(0) + + _apply_group_updates(self, group_stats_list, iou_idx=0, ap_fn=average_precisions_per_class) class mAP75(BaseMetric): - def __init__(self, num_classes, classwise_analysis, **kwargs): + def __init__(self, num_classes, classwise_analysis, group_configs=None, **kwargs): # TODO: Select metrics by user metric_name = 'mAP75' - super().__init__(metric_name=metric_name, num_classes=num_classes, classwise_analysis=classwise_analysis) + super().__init__(metric_name=metric_name, num_classes=num_classes, classwise_analysis=classwise_analysis, group_configs=group_configs or []) + self.weighted_metric_meter = MetricMeter(f'{metric_name}_weighted', ':6.2f') + if self.group_configs: + _init_group_meter_sets(self, metric_name, self.group_configs) def calibrate(self, predictions, targets, **kwargs): - stats = kwargs['stats'] # Get from DetectionMetricAdapter + stats = kwargs['stats'] + group_stats_list = kwargs.get('group_stats_list', []) - # Compute average precisions if any matches exist if stats: concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)] average_precisions = average_precisions_per_class(*concatenated_stats, num_classes=self.num_classes) @@ -341,20 +446,33 @@ def calibrate(self, predictions, targets, **kwargs): for i, classwise_meter in enumerate(self.classwise_metric_meters): classwise_meter.update(average_precisions[i, 5]) self.metric_meter.update(np.nanmean(average_precisions[:, 5])) + + true_class_ids = concatenated_stats[3] + unique_classes, class_counts = np.unique(true_class_ids[true_class_ids >= 0], return_counts=True) + ap_at_iou = average_precisions[unique_classes.astype(int), 5] + valid = ~np.isnan(ap_at_iou) + weighted = float(np.sum(ap_at_iou[valid] * class_counts[valid]) / np.sum(class_counts[valid])) if valid.sum() > 0 else 0.0 + self.weighted_metric_meter.update(weighted) else: self.metric_meter.update(0) + self.weighted_metric_meter.update(0) + + _apply_group_updates(self, group_stats_list, iou_idx=5, ap_fn=average_precisions_per_class) class mAP50_95(BaseMetric): - def __init__(self, num_classes, classwise_analysis, **kwargs): + def __init__(self, num_classes, classwise_analysis, group_configs=None, **kwargs): # TODO: Select metrics by user metric_name = 'mAP50_95' - super().__init__(metric_name=metric_name, num_classes=num_classes, classwise_analysis=classwise_analysis) + super().__init__(metric_name=metric_name, num_classes=num_classes, classwise_analysis=classwise_analysis, group_configs=group_configs or []) + self.weighted_metric_meter = MetricMeter(f'{metric_name}_weighted', ':6.2f') + if self.group_configs: + _init_group_meter_sets(self, metric_name, self.group_configs) def calibrate(self, predictions, targets, **kwargs): - stats = kwargs['stats'] # Get from DetectionMetricAdapter + stats = kwargs['stats'] + group_stats_list = kwargs.get('group_stats_list', []) - # Compute average precisions if any matches exist if stats: concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)] average_precisions = average_precisions_per_class(*concatenated_stats, num_classes=self.num_classes) @@ -363,17 +481,31 @@ def calibrate(self, predictions, targets, **kwargs): for i, classwise_meter in enumerate(self.classwise_metric_meters): classwise_meter.update(np.nanmean(average_precisions[i, :])) self.metric_meter.update(np.nanmean(average_precisions)) + + true_class_ids = concatenated_stats[3] + unique_classes, class_counts = np.unique(true_class_ids[true_class_ids >= 0], return_counts=True) + ap_mean_per_class = np.nanmean(average_precisions[unique_classes.astype(int), :], axis=1) + valid = ~np.isnan(ap_mean_per_class) + weighted = float(np.sum(ap_mean_per_class[valid] * class_counts[valid]) / np.sum(class_counts[valid])) if valid.sum() > 0 else 0.0 + self.weighted_metric_meter.update(weighted) else: self.metric_meter.update(0) + self.weighted_metric_meter.update(0) + + _apply_group_updates(self, group_stats_list, iou_idx=0, ap_fn=average_precisions_per_class, mean_over_iou=True) class Precision50(BaseMetric): - def __init__(self, num_classes, classwise_analysis, **kwargs): + def __init__(self, num_classes, classwise_analysis, group_configs=None, **kwargs): metric_name = 'Precision50' - super().__init__(metric_name=metric_name, num_classes=num_classes, classwise_analysis=classwise_analysis) + super().__init__(metric_name=metric_name, num_classes=num_classes, classwise_analysis=classwise_analysis, group_configs=group_configs or []) + self.weighted_metric_meter = MetricMeter(f'{metric_name}_weighted', ':6.2f') + if self.group_configs: + _init_group_meter_sets(self, metric_name, self.group_configs) def calibrate(self, predictions, targets, **kwargs): stats = kwargs['stats'] + group_stats_list = kwargs.get('group_stats_list', []) if stats: concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)] @@ -381,27 +513,51 @@ def calibrate(self, predictions, targets, **kwargs): if self.classwise_analysis: for i, classwise_meter in enumerate(self.classwise_metric_meters): - classwise_meter.update(np.nanmean(precisions[i, :])) - self.metric_meter.update(np.nanmean(precisions)) + classwise_meter.update(precisions[i, 0]) # IoU=0.5 only (index 0) + self.metric_meter.update(np.nanmean(precisions[:, 0])) # IoU=0.5 only (index 0) + + true_class_ids = concatenated_stats[3] + unique_classes, class_counts = np.unique(true_class_ids[true_class_ids >= 0], return_counts=True) + p_at_iou = precisions[unique_classes.astype(int), 0] + valid = ~np.isnan(p_at_iou) + weighted = float(np.sum(p_at_iou[valid] * class_counts[valid]) / np.sum(class_counts[valid])) if valid.sum() > 0 else 0.0 + self.weighted_metric_meter.update(weighted) else: self.metric_meter.update(0) + self.weighted_metric_meter.update(0) + + _apply_group_updates(self, group_stats_list, iou_idx=0, ap_fn=precisions_per_class) class Recall50(BaseMetric): - def __init__(self, num_classes, classwise_analysis, **kwargs): + def __init__(self, num_classes, classwise_analysis, group_configs=None, **kwargs): metric_name = 'Recall50' - super().__init__(metric_name=metric_name, num_classes=num_classes, classwise_analysis=classwise_analysis) + super().__init__(metric_name=metric_name, num_classes=num_classes, classwise_analysis=classwise_analysis, group_configs=group_configs or []) + self.weighted_metric_meter = MetricMeter(f'{metric_name}_weighted', ':6.2f') + if self.group_configs: + _init_group_meter_sets(self, metric_name, self.group_configs) def calibrate(self, predictions, targets, **kwargs): stats = kwargs['stats'] + group_stats_list = kwargs.get('group_stats_list', []) if stats: concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)] - recalls = recall_per_class(*concatenated_stats, num_classes=self.num_classes)[:, 0:1] + recalls = recall_per_class(*concatenated_stats, num_classes=self.num_classes)[:, 0:1] # IoU=0.5 only if self.classwise_analysis: for i, classwise_meter in enumerate(self.classwise_metric_meters): classwise_meter.update(np.nanmean(recalls[i, :])) self.metric_meter.update(np.nanmean(recalls)) + + true_class_ids = concatenated_stats[3] + unique_classes, class_counts = np.unique(true_class_ids[true_class_ids >= 0], return_counts=True) + r_at_iou = recalls[unique_classes.astype(int), 0] + valid = ~np.isnan(r_at_iou) + weighted = float(np.sum(r_at_iou[valid] * class_counts[valid]) / np.sum(class_counts[valid])) if valid.sum() > 0 else 0.0 + self.weighted_metric_meter.update(weighted) else: self.metric_meter.update(0) + self.weighted_metric_meter.update(0) + + _apply_group_updates(self, group_stats_list, iou_idx=0, ap_fn=recall_per_class) diff --git a/src/netspresso_trainer/models/utils.py b/src/netspresso_trainer/models/utils.py index df54087a..225ae43a 100644 --- a/src/netspresso_trainer/models/utils.py +++ b/src/netspresso_trainer/models/utils.py @@ -56,6 +56,10 @@ 'yolox_m': 'coco', 'yolox_l': 'coco', 'yolox_x': 'coco', + 'yolov9_tiny': 'coco', + 'yolov9_s': 'coco', + 'yolov9_c': 'coco', + 'yolov9_m': 'coco', 'rtdetr_res18': 'coco', 'rtdetr_res50': 'coco', 'yolo_fastest_v2': 'coco', @@ -134,11 +138,23 @@ 'yolox_x': { 'coco': "https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolox/yolox_x_coco.safetensors?versionId=NWskUEbSGviBWskHQ3P1dQZXnRXOR1WN", }, + 'yolov9_tiny': { + "coco": "https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolov9/yolov9_tiny_coco.safetensors?versionId=lFU6CTU6CayTyETvHr4o8k_Sh26vRH2F", + }, + "yolov9_s": { + "coco": "https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolov9/yolov9_s_coco.safetensors?versionId=EpMf6UaAZC0qwIRmQVR_mqeRObskt2PK", + }, + "yolov9_m": { + "coco": "https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolov9/yolov9_m_coco.safetensors?versionId=jjmbLq_06YW1VyUPexSvl6KDOBQJpFd5" + }, + "yolov9_c": { + "coco": "https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolov9/yolov9_c_coco.safetensors?versionId=TDZWEU8pi_c0ZHPS_U073BoqXaUFCviN", + }, 'rtdetr_res18': { - 'coco': "https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/rtdetr/rtdetr_res18_coco.safetensors?versionId=uu9v49NI6rQx8wOY6bJbEXUFOG_R9xqH", + 'coco': "https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/rtdetr/rtdetr_res18_coco.safetensors?versionId=9uegrNukkbp5ySO4vC52WPFhUEbEpEbD", }, 'rtdetr_res50': { - 'coco': "https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/rtdetr/rtdetr_res50_coco.safetensors?versionId=JHmnjY13BEflpnDCYPFJ1c17UwpqDrLQ", + 'coco': "https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/rtdetr/rtdetr_res50_coco.safetensors?versionId=ZwwBP5C9CE2oRoBJy5Gjr7aTFMAb2hdz", }, 'yolo_fastest_v2': { 'coco': "https://netspresso-trainer-public.s3.ap-northeast-2.amazonaws.com/checkpoint/yolofastest/yolo_fastest_v2_coco.safetensors?versionId=CGhNjiZygGVjtHm0M586DzQ6.2FqWvl1" diff --git a/src/netspresso_trainer/pipelines/base.py b/src/netspresso_trainer/pipelines/base.py index 5fd29bce..888e1a3a 100644 --- a/src/netspresso_trainer/pipelines/base.py +++ b/src/netspresso_trainer/pipelines/base.py @@ -72,6 +72,56 @@ def log_results( elapsed_time=elapsed_time ) + def _enrich_data_stats_with_groups(self, data_stats): + """Add instances_per_group_list to data_stats by aggregating instances_per_class via each group_map.""" + if data_stats is None: + return None + group_configs = getattr(getattr(self, 'metric_factory', None), 'group_configs', None) or [] + if not group_configs: + return data_stats + instances_per_class = data_stats.get('instances_per_class', {}) + instances_per_group_list = [] + for gc in group_configs: + gmap = gc['group_map'] + ipg = {} + for class_id, count in instances_per_class.items(): + gid = gmap.get(int(class_id), int(class_id)) + ipg[gid] = ipg.get(gid, 0) + count + instances_per_group_list.append(ipg) + return {**data_stats, 'instances_per_group_list': instances_per_group_list} + + def _convert_classwise_to_names(self, metrics): + """Convert integer class keys in classwise and group_results dicts to 'id_name' strings.""" + group_configs = getattr(getattr(self, 'metric_factory', None), 'group_configs', None) or [] + first_metric = metrics[list(metrics.keys())[0]] + has_classwise = 'classwise' in first_metric + has_group_results = 'group_results' in first_metric + if not has_classwise and not has_group_results: + return metrics + + tmp_metrics = {} + for metric_name, metric in metrics.items(): + tmp_metrics[metric_name] = {k: v for k, v in metric.items() if k not in ('classwise', 'group_results')} + if has_classwise and 'classwise' in metric: + tmp_metrics[metric_name]['classwise'] = {} + for cls_num, score in metric['classwise'].items(): + cls_name = self.logger.class_map.get(cls_num, str(cls_num)) + tmp_metrics[metric_name]['classwise'][f'{cls_num}_{cls_name}'] = score + if has_group_results and 'group_results' in metric: + converted_group_results = [] + for ci, gr in enumerate(metric['group_results']): + gc = group_configs[ci] if ci < len(group_configs) else {} + group_names = gc.get('group_names', {}) + converted_gc = {k: v for k, v in gr.items() if k != 'group_classwise'} + if 'group_classwise' in gr: + converted_gc['group_classwise'] = {} + for grp_num, score in gr['group_classwise'].items(): + grp_name = group_names.get(grp_num, str(grp_num)) + converted_gc['group_classwise'][f'{grp_num}_{grp_name}'] = score + converted_group_results.append(converted_gc) + tmp_metrics[metric_name]['group_results'] = converted_group_results + return tmp_metrics + @abstractmethod def save_summary(self): raise NotImplementedError diff --git a/src/netspresso_trainer/pipelines/builder.py b/src/netspresso_trainer/pipelines/builder.py index 758c0ded..8214dcb9 100644 --- a/src/netspresso_trainer/pipelines/builder.py +++ b/src/netspresso_trainer/pipelines/builder.py @@ -101,7 +101,7 @@ def build_pipeline( # Build loss and metric modules loss_factory = build_losses(conf.model, cur_epoch=cur_epoch) - metric_factory = build_metrics(task, conf.model, conf.logging.metrics, train_dataloader.dataset.num_classes) + metric_factory = build_metrics(task, conf.model, conf.logging.metrics, train_dataloader.dataset.num_classes, class_map=class_map) # Set model EMA model_ema = None @@ -148,7 +148,7 @@ def build_pipeline( # Build modules for evaluation loss_factory = build_losses(conf.model) - metric_factory = build_metrics(task, conf.model, conf.logging.metrics, eval_dataloader.dataset.num_classes) + metric_factory = build_metrics(task, conf.model, conf.logging.metrics, eval_dataloader.dataset.num_classes, class_map=class_map) # Build logger single_gpu_or_rank_zero = (not conf.distributed) or (conf.distributed and dist.get_rank() == 0) diff --git a/src/netspresso_trainer/pipelines/evaluation.py b/src/netspresso_trainer/pipelines/evaluation.py index cb452675..0419be03 100644 --- a/src/netspresso_trainer/pipelines/evaluation.py +++ b/src/netspresso_trainer/pipelines/evaluation.py @@ -98,22 +98,15 @@ def log_end_evaluation( metrics = self.metric_factory.result('valid') # TODO: Move to logger - # If class-wise metrics, convert to class names - if 'classwise' in metrics[list(metrics.keys())[0]]: - tmp_metrics = {} - for metric_name, metric in metrics.items(): - tmp_metrics[metric_name] = {'mean': metric['mean'], 'classwise': {}} - for cls_num, score in metric['classwise'].items(): - cls_name = self.logger.class_map[cls_num] if cls_num in self.logger.class_map else 'mean' - tmp_metrics[metric_name]['classwise'][f'{cls_num}_{cls_name}'] = score - metrics = tmp_metrics + # If class-wise or group metrics, convert integer keys to name strings + metrics = self._convert_classwise_to_names(metrics) self.log_results( prefix='evaluation', samples=valid_samples, losses=losses, metrics=metrics, - data_stats=self.eval_data_stats, + data_stats=self._enrich_data_stats_with_groups(self.eval_data_stats), elapsed_time=time_for_evaluation, ) predictions = self.task_processor.get_predictions(valid_samples, self.logger.class_map) diff --git a/src/netspresso_trainer/pipelines/train.py b/src/netspresso_trainer/pipelines/train.py index cb42080e..9656372e 100644 --- a/src/netspresso_trainer/pipelines/train.py +++ b/src/netspresso_trainer/pipelines/train.py @@ -35,6 +35,7 @@ from ..losses.builder import LossFactory from ..metrics.builder import MetricFactory from ..utils.checkpoint import load_checkpoint, save_checkpoint +from ..utils.exir import save_exir from ..utils.fx import save_graphmodule from ..utils.logger import yaml_for_logging from ..utils.model_ema import ModelEMA @@ -254,8 +255,14 @@ def log_end_epoch( ): train_losses = self.loss_factory.result('train') train_metrics = self.metric_factory.result('train') + + # TODO: Move to logger + # If class-wise metrics, convert to class names + train_metrics = self._convert_classwise_to_names(train_metrics) + self.log_results(prefix='training', epoch=epoch, losses=train_losses, metrics=train_metrics, - data_stats=self.train_data_stats, learning_rate=self.learning_rate, elapsed_time=time_for_epoch) + data_stats=self._enrich_data_stats_with_groups(self.train_data_stats), + learning_rate=self.learning_rate, elapsed_time=time_for_epoch) if valid_logging: valid_losses = self.loss_factory.result('valid') if valid_logging else None @@ -263,17 +270,11 @@ def log_end_epoch( # TODO: Move to logger # If class-wise metrics, convert to class names - if 'classwise' in valid_metrics[list(valid_metrics.keys())[0]]: - tmp_metrics = {} - for metric_name, metric in valid_metrics.items(): - tmp_metrics[metric_name] = {'mean': metric['mean'], 'classwise': {}} - for cls_num, score in metric['classwise'].items(): - cls_name = self.logger.class_map[cls_num] if cls_num in self.logger.class_map else 'mean' - tmp_metrics[metric_name]['classwise'][f'{cls_num}_{cls_name}'] = score - valid_metrics = tmp_metrics + valid_metrics = self._convert_classwise_to_names(valid_metrics) self.log_results(prefix='validation', epoch=epoch, samples=valid_samples, losses=valid_losses, - metrics=valid_metrics, data_stats=self.eval_data_stats) + metrics=valid_metrics, + data_stats=self._enrich_data_stats_with_groups(self.eval_data_stats)) summary_record = {'train_losses': train_losses, 'train_metrics': train_metrics} if valid_logging: @@ -355,6 +356,12 @@ def save_best(self): sample_input=self.sample_input.type(save_dtype), opset_version=opset_version) logger.info(f"ONNX model converting and saved at {str(model_save_path.with_suffix('.onnx'))}") + + save_exir(best_model, + model_save_path.with_suffix('.exir'), + sample_input=self.sample_input.type(save_dtype)) + logger.info(f"EXIR model converting and saved at {str(model_save_path.with_suffix('.exir'))}") + if self.logger.use_mlflow: self.logger.mlflow_logger.log_onnx_model(model_save_path.with_suffix('.onnx'), input_example=self.sample_input.type(save_dtype)) diff --git a/src/netspresso_trainer/utils/exir.py b/src/netspresso_trainer/utils/exir.py new file mode 100644 index 00000000..4dda99a1 --- /dev/null +++ b/src/netspresso_trainer/utils/exir.py @@ -0,0 +1,36 @@ +# Copyright (C) 2024 Nota Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ---------------------------------------------------------------------------- +from pathlib import Path +from typing import Union + +import torch +import torch.nn as nn +from loguru import logger +from torch import Tensor + +from .environment import get_device + +__all__ = ['save_exir'] + + +def save_exir(model: nn.Module, f: Union[str, Path], sample_input: Tensor): + if not hasattr(torch, 'export'): + logger.warning("Current torch version does not support torch.export. Please upgrade torch.") + return + sample_input = sample_input.to(get_device(model)) + exported_program = torch.export.export(model, (sample_input, )) + torch.export.save(exported_program, f) + return exported_program diff --git a/src/netspresso_trainer/utils/logger.py b/src/netspresso_trainer/utils/logger.py index d167b122..d3c7412b 100644 --- a/src/netspresso_trainer/utils/logger.py +++ b/src/netspresso_trainer/utils/logger.py @@ -40,7 +40,7 @@ def rank_filter(record): try: return dist.get_rank() == 0 - except RuntimeError: # Default process group has not been initialized, please make sure to call init_process_group. + except (RuntimeError, ValueError): # Default process group has not been initialized, please make sure to call init_process_group. return True def get_format(level: str, distributed: bool = False): diff --git a/tools/exir_convert.py b/tools/exir_convert.py new file mode 100644 index 00000000..ffaee37d --- /dev/null +++ b/tools/exir_convert.py @@ -0,0 +1,90 @@ +# Copyright (C) 2024 Nota Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ---------------------------------------------------------------------------- + +import argparse +import os +from itertools import chain +from pathlib import Path +from typing import List + +import torch +import torch.nn as nn +from netspresso_trainer.models import build_model, is_single_task_model +from netspresso_trainer.utils.exir import save_exir +from omegaconf import OmegaConf + + + +TEMP_NUM_CLASSES = 80 + + +def parse_args(): + parser = argparse.ArgumentParser(description="Parser for NetsPresso Export conversion") + + parser.add_argument( + '-c', '--config-path', type=str, default="config/model/yolox/yolox-s-detection.yaml", + help="Model config path") + parser.add_argument( + '-n', '--num-classes', type=int, default=TEMP_NUM_CLASSES, + help="Number of classes") + parser.add_argument( + '-o', '--output-dir', type=str, default="exir/", + help="Export model output directory") + parser.add_argument( + '--sample-size', type=int, nargs=2, default=(640, 640), + help="Input sample size") + parser.add_argument( + '--debug', action='store_true', help="Debug mode to check with the error message") + + args, _ = parser.parse_known_args() + return args + + +def get_model_config_path_list(config_path_or_dir: Path) -> List[Path]: + if config_path_or_dir.is_dir(): + config_dir = config_path_or_dir + return sorted(chain(config_dir.glob("*.yaml"), config_dir.glob("*.yml"))) + config_path = config_path_or_dir + return [config_path] + + +if __name__ == '__main__': + args = parse_args() + + config_path_list = get_model_config_path_list(Path(args.config_path)) + os.makedirs(args.output_dir, exist_ok=True) + + for model_config_path in config_path_list: + try: + print(f"Export conversion for ({model_config_path})..... ", end='', flush=True) + config = OmegaConf.load(model_config_path) + config = config.model + config.single_task_model = is_single_task_model(config) + torch_model: nn.Module = build_model(config, num_classes=args.num_classes, devices=torch.device("cpu"), distributed=False) + torch_model.eval() + sample_input = torch.randn(1, 3, *args.sample_size) + save_exir(torch_model, + f=Path(args.output_dir) / f"{model_config_path.stem}.exir", + sample_input=sample_input) + print("Success!") + except KeyboardInterrupt: + print("") + break + except Exception as e: + print("Failed!") + if args.debug: + raise e + print(e)