🚚 [Rename] All utils, tools, higher readability

Author: henrytsui000

docs/HOWTO.md

@@ -88,3 +88,61 @@ Custom transformations should be designed to accept an image and its bounding bo
     # ... (Other Transform)
     CustomTransform: 0.5
     ```
+
+
+- **Utils**
+    - **bbox_utils**
+        - `class` Anchor2Box: transform predicted anchor to bounding box
+        - `class` Matcher: given prediction and groudtruth, find the groundtruth for each prediction
+        - `func` calculate_iou: calculate iou for given two list of bbox
+        - `func` transform_bbox: transform bbox from {xywh, xyxy, xcycwh} to {xywh, xyxy, xcycwh}
+        - `func` generate_anchors: given image size, make the anchor point for the given size
+    - **dataset_utils**
+        - `func` locate_label_paths:
+        - `func` create_image_metadata:
+        - `func` organize_annotations_by_image:
+        - `func` scale_segmentation:
+    - **logging_utils**
+        - `func` custom_log: custom loguru, overiding the origin logger
+        - `class` ProgressTracker: A class to handle output for each batch, epoch
+        - `func` log_model_structure: give a torch model, print it as a table
+        - `func` validate_log_directory: for given experiemnt, check if the log folder already existed
+    - **model_utils**
+        - `class` ExponentialMovingAverage: a mirror of model, do ema on model
+        - `func` create_optimizer: return a optimzer, for example SDG, ADAM
+        - `func` create_scheduler: return a scheduler, for example Step, Lambda
+    - **module_utils**
+        - `func` get_layer_map:
+        - `func` auto_pad: given a convolution block, return how many pixel should conv padding
+        - `func` create_activation_function: given a `func` name, return a activation `func`tion
+        - `func` round_up: given number and divider, return a number is mutliplcation of divider
+        - `func` divide_into_chunks: for a given list and n, seperate list to n sub list
+    - **trainer**
+        - `class` Trainer: a class can automatic train the model
+- **Tools**
+    - **converter_json2txt**
+        - `func` discretize_categories: given the dictionary class, turn id from 1: class
+        - `func` process_annotations: handle the whole dataset annotations
+        - `func` process_annotation: handle a annotation(a list of bounding box)
+        - `func` normalize_segmentation: normalize segmentation position to 0~1
+        - `func` convert_annotations: convert json annotations to txt file structure
+    - **data_augment**
+        - `class` AugmentationComposer: Compose a list of data augmentation strategy
+        - `class` VerticalFlip: a custom data augmentation, Random Vertical Flip
+        - `class` Mosaic: a data augmentation strategy, follow YOLOv5
+    - **dataloader**
+        - `class` YoloDataset: a custom dataset for training yolo's model
+        - `class` YoloDataLoader: a dataloader base on torch's dataloader, with custom allocate function
+        - `func` create_dataloader: given a config file, return a YOLO dataloader
+    - **drawer**
+        - `func` draw_bboxes: given a image and list of bbox, draw bbox on the image
+        - `func` draw_model: visualize the given model
+    - **get_dataset**
+        - `func` download_file: for a given link, downlaod the file
+        - `func` unzip_file: unzip the downlaoded zip to data/
+        - `func` check_files: check if the dataset file numbers is correct
+        - `func` prepare_dataset: automatic downlaod the dataset and check if it is correct
+    - **loss**
+        - `class` BoxLoss: a Custom Loss for bounding box
+        - `class` YOLOLoss: a implementation of yolov9 loss
+        - `class` DualLoss: a implementation of yolov9 loss with auxiliary detection head

examples/example_train.py

@@ -9,23 +9,23 @@
 sys.path.append(str(project_root))
 
 from yolo.config.config import Config
-from yolo.tools.log_helper import custom_logger, get_valid_folder
-from yolo.tools.trainer import Trainer
-from yolo.utils.dataloader import get_dataloader
-from yolo.utils.get_dataset import prepare_dataset
+from yolo.tools.data_loader import create_dataloader
+from yolo.tools.dataset_preparation import prepare_dataset
+from yolo.tools.trainer import ModelTrainer
+from yolo.utils.logging_utils import custom_logger, validate_log_directory
 
 
 @hydra.main(config_path="../yolo/config", config_name="config", version_base=None)
 def main(cfg: Config):
     custom_logger()
-    save_path = get_valid_folder(cfg.hyper.general, cfg.name)
+    save_path = validate_log_directory(cfg.hyper.general, cfg.name)
     if cfg.download.auto:
         prepare_dataset(cfg.download)
 
-    dataloader = get_dataloader(cfg)
+    dataloader = create_dataloader(cfg)
     # TODO: get_device or rank, for DDP mode
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    trainer = Trainer(cfg, save_path, device)
+    trainer = ModelTrainer(cfg, save_path, device)
     trainer.train(dataloader, cfg.hyper.train.epoch)
 
 

tests/test_utils/test_dataaugment.py

@@ -9,7 +9,7 @@
 project_root = Path(__file__).resolve().parent.parent.parent
 sys.path.append(str(project_root))
 
-from yolo.utils.data_augment import Compose, HorizontalFlip, Mosaic, VerticalFlip
+from yolo.utils.data_augmentation import Compose, HorizontalFlip, Mosaic, VerticalFlip
 
 
 def test_horizontal_flip():

tests/test_utils/test_loss.py

@@ -8,7 +8,7 @@
 project_root = Path(__file__).resolve().parent.parent.parent
 sys.path.append(str(project_root))
 
-from yolo.utils.loss import YOLOLoss
+from yolo.utils.loss_functions import YOLOLoss
 
 
 @pytest.fixture

yolo/model/module.py

@@ -6,7 +6,7 @@
 from torch import Tensor, nn
 from torch.nn.common_types import _size_2_t
 
-from yolo.tools.module_helper import auto_pad, get_activation, round_up
+from yolo.utils.module_utils import auto_pad, create_activation_function, round_up
 
 
 # ----------- Basic Class ----------- #
@@ -26,7 +26,7 @@ def __init__(
         kwargs.setdefault("padding", auto_pad(kernel_size, **kwargs))
         self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, bias=False, **kwargs)
         self.bn = nn.BatchNorm2d(out_channels, eps=1e-3, momentum=3e-2)
-        self.act = get_activation(activation)
+        self.act = create_activation_function(activation)
 
     def forward(self, x: Tensor) -> Tensor:
         return self.act(self.bn(self.conv(x)))
@@ -109,7 +109,7 @@ def __init__(
         **kwargs
     ):
         super().__init__()
-        self.act = get_activation(activation)
+        self.act = create_activation_function(activation)
         self.conv1 = Conv(in_channels, out_channels, kernel_size, activation=False, **kwargs)
         self.conv2 = Conv(in_channels, out_channels, 1, activation=False, **kwargs)
 

yolo/model/yolo.py

@@ -5,9 +5,9 @@
 from omegaconf import ListConfig, OmegaConf
 
 from yolo.config.config import Config, Model, YOLOLayer
-from yolo.tools.log_helper import log_model
-from yolo.tools.module_helper import get_layer_map
-from yolo.utils.drawer import draw_model
+from yolo.tools.drawer import draw_model
+from yolo.utils.logging_utils import log_model_structure
+from yolo.utils.module_utils import get_layer_map
 
 
 class YOLO(nn.Module):
@@ -125,6 +125,6 @@ def get_model(cfg: Config) -> YOLO:
     OmegaConf.set_struct(cfg.model, False)
     model = YOLO(cfg.model, cfg.hyper.data.class_num)
     logger.info("✅ Success load model")
-    log_model(model.model)
+    log_model_structure(model.model)
     # draw_model(model=model)
     return model

yolo/utils/data_augment.py yolo/tools/data_augmentation.pyyolo/utils/data_augment.py renamed to yolo/tools/data_augmentation.py

@@ -4,7 +4,7 @@
 from torchvision.transforms import functional as TF
 
 
-class Compose:
+class AugmentationComposer:
     """Composes several transforms together."""
 
     def __init__(self, transforms, image_size: int = 640):

yolo/utils/converter_json2txt.py yolo/tools/data_conversion.pyyolo/utils/converter_json2txt.py renamed to yolo/tools/data_conversion.py

@@ -13,13 +13,19 @@
 from tqdm.rich import tqdm
 
 from yolo.config.config import Config
-from yolo.tools.dataset_helper import (
-    create_image_info_dict,
-    find_labels_path,
-    get_scaled_segmentation,
+from yolo.tools.data_augmentation import (
+    AugmentationComposer,
+    HorizontalFlip,
+    MixUp,
+    Mosaic,
+    VerticalFlip,
+)
+from yolo.tools.drawer import draw_bboxes
+from yolo.utils.dataset_utils import (
+    create_image_metadata,
+    locate_label_paths,
+    scale_segmentation,
 )
-from yolo.utils.data_augment import Compose, HorizontalFlip, MixUp, Mosaic, VerticalFlip
-from yolo.utils.drawer import draw_bboxes
 
 
 class YoloDataset(Dataset):
@@ -30,7 +36,7 @@ def __init__(self, config: dict, phase: str = "train2017", image_size: int = 640
         self.image_size = image_size
 
         transforms = [eval(aug)(prob) for aug, prob in augment_cfg.items()]
-        self.transform = Compose(transforms, self.image_size)
+        self.transform = AugmentationComposer(transforms, self.image_size)
         self.transform.get_more_data = self.get_more_data
         self.data = self.load_data(dataset_cfg.path, phase_name)
 
@@ -68,10 +74,10 @@ def filter_data(self, dataset_path: str, phase_name: str) -> list:
             list: A list of tuples, each containing the path to an image file and its associated segmentation as a tensor.
         """
         images_path = path.join(dataset_path, "images", phase_name)
-        labels_path, data_type = find_labels_path(dataset_path, phase_name)
+        labels_path, data_type = locate_label_paths(dataset_path, phase_name)
         images_list = sorted(os.listdir(images_path))
         if data_type == "json":
-            annotations_index, image_info_dict = create_image_info_dict(labels_path)
+            annotations_index, image_info_dict = create_image_metadata(labels_path)
 
         data = []
         valid_inputs = 0
@@ -85,7 +91,7 @@ def filter_data(self, dataset_path: str, phase_name: str) -> list:
                 if image_info is None:
                     continue
                 annotations = annotations_index.get(image_info["id"], [])
-                image_seg_annotations = get_scaled_segmentation(annotations, image_info)
+                image_seg_annotations = scale_segmentation(annotations, image_info)
                 if not image_seg_annotations:
                     continue
 
@@ -191,21 +197,21 @@ def collate_fn(self, batch: List[Tuple[torch.Tensor, torch.Tensor]]) -> Tuple[to
         return batch_images, batch_targets
 
 
-def get_dataloader(config):
+def create_dataloader(config):
     return YoloDataLoader(config)
 
 
 @hydra.main(config_path="../config", config_name="config", version_base=None)
 def main(cfg):
-    dataloader = get_dataloader(cfg)
+    dataloader = create_dataloader(cfg)
     draw_bboxes(*next(iter(dataloader)))
 
 
 if __name__ == "__main__":
     import sys
 
     sys.path.append("./")
-    from tools.log_helper import custom_logger
+    from tools.logging_utils import custom_logger
 
     custom_logger()
     main()