Merge pull request #4 from codalab/new_codabench_example_comp

Mini MNIST bundle added

Author: Didayolo

.gitignore

@@ -216,5 +216,3 @@ pip-log.txt
 
 #Mr Developer
 .mr.developer.cfg
-
-*.zip

codabench/Mini-MNIST/Mini-MNIST-Bundle.zip

@@ -0,0 +1,45 @@
+# Mini-MNIST Challenge Bundle
+# Created by: Ihsan Ullah
+# Created on: Apr 05, 2026
+
+version: 2
+title: Mini-MNIST Digit Classification
+description: A mini-MNIST challenge with 20 training images per class and 5 test images per class.
+image: logo.png
+registration_auto_approve: True
+docker_image: codalab/codalab-legacy:py39
+
+terms: pages/terms.md
+pages:  
+  - title: Overview
+    file: pages/overview.md
+  - title: Evaluation
+    file: pages/evaluation.md
+
+phases:
+  - index: 0
+    name: Competition Phase
+    description: Train on 20 images/class and predict on 5 images/class.
+    start: 2026-01-01
+    end: 2027-01-01
+    tasks:
+      - 0
+
+tasks:
+  - index: 0
+    name: MNIST Classification Task
+    description: Classify 50 handwritten digits (5 per class).
+    input_data: input_data/
+    reference_data: reference_data/  
+    scoring_program: scoring_program/
+    ingestion_program: ingestion_program/
+
+leaderboards:
+  - index: 0 
+    title: Results
+    key: main_leaderboard
+    submission_rule: "Force_Latest_Multiple"
+    columns:
+      - index: 0
+        title: Accuracy
+        key: score

codabench/Mini-MNIST/Mini-MNIST-Bundle/README.md

@@ -0,0 +1,142 @@
+# ------------------------------------------
+# Imports
+# ------------------------------------------
+import os
+import json
+import numpy as np
+import pandas as pd
+from PIL import Image
+from datetime import datetime as dt
+
+
+class Ingestion:
+    """
+    Class for handling the ingestion process.
+
+    Args:
+        None
+
+    Attributes:
+        * start_time (datetime): The start time of the ingestion process.
+        * end_time (datetime): The end time of the ingestion process.
+        * model (object): The model object.
+        * X_train (np.ndarray): The train images.
+        * y_train (np.ndarray): The train labels.
+        * X_test (np.ndarray): The test images.
+        * y_pred (np.ndarray): The test predictions.
+        * ingestion_result (dict): The ingestion result dict.
+    """
+
+    def __init__(self):
+        """
+        Initialize the Ingestion class.
+        """
+        self.start_time = None
+        self.end_time = None
+        self.model = None
+        self.X_train = None
+        self.y_train = None
+        self.X_test = None
+        self.y_pred = None
+        self.ingestion_result = {}
+
+    def start_timer(self):
+        """
+        Start the timer for the ingestion process.
+        """
+        self.start_time = dt.now()
+
+    def stop_timer(self):
+        """
+        Stop the timer for the ingestion process.
+        """
+        self.end_time = dt.now()
+
+    def get_duration(self):
+        """
+        Get the duration of the ingestion process.
+
+        Returns:
+            timedelta: The duration of the ingestion process.
+        """
+        if self.start_time is None:
+            print("[-] Timer was never started. Returning None")
+            return None
+
+        if self.end_time is None:
+            print("[-] Timer was never stopped. Returning None")
+            return None
+
+        return self.end_time - self.start_time
+
+    def load_train_and_test_data(self, input_dir):
+        """
+        Load the training and testing data.
+
+        Args:
+            input_dir (str): The input directory name.
+        """
+        print("[*] Loading data")
+
+        # Load Training Data
+        train_data_dir = os.path.join(input_dir, "train_data")
+        train_labels_file = os.path.join(input_dir, "train_labels.csv")
+        if os.path.exists(train_labels_file) and os.path.isdir(train_data_dir):
+            df_train = pd.read_csv(train_labels_file)
+            images = []
+            for filename in df_train['filename']:
+                img_path = os.path.join(train_data_dir, filename)
+                img = Image.open(img_path)
+                images.append(np.array(img))
+            self.X_train = np.array(images)
+            self.y_train = df_train['label'].values
+
+        # Load Test Data
+        test_data_dir = os.path.join(input_dir, "test_data")
+        if os.path.isdir(test_data_dir):
+            # Sort files to ensure consistency with reference labels
+            test_files = sorted([f for f in os.listdir(test_data_dir) if f.endswith('.png')])
+            images = []
+            for filename in test_files:
+                img_path = os.path.join(test_data_dir, filename)
+                img = Image.open(img_path)
+                images.append(np.array(img))
+            self.X_test = np.array(images)
+
+    def init_submission(self, Model):
+        """
+        Initialize the submitted model.
+
+        Args:
+            Model (object): The model class.
+        """
+        print("[*] Initializing Submitted Model")
+        self.model = Model()
+
+    def fit_submission(self):
+        """
+        Fit the submitted model.
+        """
+        print("[*] Fitting Submitted Model")
+        if self.X_train is not None and self.y_train is not None:
+            self.model.fit(self.X_train, self.y_train)
+
+    def predict_submission(self):
+        """
+        Make predictions using the submitted model.
+        """
+        print("[*] Predicting")
+        if self.X_test is not None:
+            self.y_pred = self.model.predict(self.X_test)
+
+    def save_predictions(self, output_dir=None):
+        """
+        Save the ingestion result to files.
+
+        Args:
+            output_dir (str): The output directory to save the result files.
+        """
+        if self.y_pred is not None:
+            prediction_file = os.path.join(output_dir, "predictions.npy")
+            np.save(prediction_file, self.y_pred)
+            print(f"[*] Saved predictions to {prediction_file}")

codabench/Mini-MNIST/Mini-MNIST-Bundle/competition.yaml

@@ -0,0 +1 @@
+command: python3 run_ingestion.py --codabench

codabench/Mini-MNIST/Mini-MNIST-Bundle/ingestion_program/README.md

@@ -0,0 +1,91 @@
+# ------------------------------------------
+# Imports
+# ------------------------------------------
+import os
+import sys
+import argparse
+
+# ------------------------------------------
+# Directories
+# ------------------------------------------
+module_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir_name = os.path.dirname(module_dir)
+
+# ------------------------------------------
+# Args
+# ------------------------------------------
+parser = argparse.ArgumentParser(
+    description="This is script to run ingestion program for the competition"
+)
+parser.add_argument(
+    "--codabench",
+    help="True when running on Codabench",
+    action="store_true",
+)
+
+# ------------------------------------------
+# Main
+# ------------------------------------------
+if __name__ == "__main__":
+
+    print("\n----------------------------------------------")
+    print("Ingestion Program started!")
+    print("----------------------------------------------\n\n")
+
+    args = parser.parse_args()
+
+    if not args.codabench:
+        # DO NOT CHANGE THESE PATHS UNLESS YOU CHANGE THE FOLDER NAMES IN THE BUNDLE
+        input_dir = os.path.join(root_dir_name, "input_data")
+        output_dir = os.path.join(root_dir_name, "sample_result_submission")
+        program_dir = os.path.join(root_dir_name, "ingestion_program")
+        submission_dir = os.path.join(root_dir_name, "sample_code_submission")
+    else:
+        # DO NOT CHANGE THESE PATHS. THESE ARE USED ON THE CODABENCH PLATFORM
+        input_dir = "/app/input_data"
+        output_dir = "/app/output"
+        program_dir = "/app/program"
+        submission_dir = "/app/ingested_program"
+
+    sys.path.append(input_dir)
+    sys.path.append(output_dir)
+    sys.path.append(program_dir)
+    sys.path.append(submission_dir)
+
+    # Import ingestion class
+    from ingestion import Ingestion
+    # Import model from submission dir
+    from model import Model
+
+    # Initialize Ingestions
+    ingestion = Ingestion()
+
+    # Start timer
+    ingestion.start_timer()
+
+    # Load train and test data
+    ingestion.load_train_and_test_data(input_dir)
+
+    # initialize submission
+    ingestion.init_submission(Model)
+
+    # fit submission
+    ingestion.fit_submission()
+
+    # predict submission
+    ingestion.predict_submission()
+
+    # save result
+    ingestion.save_predictions(output_dir)
+
+    # Stop timer
+    ingestion.stop_timer()
+
+    # Show duration
+    print("\n------------------------------------")
+    print(f"[✔] Total duration: {ingestion.get_duration()}")
+    print("------------------------------------")
+
+    print("\n----------------------------------------------")
+    print("[✔] Ingestion Program executed successfully!")
+    print("----------------------------------------------\n\n")