feat: Add GitHub Actions workflow for model export and enhance DeepLabV3 Android demo with improved segmentation visualization and inference time display.

kirklandsign · kirklandsign · commit 2207d1660263 · 2026-01-15T22:44:54.000-08:00
diff --git a/.github/workflows/export-models.yml b/.github/workflows/export-models.yml
@@ -7,6 +7,7 @@
 name: Export Models
 
 on:
+  pull_request:
   schedule:
     # Run nightly at midnight UTC
     - cron: '0 0 * * *'
@@ -41,6 +42,9 @@ jobs:
 
     name: Export ${{ matrix.name }}
 
+  export-dl3:
+    runs-on: ubuntu-latest
+    name: Export DeepLabV3 Model
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -65,3 +69,14 @@ jobs:
           name: ${{ matrix.artifact }}
           path: ${{ matrix.output }}
           if-no-files-found: error
+        run: pip install executorch torchvision
+
+      - name: Export DL3 model
+        working-directory: dl3/python
+        run: python export.py
+
+      - name: Upload PTE model
+        uses: actions/upload-artifact@v4
+        with:
+          name: dl3_xnnpack_fp32.pte
+          path: dl3/python/dl3_xnnpack_fp32.pte
diff --git a/dl3/android/DeepLabV3Demo/app/src/main/java/org/pytorch/executorchexamples/dl3/MainActivity.java b/dl3/android/DeepLabV3Demo/app/src/main/java/org/pytorch/executorchexamples/dl3/MainActivity.java
@@ -33,7 +33,11 @@
 import androidx.core.content.ContextCompat;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.util.ArrayList;
 import org.pytorch.executorch.EValue;
 import org.pytorch.executorch.Module;
@@ -42,10 +46,18 @@
 public class MainActivity extends Activity implements Runnable {
   private ImageView mImageView;
   private Button mButtonXnnpack;
+  private Button mDownloadModelButton;
   private ProgressBar mProgressBar;
+  private android.widget.TextView mInferenceTimeText;
+  private android.widget.TextView mModelStatusText;
   private Bitmap mBitmap = null;
   private Module mModule = null;
   private String mImagename = "corgi.jpeg";
+  private long mInferenceTime = 0;
+
+  // Model download configuration
+  private static final String MODEL_URL = "https://example.com/dl3_xnnpack_fp32.pte"; // TODO: Replace with actual URL
+  private static final String MODEL_PATH = "/data/local/tmp/dl3_xnnpack_fp32.pte";
 
   private final ArrayList<String> mImageFiles = new ArrayList<>();
 
@@ -57,9 +69,31 @@ public class MainActivity extends Activity implements Runnable {
   // see http://host.robots.ox.ac.uk:8080/pascal/VOC/voc2007/segexamples/index.html for the list of
   // classes with indexes
   private static final int CLASSNUM = 21;
-  private static final int DOG = 12;
-  private static final int PERSON = 15;
-  private static final int SHEEP = 17;
+
+  // Colors for all 21 PASCAL VOC classes
+  private static final int[] CLASS_COLORS = {
+      0x00000000, // 0: Background (transparent)
+      0xFFE6194B, // 1: Aeroplane (red)
+      0xFF3CB44B, // 2: Bicycle (green)
+      0xFFFFE119, // 3: Bird (yellow)
+      0xFF4363D8, // 4: Boat (blue)
+      0xFFF58231, // 5: Bottle (orange)
+      0xFF911EB4, // 6: Bus (purple)
+      0xFF46F0F0, // 7: Car (cyan)
+      0xFFF032E6, // 8: Cat (magenta)
+      0xFFBCF60C, // 9: Chair (lime)
+      0xFFFABEBE, // 10: Cow (pink)
+      0xFF008080, // 11: Dining Table (teal)
+      0xFF00FF00, // 12: Dog (bright green)
+      0xFF9A6324, // 13: Horse (brown)
+      0xFFFFD8B1, // 14: Motorbike (peach)
+      0xFFFF0000, // 15: Person (red)
+      0xFF800000, // 16: Potted Plant (maroon)
+      0xFF0000FF, // 17: Sheep (blue)
+      0xFF808000, // 18: Sofa (olive)
+      0xFFE6BEFF, // 19: Train (lavender)
+      0xFFAA6E28, // 20: TV/Monitor (tan)
+  };
 
   private void checkAndRequestStoragePermission() {
     if (ContextCompat.checkSelfPermission(this, Manifest.permission.READ_EXTERNAL_STORAGE)
@@ -197,14 +231,21 @@ protected void onCreate(Bundle savedInstanceState) {
     // Initialize all views first!
     mImageView = findViewById(R.id.imageView);
     mButtonXnnpack = findViewById(R.id.xnnpackButton);
+    mDownloadModelButton = findViewById(R.id.downloadModelButton);
     mProgressBar = findViewById(R.id.progressBar);
+    mInferenceTimeText = findViewById(R.id.inferenceTimeText);
+    mModelStatusText = findViewById(R.id.modelStatusText);
 
     populateImagePathFromAssets();
     showImage();
 
-    mModule = Module.load("/data/local/tmp/dl3_xnnpack_fp32.pte");
+    // Check if model exists and load it, otherwise show download button
+    loadModelOrShowDownloadButton();
     mImageView.setImageBitmap(mBitmap);
 
+    // Download button click handler
+    mDownloadModelButton.setOnClickListener(v -> downloadModel());
+
     final Button buttonNext = findViewById(R.id.nextButton);
     buttonNext.setOnClickListener(
         new View.OnClickListener() {
@@ -223,10 +264,9 @@ public void onClick(View v) {
     mButtonXnnpack.setOnClickListener(
         new View.OnClickListener() {
           public void onClick(View v) {
-            mModule.destroy();
-            mModule = Module.load("/data/local/tmp/dl3_xnnpack_fp32.pte");
             mButtonXnnpack.setEnabled(false);
             mProgressBar.setVisibility(ProgressBar.VISIBLE);
+            mInferenceTimeText.setVisibility(View.INVISIBLE);
             mButtonXnnpack.setText(getString(R.string.run_model));
 
             Thread thread = new Thread(MainActivity.this);
@@ -262,32 +302,38 @@ public void run() {
     boolean imageSegementationSuccess = false;
     final long startTime = SystemClock.elapsedRealtime();
     Tensor outputTensor = mModule.forward(EValue.from(inputTensor))[0].toTensor();
-    final long inferenceTime = SystemClock.elapsedRealtime() - startTime;
-    Log.d("ImageSegmentation", "inference time (ms): " + inferenceTime);
+    mInferenceTime = SystemClock.elapsedRealtime() - startTime;
+    Log.d("ImageSegmentation", "inference time (ms): " + mInferenceTime);
 
     final float[] scores = outputTensor.getDataAsFloatArray();
     int width = mBitmap.getWidth();
     int height = mBitmap.getHeight();
 
+    // Get original pixels for blending
+    int[] originalPixels = new int[width * height];
+    mBitmap.getPixels(originalPixels, 0, width, 0, 0, width, height);
+
     int[] intValues = new int[width * height];
     for (int j = 0; j < height; j++) {
       for (int k = 0; k < width; k++) {
-        int maxi = 0, maxj = 0, maxk = 0;
+        int maxi = 0;
         double maxnum = -Double.MAX_VALUE;
         for (int i = 0; i < CLASSNUM; i++) {
           float score = scores[i * (width * height) + j * width + k];
           if (score > maxnum) {
             maxnum = score;
             maxi = i;
-            maxj = j;
-            maxk = k;
           }
         }
-        if (maxi == PERSON) intValues[maxj * width + maxk] = 0xFFFF0000; // R
-        else if (maxi == DOG) intValues[maxj * width + maxk] = 0xFF00FF00; // G
-        else if (maxi == SHEEP) intValues[maxj * width + maxk] = 0xFF0000FF; // B
-        else intValues[maxj * width + maxk] = 0xFF000000;
-        if (maxi == PERSON || maxi == DOG || maxi == SHEEP) {
+        int pixelIndex = j * width + k;
+        int classColor = CLASS_COLORS[maxi];
+        
+        if (maxi == 0) {
+          // Background: show original image
+          intValues[pixelIndex] = originalPixels[pixelIndex];
+        } else {
+          // Blend segmentation color with original at 50% opacity
+          intValues[pixelIndex] = blendColors(originalPixels[pixelIndex], classColor, 0.5f);
           imageSegementationSuccess = true;
         }
       }
@@ -310,12 +356,14 @@ public void run() {
     runOnUiThread(
             () -> {
               if (showUserIndicationOnImgSegFail) {
-                Toast.makeText(this, "ImageSegmentation Failed", Toast.LENGTH_SHORT).show();
+                Toast.makeText(this, "No objects detected", Toast.LENGTH_SHORT).show();
               }
               mImageView.setImageBitmap(transferredBitmap);
               mButtonXnnpack.setEnabled(true);
               mButtonXnnpack.setText(R.string.run_xnnpack);
               mProgressBar.setVisibility(ProgressBar.INVISIBLE);
+              mInferenceTimeText.setText("Inference: " + mInferenceTime + " ms");
+              mInferenceTimeText.setVisibility(View.VISIBLE);
             });
   }
 
@@ -326,4 +374,89 @@ public void run() {
       }
     });
   }
+
+  // Blend two colors with given alpha for the overlay
+  private int blendColors(int background, int foreground, float alpha) {
+    int bgR = (background >> 16) & 0xFF;
+    int bgG = (background >> 8) & 0xFF;
+    int bgB = background & 0xFF;
+    int fgR = (foreground >> 16) & 0xFF;
+    int fgG = (foreground >> 8) & 0xFF;
+    int fgB = foreground & 0xFF;
+    int r = (int) (bgR * (1 - alpha) + fgR * alpha);
+    int g = (int) (bgG * (1 - alpha) + fgG * alpha);
+    int b = (int) (bgB * (1 - alpha) + fgB * alpha);
+    return 0xFF000000 | (r << 16) | (g << 8) | b;
+  }
+
+  private void loadModelOrShowDownloadButton() {
+    File modelFile = new File(MODEL_PATH);
+    if (modelFile.exists()) {
+      try {
+        mModule = Module.load(MODEL_PATH);
+        mButtonXnnpack.setEnabled(true);
+        mDownloadModelButton.setVisibility(View.GONE);
+        mModelStatusText.setText("Model loaded");
+        mModelStatusText.setVisibility(View.VISIBLE);
+      } catch (Exception e) {
+        Log.e("MainActivity", "Failed to load model", e);
+        showUIMessage(this, "Failed to load model: " + e.getMessage());
+        mButtonXnnpack.setEnabled(false);
+        mDownloadModelButton.setVisibility(View.VISIBLE);
+        mModelStatusText.setText("Model load failed");
+        mModelStatusText.setVisibility(View.VISIBLE);
+      }
+    } else {
+      mButtonXnnpack.setEnabled(false);
+      mDownloadModelButton.setVisibility(View.VISIBLE);
+      mModelStatusText.setText("Model not found");
+      mModelStatusText.setVisibility(View.VISIBLE);
+    }
+  }
+
+  private void downloadModel() {
+    mDownloadModelButton.setEnabled(false);
+    mDownloadModelButton.setText(R.string.downloading);
+    mProgressBar.setVisibility(View.VISIBLE);
+    mModelStatusText.setText("Downloading...");
+
+    new Thread(() -> {
+      try {
+        URL url = new URL(MODEL_URL);
+        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+        connection.setRequestMethod("GET");
+        connection.connect();
+
+        if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
+          throw new IOException("Server returned HTTP " + connection.getResponseCode());
+        }
+
+        File outputFile = new File(MODEL_PATH);
+        try (InputStream input = connection.getInputStream();
+             FileOutputStream output = new FileOutputStream(outputFile)) {
+          byte[] buffer = new byte[4096];
+          int bytesRead;
+          while ((bytesRead = input.read(buffer)) != -1) {
+            output.write(buffer, 0, bytesRead);
+          }
+        }
+
+        runOnUiThread(() -> {
+          mDownloadModelButton.setText(R.string.download_model);
+          mProgressBar.setVisibility(View.INVISIBLE);
+          loadModelOrShowDownloadButton();
+          showUIMessage(this, "Model downloaded successfully!");
+        });
+      } catch (Exception e) {
+        Log.e("MainActivity", "Failed to download model", e);
+        runOnUiThread(() -> {
+          mDownloadModelButton.setEnabled(true);
+          mDownloadModelButton.setText(R.string.download_model);
+          mProgressBar.setVisibility(View.INVISIBLE);
+          mModelStatusText.setText("Download failed");
+          showUIMessage(this, "Download failed: " + e.getMessage());
+        });
+      }
+    }).start();
+  }
 }
diff --git a/dl3/android/DeepLabV3Demo/app/src/main/java/org/pytorch/executorchexamples/dl3/TensorImageUtils.java b/dl3/android/DeepLabV3Demo/app/src/main/java/org/pytorch/executorchexamples/dl3/TensorImageUtils.java
@@ -70,10 +70,6 @@ public static void bitmapToFloatBuffer(
     bitmap.getPixels(pixels, 0, width, x, y, width, height);
     final int offset_g = pixelsCount;
     final int offset_b = 2 * pixelsCount;
-    for (int i = 0; i < 100; i++) {
-      final int c = pixels[i];
-      Log.i("Image", ": " + i + " " + ((c >> 16) & 0xff));
-    }
     for (int i = 0; i < pixelsCount; i++) {
       final int c = pixels[i];
       float r = ((c >> 16) & 0xff) / 255.0f;
diff --git a/dl3/android/DeepLabV3Demo/app/src/main/res/layout/activity_main.xml b/dl3/android/DeepLabV3Demo/app/src/main/res/layout/activity_main.xml
@@ -30,14 +30,28 @@
         app:layout_constraintStart_toStartOf="parent"
         app:layout_constraintEnd_toEndOf="parent" />
 
+    <!-- Inference Time Display -->
+    <TextView
+        android:id="@+id/inferenceTimeText"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:text="@string/inference_time_placeholder"
+        android:textSize="16sp"
+        android:textStyle="bold"
+        android:textColor="#2196F3"
+        android:visibility="invisible"
+        app:layout_constraintTop_toBottomOf="@+id/progressBar"
+        app:layout_constraintStart_toStartOf="parent"
+        app:layout_constraintEnd_toEndOf="parent" />
+
     <!-- Row 1: Next and Reset buttons side by side -->
     <Button
         android:id="@+id/nextButton"
         android:layout_width="0dp"
         android:layout_height="wrap_content"
         android:text="@string/next"
         android:textAllCaps="false"
-        app:layout_constraintTop_toBottomOf="@+id/progressBar"
+        app:layout_constraintTop_toBottomOf="@+id/inferenceTimeText"
         app:layout_constraintStart_toStartOf="parent"
         app:layout_constraintEnd_toStartOf="@+id/resetImage"
         app:layout_constraintHorizontal_weight="1" />
@@ -48,7 +62,7 @@
         android:layout_height="wrap_content"
         android:text="@string/reset"
         android:textAllCaps="false"
-        app:layout_constraintTop_toBottomOf="@+id/progressBar"
+        app:layout_constraintTop_toBottomOf="@+id/inferenceTimeText"
         app:layout_constraintStart_toEndOf="@+id/nextButton"
         app:layout_constraintEnd_toEndOf="parent"
         app:layout_constraintHorizontal_weight="1" />
diff --git a/dl3/android/DeepLabV3Demo/app/src/main/res/values/strings.xml b/dl3/android/DeepLabV3Demo/app/src/main/res/values/strings.xml
@@ -11,4 +11,5 @@
     <string name="reset">Reset</string>
     <string name="load_and_refresh">Load And Refresh</string>
     <string name="run">Run</string>
+    <string name="inference_time_placeholder">Inference: -- ms</string>
 </resources>
