feat: Add GitHub Actions workflow for model export and enhance DeepLabV3 Android demo with improved segmentation visualization and inference time display.

Author: kirklandsign

.github/workflows/export-models.yml

@@ -41,6 +41,9 @@ jobs:
 
     name: Export ${{ matrix.name }}
 
+  export-dl3:
+    runs-on: ubuntu-latest
+    name: Export DeepLabV3 Model
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -65,3 +68,14 @@ jobs:
           name: ${{ matrix.artifact }}
           path: ${{ matrix.output }}
           if-no-files-found: error
+        run: pip install executorch torchvision
+
+      - name: Export DL3 model
+        working-directory: dl3/python
+        run: python export.py
+
+      - name: Upload PTE model
+        uses: actions/upload-artifact@v4
+        with:
+          name: dl3_xnnpack_fp32.pte
+          path: dl3/python/dl3_xnnpack_fp32.pte

dl3/android/DeepLabV3Demo/app/src/main/java/org/pytorch/executorchexamples/dl3/MainActivity.java

@@ -43,9 +43,11 @@ public class MainActivity extends Activity implements Runnable {
   private ImageView mImageView;
   private Button mButtonXnnpack;
   private ProgressBar mProgressBar;
+  private android.widget.TextView mInferenceTimeText;
   private Bitmap mBitmap = null;
   private Module mModule = null;
   private String mImagename = "corgi.jpeg";
+  private long mInferenceTime = 0;
 
   private final ArrayList<String> mImageFiles = new ArrayList<>();
 
@@ -57,9 +59,31 @@ public class MainActivity extends Activity implements Runnable {
   // see http://host.robots.ox.ac.uk:8080/pascal/VOC/voc2007/segexamples/index.html for the list of
   // classes with indexes
   private static final int CLASSNUM = 21;
-  private static final int DOG = 12;
-  private static final int PERSON = 15;
-  private static final int SHEEP = 17;
+
+  // Colors for all 21 PASCAL VOC classes
+  private static final int[] CLASS_COLORS = {
+      0x00000000, // 0: Background (transparent)
+      0xFFE6194B, // 1: Aeroplane (red)
+      0xFF3CB44B, // 2: Bicycle (green)
+      0xFFFFE119, // 3: Bird (yellow)
+      0xFF4363D8, // 4: Boat (blue)
+      0xFFF58231, // 5: Bottle (orange)
+      0xFF911EB4, // 6: Bus (purple)
+      0xFF46F0F0, // 7: Car (cyan)
+      0xFFF032E6, // 8: Cat (magenta)
+      0xFFBCF60C, // 9: Chair (lime)
+      0xFFFABEBE, // 10: Cow (pink)
+      0xFF008080, // 11: Dining Table (teal)
+      0xFF00FF00, // 12: Dog (bright green)
+      0xFF9A6324, // 13: Horse (brown)
+      0xFFFFD8B1, // 14: Motorbike (peach)
+      0xFFFF0000, // 15: Person (red)
+      0xFF800000, // 16: Potted Plant (maroon)
+      0xFF0000FF, // 17: Sheep (blue)
+      0xFF808000, // 18: Sofa (olive)
+      0xFFE6BEFF, // 19: Train (lavender)
+      0xFFAA6E28, // 20: TV/Monitor (tan)
+  };
 
   private void checkAndRequestStoragePermission() {
     if (ContextCompat.checkSelfPermission(this, Manifest.permission.READ_EXTERNAL_STORAGE)
@@ -198,6 +222,7 @@ protected void onCreate(Bundle savedInstanceState) {
     mImageView = findViewById(R.id.imageView);
     mButtonXnnpack = findViewById(R.id.xnnpackButton);
     mProgressBar = findViewById(R.id.progressBar);
+    mInferenceTimeText = findViewById(R.id.inferenceTimeText);
 
     populateImagePathFromAssets();
     showImage();
@@ -223,10 +248,9 @@ public void onClick(View v) {
     mButtonXnnpack.setOnClickListener(
         new View.OnClickListener() {
           public void onClick(View v) {
-            mModule.destroy();
-            mModule = Module.load("/data/local/tmp/dl3_xnnpack_fp32.pte");
             mButtonXnnpack.setEnabled(false);
             mProgressBar.setVisibility(ProgressBar.VISIBLE);
+            mInferenceTimeText.setVisibility(View.INVISIBLE);
             mButtonXnnpack.setText(getString(R.string.run_model));
 
             Thread thread = new Thread(MainActivity.this);
@@ -262,32 +286,38 @@ public void run() {
     boolean imageSegementationSuccess = false;
     final long startTime = SystemClock.elapsedRealtime();
     Tensor outputTensor = mModule.forward(EValue.from(inputTensor))[0].toTensor();
-    final long inferenceTime = SystemClock.elapsedRealtime() - startTime;
-    Log.d("ImageSegmentation", "inference time (ms): " + inferenceTime);
+    mInferenceTime = SystemClock.elapsedRealtime() - startTime;
+    Log.d("ImageSegmentation", "inference time (ms): " + mInferenceTime);
 
     final float[] scores = outputTensor.getDataAsFloatArray();
     int width = mBitmap.getWidth();
     int height = mBitmap.getHeight();
 
+    // Get original pixels for blending
+    int[] originalPixels = new int[width * height];
+    mBitmap.getPixels(originalPixels, 0, width, 0, 0, width, height);
+
     int[] intValues = new int[width * height];
     for (int j = 0; j < height; j++) {
       for (int k = 0; k < width; k++) {
-        int maxi = 0, maxj = 0, maxk = 0;
+        int maxi = 0;
         double maxnum = -Double.MAX_VALUE;
         for (int i = 0; i < CLASSNUM; i++) {
           float score = scores[i * (width * height) + j * width + k];
           if (score > maxnum) {
             maxnum = score;
             maxi = i;
-            maxj = j;
-            maxk = k;
           }
         }
-        if (maxi == PERSON) intValues[maxj * width + maxk] = 0xFFFF0000; // R
-        else if (maxi == DOG) intValues[maxj * width + maxk] = 0xFF00FF00; // G
-        else if (maxi == SHEEP) intValues[maxj * width + maxk] = 0xFF0000FF; // B
-        else intValues[maxj * width + maxk] = 0xFF000000;
-        if (maxi == PERSON || maxi == DOG || maxi == SHEEP) {
+        int pixelIndex = j * width + k;
+        int classColor = CLASS_COLORS[maxi];
+        
+        if (maxi == 0) {
+          // Background: show original image
+          intValues[pixelIndex] = originalPixels[pixelIndex];
+        } else {
+          // Blend segmentation color with original at 50% opacity
+          intValues[pixelIndex] = blendColors(originalPixels[pixelIndex], classColor, 0.5f);
           imageSegementationSuccess = true;
         }
       }
@@ -310,12 +340,14 @@ public void run() {
     runOnUiThread(
             () -> {
               if (showUserIndicationOnImgSegFail) {
-                Toast.makeText(this, "ImageSegmentation Failed", Toast.LENGTH_SHORT).show();
+                Toast.makeText(this, "No objects detected", Toast.LENGTH_SHORT).show();
               }
               mImageView.setImageBitmap(transferredBitmap);
               mButtonXnnpack.setEnabled(true);
               mButtonXnnpack.setText(R.string.run_xnnpack);
               mProgressBar.setVisibility(ProgressBar.INVISIBLE);
+              mInferenceTimeText.setText("Inference: " + mInferenceTime + " ms");
+              mInferenceTimeText.setVisibility(View.VISIBLE);
             });
   }
 
@@ -326,4 +358,18 @@ public void run() {
       }
     });
   }
+
+  // Blend two colors with given alpha for the overlay
+  private int blendColors(int background, int foreground, float alpha) {
+    int bgR = (background >> 16) & 0xFF;
+    int bgG = (background >> 8) & 0xFF;
+    int bgB = background & 0xFF;
+    int fgR = (foreground >> 16) & 0xFF;
+    int fgG = (foreground >> 8) & 0xFF;
+    int fgB = foreground & 0xFF;
+    int r = (int) (bgR * (1 - alpha) + fgR * alpha);
+    int g = (int) (bgG * (1 - alpha) + fgG * alpha);
+    int b = (int) (bgB * (1 - alpha) + fgB * alpha);
+    return 0xFF000000 | (r << 16) | (g << 8) | b;
+  }
 }

dl3/android/DeepLabV3Demo/app/src/main/java/org/pytorch/executorchexamples/dl3/TensorImageUtils.java

@@ -70,10 +70,6 @@ public static void bitmapToFloatBuffer(
     bitmap.getPixels(pixels, 0, width, x, y, width, height);
     final int offset_g = pixelsCount;
     final int offset_b = 2 * pixelsCount;
-    for (int i = 0; i < 100; i++) {
-      final int c = pixels[i];
-      Log.i("Image", ": " + i + " " + ((c >> 16) & 0xff));
-    }
     for (int i = 0; i < pixelsCount; i++) {
       final int c = pixels[i];
       float r = ((c >> 16) & 0xff) / 255.0f;

dl3/android/DeepLabV3Demo/app/src/main/res/layout/activity_main.xml

@@ -30,14 +30,28 @@
         app:layout_constraintStart_toStartOf="parent"
         app:layout_constraintEnd_toEndOf="parent" />
 
+    <!-- Inference Time Display -->
+    <TextView
+        android:id="@+id/inferenceTimeText"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:text="@string/inference_time_placeholder"
+        android:textSize="16sp"
+        android:textStyle="bold"
+        android:textColor="#2196F3"
+        android:visibility="invisible"
+        app:layout_constraintTop_toBottomOf="@+id/progressBar"
+        app:layout_constraintStart_toStartOf="parent"
+        app:layout_constraintEnd_toEndOf="parent" />
+
     <!-- Row 1: Next and Reset buttons side by side -->
     <Button
         android:id="@+id/nextButton"
         android:layout_width="0dp"
         android:layout_height="wrap_content"
         android:text="@string/next"
         android:textAllCaps="false"
-        app:layout_constraintTop_toBottomOf="@+id/progressBar"
+        app:layout_constraintTop_toBottomOf="@+id/inferenceTimeText"
         app:layout_constraintStart_toStartOf="parent"
         app:layout_constraintEnd_toStartOf="@+id/resetImage"
         app:layout_constraintHorizontal_weight="1" />
@@ -48,7 +62,7 @@
         android:layout_height="wrap_content"
         android:text="@string/reset"
         android:textAllCaps="false"
-        app:layout_constraintTop_toBottomOf="@+id/progressBar"
+        app:layout_constraintTop_toBottomOf="@+id/inferenceTimeText"
         app:layout_constraintStart_toEndOf="@+id/nextButton"
         app:layout_constraintEnd_toEndOf="parent"
         app:layout_constraintHorizontal_weight="1" />

dl3/android/DeepLabV3Demo/app/src/main/res/values/strings.xml

@@ -11,4 +11,5 @@
     <string name="reset">Reset</string>
     <string name="load_and_refresh">Load And Refresh</string>
     <string name="run">Run</string>
+    <string name="inference_time_placeholder">Inference: -- ms</string>
 </resources>