|
| 1 | +#import "ImageSegmentationModel.h" |
| 2 | +#import <unordered_set> |
| 3 | +#import <algorithm> |
| 4 | +#import <vector> |
| 5 | +#import "../../utils/ImageProcessor.h" |
| 6 | +#import "../../utils/Numerical.h" |
| 7 | +#import "../../utils/Conversions.h" |
| 8 | +#import "opencv2/opencv.hpp" |
| 9 | +#import "Constants.h" |
| 10 | + |
| 11 | +@interface ImageSegmentationModel () |
| 12 | + - (NSArray *)preprocess:(cv::Mat &)input; |
| 13 | + - (NSDictionary *)postprocess:(NSArray *)output |
| 14 | + returnClasses:(NSArray *)classesOfInterest |
| 15 | + resize:(BOOL)resize; |
| 16 | +@end |
| 17 | + |
| 18 | +@implementation ImageSegmentationModel { |
| 19 | + cv::Size originalSize; |
| 20 | +} |
| 21 | + |
| 22 | +- (cv::Size)getModelImageSize { |
| 23 | + NSArray *inputShape = [module getInputShape:@0]; |
| 24 | + NSNumber *widthNumber = inputShape.lastObject; |
| 25 | + NSNumber *heightNumber = inputShape[inputShape.count - 2]; |
| 26 | + |
| 27 | + int height = [heightNumber intValue]; |
| 28 | + int width = [widthNumber intValue]; |
| 29 | + |
| 30 | + return cv::Size(height, width); |
| 31 | +} |
| 32 | + |
| 33 | +- (NSArray *)preprocess:(cv::Mat &)input { |
| 34 | + originalSize = cv::Size(input.cols, input.rows); |
| 35 | + |
| 36 | + cv::Size modelImageSize = [self getModelImageSize]; |
| 37 | + cv::Mat output; |
| 38 | + cv::resize(input, output, modelImageSize); |
| 39 | + |
| 40 | + NSArray *modelInput = [ImageProcessor matToNSArray:output]; |
| 41 | + return modelInput; |
| 42 | +} |
| 43 | + |
| 44 | +std::vector<cv::Mat> extractResults(NSArray *result, std::size_t numLabels, |
| 45 | + cv::Size modelImageSize, cv::Size originalSize, BOOL resize) { |
| 46 | + std::size_t numModelPixels = modelImageSize.height * modelImageSize.width; |
| 47 | + |
| 48 | + std::vector<cv::Mat> resizedLabelScores(numLabels); |
| 49 | + for (std::size_t label = 0; label < numLabels; ++label) { |
| 50 | + cv::Mat labelMat = cv::Mat(modelImageSize, CV_64F); |
| 51 | + |
| 52 | + for(std::size_t pixel = 0; pixel < numModelPixels; ++pixel){ |
| 53 | + int row = pixel / modelImageSize.width; |
| 54 | + int col = pixel % modelImageSize.width; |
| 55 | + labelMat.at<double>(row, col) = [result[label * numModelPixels + pixel] doubleValue]; |
| 56 | + } |
| 57 | + |
| 58 | + if (resize) { |
| 59 | + cv::resize(labelMat, resizedLabelScores[label], originalSize); |
| 60 | + } |
| 61 | + else { |
| 62 | + resizedLabelScores[label] = std::move(labelMat); |
| 63 | + } |
| 64 | + } |
| 65 | + return resizedLabelScores; |
| 66 | +} |
| 67 | + |
| 68 | +void adjustScoresPerPixel(std::vector<cv::Mat>& labelScores, cv::Mat& argMax, |
| 69 | + cv::Size outputSize, std::size_t numLabels) { |
| 70 | + std::size_t numOutputPixels = outputSize.height * outputSize.width; |
| 71 | + for (std::size_t pixel = 0; pixel < numOutputPixels; ++pixel) { |
| 72 | + int row = pixel / outputSize.width; |
| 73 | + int col = pixel % outputSize.width; |
| 74 | + std::vector<double> scores; |
| 75 | + scores.reserve(numLabels); |
| 76 | + for (const auto& mat : labelScores) { |
| 77 | + scores.push_back(mat.at<double>(row, col)); |
| 78 | + } |
| 79 | + |
| 80 | + std::vector<double> adjustedScores = softmax(scores); |
| 81 | + |
| 82 | + for (std::size_t label = 0; label < numLabels; ++label) { |
| 83 | + labelScores[label].at<double>(row, col) = adjustedScores[label]; |
| 84 | + } |
| 85 | + |
| 86 | + auto maxIt = std::max_element(scores.begin(), scores.end()); |
| 87 | + argMax.at<int>(row, col) = std::distance(scores.begin(), maxIt); |
| 88 | + } |
| 89 | +} |
| 90 | + |
| 91 | +- (NSDictionary *)postprocess:(NSArray *)output |
| 92 | + returnClasses:(NSArray *)classesOfInterest |
| 93 | + resize:(BOOL)resize { |
| 94 | + cv::Size modelImageSize = [self getModelImageSize]; |
| 95 | + |
| 96 | + std::size_t numLabels = deeplabv3_resnet50_labels.size(); |
| 97 | + |
| 98 | + NSAssert((std::size_t)output.count == numLabels * modelImageSize.height * modelImageSize.width, |
| 99 | + @"Model generated unexpected output size."); |
| 100 | + |
| 101 | + // For each label extract it's matrix, |
| 102 | + // and rescale it to the original size if `resize` |
| 103 | + std::vector<cv::Mat> resizedLabelScores = |
| 104 | + extractResults(output, numLabels, modelImageSize, originalSize, resize); |
| 105 | + |
| 106 | + cv::Size outputSize = resize ? originalSize : modelImageSize; |
| 107 | + cv::Mat argMax = cv::Mat(outputSize, CV_32S); |
| 108 | + |
| 109 | + // For each pixel apply softmax across all the labels and calculate the argMax |
| 110 | + adjustScoresPerPixel(resizedLabelScores, argMax, outputSize, numLabels); |
| 111 | + |
| 112 | + std::unordered_set<std::string> labelSet; |
| 113 | + |
| 114 | + for (id label in classesOfInterest) { |
| 115 | + labelSet.insert(std::string([label UTF8String])); |
| 116 | + } |
| 117 | + |
| 118 | + NSMutableDictionary *result = [NSMutableDictionary dictionary]; |
| 119 | + |
| 120 | + // Convert to NSArray and populate the final dictionary |
| 121 | + for (std::size_t label = 0; label < numLabels; ++label) { |
| 122 | + if (labelSet.contains(deeplabv3_resnet50_labels[label])){ |
| 123 | + NSString *labelString = @(deeplabv3_resnet50_labels[label].c_str()); |
| 124 | + NSArray *arr = simpleMatToNSArray<double>(resizedLabelScores[label]); |
| 125 | + result[labelString] = arr; |
| 126 | + } |
| 127 | + } |
| 128 | + |
| 129 | + result[@"ARGMAX"] = simpleMatToNSArray<int>(argMax); |
| 130 | + |
| 131 | + return result; |
| 132 | +} |
| 133 | + |
| 134 | +- (NSDictionary *)runModel:(cv::Mat &)input |
| 135 | + returnClasses:(NSArray *)classesOfInterest |
| 136 | + resize:(BOOL)resize { |
| 137 | + NSArray *modelInput = [self preprocess:input]; |
| 138 | + NSArray *result = [self forward:modelInput]; |
| 139 | + |
| 140 | + NSDictionary *output = [self postprocess:result[0] |
| 141 | + returnClasses:classesOfInterest |
| 142 | + resize:resize]; |
| 143 | + |
| 144 | + return output; |
| 145 | +} |
| 146 | + |
| 147 | +@end |
0 commit comments