Merge branch 'main' into urm-memory-reduction

Author: tnidigan

.github/workflows/qcom-preflight-checks.yml

@@ -12,13 +12,7 @@ permissions:
 
 jobs:
   qcom-preflight-checks:
-    uses: qualcomm/qcom-reusable-workflows/.github/workflows/qcom-preflight-checks-reusable-workflow.yml@v1.1.4
+    uses: qualcomm/qcom-reusable-workflows/.github/workflows/reusable-qcom-preflight-checks-orchestrator.yml@v2
     with:
-        # ✅ Preflight Checkers
-        repolinter: true                   # default: true
-        semgrep: true                      # default: true
-        copyright-license-detector: true   # default: true
-        pr-check-emails: true              # default: true
-        dependency-review: true            # default: true
-    secrets:
-      SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }}
+        enable-armor-checkers: true
+        armor-checker-options: '{"build-script":"ci/build.sh","runs-on":{"group":"GHA-CSEPerf-prd-SelfHosted-RG","labels":["self-hosted", "cseperf-prd-u2204-arm64-xlrg-od-ephem"]}}'

.github/workflows/run_armor.yml

@@ -0,0 +1,22 @@
+#!/usr/bin/bash
+
+# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+# SPDX-License-Identifier: BSD-3-Clause-Clear
+
+set -euo pipefail
+
+echo "Installing required packages"
+sudo apt-get update -y
+sudo apt-get install -y cmake pkg-config libyaml-dev libsystemd-dev
+
+echo "Installing optional packages (fasttext)"
+sudo apt-get install -y fasttext libfasttext-dev || {
+    echo "Warning: optional fasttext packages could not be installed."
+}
+
+echo "Running CMake configure"
+mkdir -p build
+cd build
+cmake .. -DBUILD_TESTS=ON -DBUILD_CLASSIFIER=OFF -DCMAKE_INSTALL_PREFIX="./urm-bins/"
+cmake --build .
+sudo make install

ci/build.sh

@@ -39,4 +39,4 @@ PropertyConfigs:
     Value: "SYSLOG"
 
   - Name: urm.extensions_lib.count
-    Value: "3"
+    Value: "6"

configs/PropertiesConfig.yaml

@@ -398,3 +398,12 @@ ResourceConfigs:
     Modes: ["display_on"]
     Policy: "pass_through"
     ApplyType: "global"
+
+  - ResType: "0x0c"
+    ResID: "0x0000"
+    Name: "RES_PER_TASK_AFFINITY"
+    Supported: true
+    Permissions: "third_party"
+    Modes: ["display_on"]
+    Policy: "pass_through"
+    ApplyType: "global"

configs/ResourcesConfig.yaml

@@ -6,6 +6,7 @@ include_directories(${CMAKE_SOURCE_DIR}/modula/CoreModules/Include)
 include_directories(${CMAKE_SOURCE_DIR}/extensions/Include)
 include_directories(${CMAKE_SOURCE_DIR}/resource-tuner/core/Include)
 include_directories(${CMAKE_SOURCE_DIR}/resource-tuner/signals/Include/)
+include_directories(${CMAKE_SOURCE_DIR}/resource-tuner/dbus-modules/Include/)
 
 # Single shared library that contains parser + classifier + netlink glue
 add_library(ContextualClassifier SHARED
@@ -22,60 +23,61 @@ target_include_directories(ContextualClassifier
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/Include
 )
 
-# fastText detection and USE_FASTTEXT macro
-option(ENABLE_FASTTEXT "Build ML inference with fastText support" ON)
+# floret detection and USE_FLORET macro using pkg-config
+option(ENABLE_FLORET "Build ML inference with floret support" ON)
 
-set(FASTTEXT_FOUND FALSE)
-set(FASTTEXT_TARGET "")
+set(FLORET_FOUND FALSE)
+set(FLORET_LIBRARIES "")
+set(FLORET_INCLUDE_DIRS "")
 
-if(ENABLE_FASTTEXT)
-    # First try a CMake package
-    find_package(fasttext QUIET)
-
-    if(TARGET fasttext::fasttext)
-        set(FASTTEXT_FOUND TRUE)
-        set(FASTTEXT_TARGET fasttext::fasttext)
-    elseif(TARGET fasttext)
-        set(FASTTEXT_FOUND TRUE)
-        set(FASTTEXT_TARGET fasttext)
-    else()
-        # Fallback: probe the linker directly for -lfasttext
-        include(CheckLibraryExists)
-        check_library_exists(fasttext fasttext_version "" FASTTEXT_LINKABLE)
-        if(FASTTEXT_LINKABLE)
-            set(FASTTEXT_FOUND TRUE)
-            set(FASTTEXT_TARGET fasttext)
-        endif()
+if(ENABLE_FLORET)
+    # Use pkg-config to find floret
+    pkg_check_modules(PC_FLORET QUIET floret)
+    if(PC_FLORET_FOUND)
+        message(STATUS "floret not found via pkg-config")
+        set(FLORET_FOUND TRUE)
+        set(FLORET_LIBRARIES ${PC_FLORET_LIBRARIES})
+        set(FLORET_INCLUDE_DIRS ${PC_FLORET_INCLUDE_DIRS})
     endif()
 endif()
 
-if(FASTTEXT_FOUND)
-    message(STATUS "fastText found, building MLInference with USE_FASTTEXT=1")
-    add_definitions(-DUSE_FASTTEXT=1)
+if(FLORET_FOUND)
+    message(STATUS "floret found via pkg-config, building MLInference with USE_FLORET=1")
+    add_definitions(-DUSE_FLORET=1)
 
-    # Define the ML inference library that actually uses fastText
-    add_library(ml_inference_lib STATIC
+    # Define the ML inference library that actually uses FLORET
+    add_library(ml_inference_lib SHARED
         FeatureExtractor.cpp
         FeaturePruner.cpp
         MLInference.cpp
     )
 
+    set_target_properties(ml_inference_lib PROPERTIES
+        VERSION 1.0.0
+        SOVERSION 1
+    )
+
+    target_link_libraries(ContextualClassifier PRIVATE ml_inference_lib)
+
     target_link_libraries(ml_inference_lib
-        PRIVATE ${FASTTEXT_TARGET}
+        PRIVATE
+            ${FLORET_LIBRARIES}
     )
 
     target_include_directories(ml_inference_lib
-        PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/Include
+        PUBLIC
+            ${CMAKE_CURRENT_SOURCE_DIR}/Include
+            ${FLORET_INCLUDE_DIRS}
     )
 else()
-    message(STATUS "fastText not found or ENABLE_FASTTEXT=OFF — falling back to base Inference only")
+    message(STATUS "floret not found via pkg-config or ENABLE_FLORET=OFF — falling back to base Inference only")
 endif()
 
 # Installation rules
-install(TARGETS ContextualClassifier DESTINATION ${CMAKE_INSTALL_LIBDIR})
-if(FASTTEXT_FOUND)
-    install(TARGETS ml_inference_lib DESTINATION ${CMAKE_INSTALL_LIBDIR})
-    install(FILES ${CMAKE_SOURCE_DIR}/contextual-classifier/Artifacts/fasttext_model_supervised.bin DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/urm/classifier)
+install(TARGETS ContextualClassifier DESTINATION lib)
+if(FLORET_FOUND)
+    install(TARGETS ml_inference_lib DESTINATION lib)
+    install(FILES ${CMAKE_SOURCE_DIR}/contextual-classifier/Artifacts/floret_model_supervised.bin DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/urm/classifier)
 endif()
 
 install(

…/Artifacts/fasttext_model_supervised.bin …er/Artifacts/floret_model_supervised.bincontextual-classifier/Artifacts/fasttext_model_supervised.bin renamed to contextual-classifier/Artifacts/floret_model_supervised.bin contextual-classifier/Artifacts/fasttext_model_supervised.bin renamed to contextual-classifier/Artifacts/floret_model_supervised.bin

@@ -27,19 +27,16 @@
 #define CLASSIFIER_TAG "CONTEXTUAL_CLASSIFIER"
 #define CLASSIFIER_CONFIGS_DIR "/etc/urm/classifier/"
 
-#define CURR_RESTUNE_CGRP_HNDL 0
-#define CURR_RESTUNE_SIG_HNDL 1
-
-static const std::string FT_MODEL_PATH =
-    CLASSIFIER_CONFIGS_DIR "fasttext_model_supervised.bin";
-static const std::string IGNORE_PROC_PATH =
+const std::string FT_MODEL_PATH =
+    CLASSIFIER_CONFIGS_DIR "floret_model_supervised.bin";
+const std::string IGNORE_PROC_PATH =
     CLASSIFIER_CONFIGS_DIR "classifier-blocklist.txt";
 static const std::string IGNORE_TOKENS_PATH =
     CLASSIFIER_CONFIGS_DIR "ignore-tokens.txt";
 static const std::string ALLOW_LIST_PATH =
     CLASSIFIER_CONFIGS_DIR "allow-list.txt";
 
-#ifdef USE_FASTTEXT
+#ifdef USE_FLORET
 #include "MLInference.h"
 #include "FeatureExtractor.h"
 #include "FeaturePruner.h"
@@ -60,17 +57,15 @@ static ContextualClassifier *gClassifier = nullptr;
 static const int32_t pendingQueueControlSize = 30;
 
 ContextualClassifier::ContextualClassifier() {
-    this->mRestuneHandles[CURR_RESTUNE_CGRP_HNDL].mCurHandle = -1;
-    this->mRestuneHandles[CURR_RESTUNE_SIG_HNDL].mCurHandle = -1;
     mInference = GetInferenceObject();
 }
 
-void ContextualClassifier::untuneRequestHelper(int32_t index) {
+void ContextualClassifier::untuneRequestHelper(int64_t handle) {
     try {
         Request* untuneRequest = MPLACED(Request);
 
         untuneRequest->setRequestType(REQ_RESOURCE_UNTUNING);
-        untuneRequest->setHandle(this->mRestuneHandles[index].mCurHandle);
+        untuneRequest->setHandle(handle);
         untuneRequest->setDuration(-1);
         // Passing priority as HIGH_TRANSFER_PRIORITY (= -1)
         // - Ensures untune requests are processed before even SERVER_HIGH priority tune requests
@@ -79,8 +74,8 @@ void ContextualClassifier::untuneRequestHelper(int32_t index) {
         //   not free up the underlying Request object, allowing for reuse.
         // Priority Level: -2 is used to force server termination and cleanup so should not be used otherwise.
         untuneRequest->setPriority(SYSTEM_HIGH);
-        untuneRequest->setClientPID(this->mRestuneHandles[index].mCurReqPid);
-        untuneRequest->setClientTID(this->mRestuneHandles[index].mCurReqTid);
+        untuneRequest->setClientPID(0);
+        untuneRequest->setClientTID(0);
 
         // fast path to Request Queue
         // Mark verification status as true. Request still goes through RequestManager though.
@@ -257,6 +252,15 @@ void ContextualClassifier::ClassifierMain() {
                 // For example: game, browser, multimedia
                 sigId = this->GetSignalIDForWorkload(contextType);
 
+                // Step 2:
+                // Untune any Configurations from the last proc-invocation
+                for(int64_t handle: this->mCurrRestuneHandles) {
+                    if(handle > 0) {
+                        untuneRequestHelper(handle);
+                    }
+                }
+                this->mCurrRestuneHandles.clear();
+
                 // Step 2:
                 // - Move the process to focused-cgroup, Also involves removing the process
                 //  already there from the cgroup.
@@ -379,26 +383,17 @@ void ContextualClassifier::ApplyActions(uint32_t sigId,
                                         uint32_t sigType,
                                         pid_t incomingPID,
                                         pid_t incomingTID) {
-
-    if(this->mRestuneHandles[CURR_RESTUNE_SIG_HNDL].mCurHandle != -1) {
-        untuneRequestHelper(CURR_RESTUNE_SIG_HNDL);
-        mRestuneHandles[CURR_RESTUNE_SIG_HNDL].mCurHandle = -1;
-    }
-
     Request* request = createTuneRequestFromSignal(sigId, sigType, incomingPID, incomingTID);
     if(request != nullptr) {
         if(request->getResourcesCount() > 0) {
             // Record:
-            this->mRestuneHandles[CURR_RESTUNE_SIG_HNDL].mCurHandle = request->getHandle();
-            this->mRestuneHandles[CURR_RESTUNE_SIG_HNDL].mCurReqPid = incomingPID;
-            this->mRestuneHandles[CURR_RESTUNE_SIG_HNDL].mCurReqTid = incomingTID;
+            this->mCurrRestuneHandles.push_back(request->getHandle());
 
             // fast path to Request Queue
             submitResProvisionRequest(request, true);
 
         } else {
             Request::cleanUpRequest(request);
-            this->mRestuneHandles[CURR_RESTUNE_SIG_HNDL].mCurHandle = -1;
         }
     }
 }
@@ -496,12 +491,6 @@ void ContextualClassifier::MoveAppThreadsToCGroup(pid_t incomingPID,
                                                   const std::string& comm,
                                                   int32_t cgroupIdentifier) {
     try {
-        // Check for any outstanding request, if found untune it.
-        if(this->mRestuneHandles[CURR_RESTUNE_CGRP_HNDL].mCurHandle != -1) {
-            untuneRequestHelper(CURR_RESTUNE_CGRP_HNDL);
-            mRestuneHandles[CURR_RESTUNE_CGRP_HNDL].mCurHandle = -1;
-        }
-
         int64_t handleGenerated = -1;
         // Issue a tune request for the new pid (and any associated app-config pids)
         Request* request = MPLACED(Request);
@@ -537,22 +526,42 @@ void ContextualClassifier::MoveAppThreadsToCGroup(pid_t incomingPID,
         // Anything to issue
         if(request->getResourcesCount() > 0) {
             // Record:
-            this->mRestuneHandles[CURR_RESTUNE_CGRP_HNDL].mCurHandle = handleGenerated;
-            this->mRestuneHandles[CURR_RESTUNE_CGRP_HNDL].mCurReqPid = incomingPID;
-            this->mRestuneHandles[CURR_RESTUNE_CGRP_HNDL].mCurReqTid = incomingTID;
+            this->mCurrRestuneHandles.push_back(request->getHandle());
 
             // fast path to Request Queue
             submitResProvisionRequest(request, true);
 
         } else {
             Request::cleanUpRequest(request);
-            this->mRestuneHandles[CURR_RESTUNE_CGRP_HNDL].mCurHandle = -1;
+        }
+
+        // Configure any associated signal
+        if(appConfig != nullptr && appConfig->mSignalCodes != nullptr) {
+            int32_t numSignals = appConfig->mNumSignals;
+            // Go over the list of proc names (comm) and get their pids
+            for(int32_t i = 0; i < numSignals; i++) {
+                Request* request = createTuneRequestFromSignal(
+                    appConfig->mSignalCodes[i],
+                    0,
+                    incomingPID,
+                    incomingTID);
+
+                if(request != nullptr) {
+                    if(request->getResourcesCount() > 0) {
+                        // fast path to Request Queue
+                        this->mCurrRestuneHandles.push_back(request->getHandle());
+                        submitResProvisionRequest(request, true);
+
+                    } else {
+                        Request::cleanUpRequest(request);
+                    }
+                }
+            }
         }
 
     } catch(const std::exception& e) {
         LOGE(CLASSIFIER_TAG,
              "Failed to move per-app threads to cgroup, Error: " + std::string(e.what()));
-        this->mRestuneHandles[CURR_RESTUNE_CGRP_HNDL].mCurHandle = -1;
     }
 }