Use floret instead of fastText (#157)

fastText is EOL, hence picking up floret, which is forked from fastText and
having active developements.

Signed-off-by: Jagadeesh Pagadala <jpagadal@qti.qualcomm.com>

Author: jpagadal

…/Artifacts/fasttext_model_supervised.bin …er/Artifacts/floret_model_supervised.bincontextual-classifier/Artifacts/fasttext_model_supervised.bin renamed to contextual-classifier/Artifacts/floret_model_supervised.bin contextual-classifier/Artifacts/fasttext_model_supervised.bin renamed to contextual-classifier/Artifacts/floret_model_supervised.bin

@@ -23,60 +23,61 @@ target_include_directories(ContextualClassifier
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/Include
 )
 
-# fastText detection and USE_FASTTEXT macro
-option(ENABLE_FASTTEXT "Build ML inference with fastText support" ON)
+# floret detection and USE_FLORET macro using pkg-config
+option(ENABLE_FLORET "Build ML inference with floret support" ON)
 
-set(FASTTEXT_FOUND FALSE)
-set(FASTTEXT_TARGET "")
+set(FLORET_FOUND FALSE)
+set(FLORET_LIBRARIES "")
+set(FLORET_INCLUDE_DIRS "")
 
-if(ENABLE_FASTTEXT)
-    # First try a CMake package
-    find_package(fasttext QUIET)
-
-    if(TARGET fasttext::fasttext)
-        set(FASTTEXT_FOUND TRUE)
-        set(FASTTEXT_TARGET fasttext::fasttext)
-    elseif(TARGET fasttext)
-        set(FASTTEXT_FOUND TRUE)
-        set(FASTTEXT_TARGET fasttext)
-    else()
-        # Fallback: probe the linker directly for -lfasttext
-        include(CheckLibraryExists)
-        check_library_exists(fasttext fasttext_version "" FASTTEXT_LINKABLE)
-        if(FASTTEXT_LINKABLE)
-            set(FASTTEXT_FOUND TRUE)
-            set(FASTTEXT_TARGET fasttext)
-        endif()
+if(ENABLE_FLORET)
+    # Use pkg-config to find floret
+    pkg_check_modules(PC_FLORET QUIET floret)
+    if(PC_FLORET_FOUND)
+        message(STATUS "floret not found via pkg-config")
+        set(FLORET_FOUND TRUE)
+        set(FLORET_LIBRARIES ${PC_FLORET_LIBRARIES})
+        set(FLORET_INCLUDE_DIRS ${PC_FLORET_INCLUDE_DIRS})
     endif()
 endif()
 
-if(FASTTEXT_FOUND)
-    message(STATUS "fastText found, building MLInference with USE_FASTTEXT=1")
-    add_definitions(-DUSE_FASTTEXT=1)
+if(FLORET_FOUND)
+    message(STATUS "floret found via pkg-config, building MLInference with USE_FLORET=1")
+    add_definitions(-DUSE_FLORET=1)
 
-    # Define the ML inference library that actually uses fastText
-    add_library(ml_inference_lib STATIC
+    # Define the ML inference library that actually uses FLORET
+    add_library(ml_inference_lib SHARED
         FeatureExtractor.cpp
         FeaturePruner.cpp
         MLInference.cpp
     )
 
+    set_target_properties(ml_inference_lib PROPERTIES
+        VERSION 1.0.0
+        SOVERSION 1
+    )
+
+    target_link_libraries(ContextualClassifier PRIVATE ml_inference_lib)
+
     target_link_libraries(ml_inference_lib
-        PRIVATE ${FASTTEXT_TARGET}
+        PRIVATE
+            ${FLORET_LIBRARIES}
     )
 
     target_include_directories(ml_inference_lib
-        PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/Include
+        PUBLIC
+            ${CMAKE_CURRENT_SOURCE_DIR}/Include
+            ${FLORET_INCLUDE_DIRS}
     )
 else()
-    message(STATUS "fastText not found or ENABLE_FASTTEXT=OFF — falling back to base Inference only")
+    message(STATUS "floret not found via pkg-config or ENABLE_FLORET=OFF — falling back to base Inference only")
 endif()
 
 # Installation rules
-install(TARGETS ContextualClassifier DESTINATION ${CMAKE_INSTALL_LIBDIR})
-if(FASTTEXT_FOUND)
-    install(TARGETS ml_inference_lib DESTINATION ${CMAKE_INSTALL_LIBDIR})
-    install(FILES ${CMAKE_SOURCE_DIR}/contextual-classifier/Artifacts/fasttext_model_supervised.bin DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/urm/classifier)
+install(TARGETS ContextualClassifier DESTINATION lib)
+if(FLORET_FOUND)
+    install(TARGETS ml_inference_lib DESTINATION lib)
+    install(FILES ${CMAKE_SOURCE_DIR}/contextual-classifier/Artifacts/floret_model_supervised.bin DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/urm/classifier)
 endif()
 
 install(

contextual-classifier/CMakeLists.txt

@@ -27,16 +27,16 @@
 #define CLASSIFIER_TAG "CONTEXTUAL_CLASSIFIER"
 #define CLASSIFIER_CONFIGS_DIR "/etc/urm/classifier/"
 
-static const std::string FT_MODEL_PATH =
-    CLASSIFIER_CONFIGS_DIR "fasttext_model_supervised.bin";
-static const std::string IGNORE_PROC_PATH =
+const std::string FT_MODEL_PATH =
+    CLASSIFIER_CONFIGS_DIR "floret_model_supervised.bin";
+const std::string IGNORE_PROC_PATH =
     CLASSIFIER_CONFIGS_DIR "classifier-blocklist.txt";
 static const std::string IGNORE_TOKENS_PATH =
     CLASSIFIER_CONFIGS_DIR "ignore-tokens.txt";
 static const std::string ALLOW_LIST_PATH =
     CLASSIFIER_CONFIGS_DIR "allow-list.txt";
 
-#ifdef USE_FASTTEXT
+#ifdef USE_FLORET
 #include "MLInference.h"
 #include "FeatureExtractor.h"
 #include "FeaturePruner.h"

contextual-classifier/ContextualClassifier.cpp

@@ -1,6 +1,7 @@
 // Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 
+#include "AuxRoutines.h"
 #include "FeatureExtractor.h"
 #include "FeaturePruner.h"
 #include "Logger.h"
@@ -22,6 +23,8 @@
 #define SCANNER_TAG "FeatureExtractor"
 #define LOG_LINES 20
 
+std::unordered_map<std::string, std::unordered_set<std::string>> FeatureExtractor::mTokenIgnoreMap;
+
 static std::string format_string(const char *fmt, ...) {
     char buffer[1024];
     va_list args;

contextual-classifier/FeatureExtractor.cpp

@@ -113,6 +113,9 @@ class ContextualClassifier {
 public:
     ContextualClassifier();
     ~ContextualClassifier();
+    Inference *getInference() {
+        return mInference;
+    }
 
     ErrCode Init();
     ErrCode Terminate();

contextual-classifier/Include/ContextualClassifier.h

@@ -4,18 +4,19 @@
 #ifndef ML_INFERENCE_H
 #define ML_INFERENCE_H
 
+#include "AuxRoutines.h"
 #include "Inference.h"
 #include <vector>
 
-#include <fasttext/fasttext.h>
+#include <floret/fasttext.h>
 #include <mutex>
 #include <string>
 
 class MLInference : public Inference {
   public:
     MLInference(const std::string &ft_model_path);
     ~MLInference();
-    int Classify(int process_pid) override;
+    CC_TYPE Classify(int process_pid) override;
   private:
     // Derived implementation using fastText.
     uint32_t predict(int pid,

contextual-classifier/Include/MLInference.h

@@ -3,6 +3,7 @@
 
 #include "MLInference.h"
 #include "ContextualClassifier.h"
+#include "FeatureExtractor.h"
 #include <algorithm>
 #include <cmath>
 #include <fstream>
@@ -16,6 +17,15 @@
 
 #define CLASSIFIER_TAG "MLInference"
 
+static std::string format_string(const char *fmt, ...) {
+    char buffer[1024];
+    va_list args;
+    va_start(args, fmt);
+    vsnprintf(buffer, sizeof(buffer), fmt, args);
+    va_end(args);
+    return std::string(buffer);
+}
+
 MLInference::MLInference(const std::string &ft_model_path)
     : Inference(ft_model_path) {
     text_cols_ = {"attr", "cgroup",  "cmdline", "comm", "maps",
@@ -46,8 +56,8 @@ std::string MLInference::normalize_text(const std::string &text) {
 }
 
 CC_TYPE MLInference::Classify(int process_pid) {
-    LOGD(CLASSIFIER_TAG,
-         format_string("Starting classification for PID:%d", process_pid));
+    //LOGD(CLASSIFIER_TAG,
+    //     format_string("Starting classification for PID:%d", process_pid));
 
     const std::string proc_path = "/proc/" + std::to_string(process_pid);
     CC_TYPE contextType = CC_APP;
@@ -56,24 +66,24 @@ CC_TYPE MLInference::Classify(int process_pid) {
 
     auto start_collect = std::chrono::high_resolution_clock::now();
     int collect_rc = FeatureExtractor::CollectAndStoreData(
-        process_pid, raw_data, mDebugMode);
+        process_pid, raw_data, false);
     auto end_collect = std::chrono::high_resolution_clock::now();
     std::chrono::duration<double, std::milli> elapsed_collect =
         end_collect - start_collect;
-    LOGD(CLASSIFIER_TAG,
-         format_string("Data collection for PID:%d took %f ms (rc=%d)",
-                       process_pid, elapsed_collect.count(), collect_rc));
+    //LOGD(CLASSIFIER_TAG,
+    //     format_string("Data collection for PID:%d took %f ms (rc=%d)",
+    //                   process_pid, elapsed_collect.count(), collect_rc));
 
     if (collect_rc != 0) {
         // Process exited or collection failed; skip further work.
-        return 0;
+        return contextType;
     }
 
-    LOGD(CLASSIFIER_TAG,
-         format_string("Text features collected for PID:%d", process_pid));
+    //LOGD(CLASSIFIER_TAG,
+    //     format_string("Text features collected for PID:%d", process_pid));
 
     if (!AuxRoutines::fileExists(proc_path)) {
-        return 0;
+        return contextType;
     }
 
     bool has_sufficient_features = false;
@@ -93,8 +103,8 @@ CC_TYPE MLInference::Classify(int process_pid) {
              format_string("Invoking ML inference for PID:%d", process_pid));
 
         auto start_inference = std::chrono::high_resolution_clock::now();
-        if (mInference) {
-            uint32_t rc = mInference->predict(process_pid, raw_data, predicted_label);
+        //if (Inference) {
+            uint32_t rc = predict(process_pid, raw_data, predicted_label);
             auto end_inference = std::chrono::high_resolution_clock::now();
             std::chrono::duration<double, std::milli> elapsed_inference =
                 end_inference - start_inference;
@@ -105,11 +115,11 @@ CC_TYPE MLInference::Classify(int process_pid) {
                 // Inference failed, keep contextType as UNKNOWN.
                 predicted_label.clear();
             }
-        } else {
+        /*} else {
             LOGW(CLASSIFIER_TAG,
                  format_string("No Inference object available for PID:%d",
                                process_pid));
-        }
+        }*/
 
         // Map stripped label -> CC_APP enum.
         // MLInference::predict() returns after stripping "__label__".