[pull] main from VirusTotal:main

.github/workflows/tests.yaml

@@ -86,7 +86,9 @@ jobs:
             os: windows-latest
             rust: stable
             target: x86_64-pc-windows-gnu
-            args: " --no-default-features --features=default-modules,rules-profiling,inventory"
+            # yara-x-py is excluded because it fails to compile in Windows with
+            # error: export ordinal too large: 252264
+            args: "--features=rules-profiling --workspace --exclude=yara-x-ls"
             rust_flags: "-Awarnings"
             experimental: false
             use_cache: true

AUTHORS

@@ -8,6 +8,7 @@
 
 # Please keep the list sorted.
 
+Akamai Technologies
 Gen Digital Inc.
 Google Inc.
 Marek Milkovič <milkovic.marek@gmail.com>

CONTRIBUTORS

@@ -26,6 +26,7 @@
 
 # Please keep the list sorted.
 
+Amanda Greene <agreene@akamai.com>
 Jacob Latonis <jlatonis@me.com>
 Marek Milkovič <milkovic.marek@gmail.com>;<marek.milkovic@gendigital.com> 
 Tomáš Ďuriš <duristomas67@gmail.com>

capi/Cargo.toml

@@ -28,22 +28,31 @@ capi = []
 # This feature is disabled by default.
 native-code-serialization = ["yara-x/native-code-serialization"]
 
+# Enables parallel compilation of WASM code. When compiling large number of
+# rules this noticeable reduces compilation time. However, this creates new
+# threads, which can be problematic in some scenarios. See:
+# https://github.com/VirusTotal/yara-x/issues/182
+#
+# This feature is disabled by default.
+parallel-compilation = ["yara-x/parallel-compilation"]
+
 # Enables rules profiling.
 #
 # This feature is disabled by default.
 rules-profiling = ["yara-x/rules-profiling"]
 
-
+# Enables the `magic` module.
+#
+# This feature is disabled by default.
 magic-module = ["yara-x/magic-module"]
 
-
 [lib]
 name = "yara_x_capi"
 crate-type = ["staticlib", "cdylib"]
 
 [dependencies]
 serde_json = { workspace = true }
-yara-x = { workspace = true,  default-features = true, features = ["parallel-compilation"] }
+yara-x = { workspace = true,  default-features = true }
 
 [build-dependencies]
 cbindgen = { workspace = true }

lib/Cargo.toml

@@ -251,7 +251,7 @@ default = [
     "exact-atoms",
     "fast-regexp",
     "generate-proto-code",
-    "linkme",
+    "inventory",
 ]
 
 [dependencies]

ls/Cargo.toml

@@ -18,21 +18,12 @@ name = "yr-ls"
 path = "src/main.rs"
 
 [features]
-# Enables the use of the YARA-X compiler in the language server.
-#
-# With this feature the language server depends on the yara-x crate, but it
-# offers more features. Without this feature, the language server is lighter
-# because it only depends on the yara-x-parser crate.
-full-compiler = ["dep:yara-x"]
-
 # Enables tracing, which helps debugging the language server.
 tracing = ["dep:tracing", "dep:tracing-subscriber"]
 
 # Enables the `magic` module.
 magic-module = ["yara-x/magic-module"]
 
-default = ["full-compiler"]
-
 [dependencies]
 bitflags = { workspace = true }
 chrono = { workspace = true, features = ["serde"] }
@@ -46,7 +37,7 @@ serde = { workspace = true }
 serde_json = { workspace = true }
 yara-x-parser = { workspace = true }
 yara-x-fmt = { workspace = true }
-yara-x = { workspace = true, default-features = true, optional = true }
+yara-x = { workspace = true, default-features = true }
 
 [target.'cfg(not(target_family = "wasm"))'.dependencies]
 tokio = { version = "1.48.0", features = ["full"] }

ls/src/features/completion.rs

@@ -6,12 +6,9 @@ use async_lsp::lsp_types::{
     InsertTextMode, Position, Url,
 };
 
-#[cfg(feature = "full-compiler")]
 use itertools::Itertools;
 
-#[cfg(feature = "full-compiler")]
 use yara_x::mods::reflect::Type;
-#[cfg(feature = "full-compiler")]
 use yara_x::mods::{module_definition, module_names};
 use yara_x_parser::cst::{CST, Immutable, Node, SyntaxKind, Token};
 
@@ -21,7 +18,6 @@ use crate::utils::cst_traversal::{
     rule_containing_token, token_at_position,
 };
 
-#[cfg(feature = "full-compiler")]
 use crate::utils::cst_traversal::find_declaration;
 
 const PATTERN_MODS: &[(SyntaxKind, &[&str])] = &[
@@ -118,10 +114,7 @@ pub fn completion(
     }
 
     if prev_token.kind() == SyntaxKind::IMPORT_KW {
-        #[cfg(feature = "full-compiler")]
         return Some(import_suggestions());
-        #[cfg(not(feature = "full-compiler"))]
-        return None;
     }
 
     if let Some(pattern_def) =
@@ -146,7 +139,6 @@ fn condition_suggestions(
 ) -> Option<Vec<CompletionItem>> {
     let mut result = Vec::new();
 
-    #[cfg(feature = "full-compiler")]
     if let Some(suggestions) = field_suggestions(&token) {
         return Some(suggestions);
     }
@@ -263,7 +255,6 @@ fn condition_suggestions(
 }
 
 /// Collects completion suggestions for import statements.
-#[cfg(feature = "full-compiler")]
 fn import_suggestions() -> Vec<CompletionItem> {
     module_names()
         .map(|name| CompletionItem {
@@ -332,15 +323,13 @@ fn rule_suggestions() -> Vec<CompletionItem> {
         .collect()
 }
 
-#[cfg(feature = "full-compiler")]
 #[derive(Debug)]
 enum Segment {
     Field(String),
     Index,
 }
 
 /// Collects completion suggestions for structure fields.
-#[cfg(feature = "full-compiler")]
 fn field_suggestions(token: &Token<Immutable>) -> Option<Vec<CompletionItem>> {
     // Check if we are at a position that triggers completion.
     let token = match token.kind() {
@@ -435,7 +424,6 @@ fn field_suggestions(token: &Token<Immutable>) -> Option<Vec<CompletionItem>> {
     Some(suggestions)
 }
 
-#[cfg(feature = "full-compiler")]
 /// Given a token, returns the type of the structure that the token is part of.
 ///
 /// This function traverses the CST backwards from the given token to determine
@@ -532,7 +520,6 @@ fn get_struct(token: &Token<Immutable>) -> Option<Type> {
     Some(current_kind)
 }
 
-#[cfg(feature = "full-compiler")]
 /// Resolves the `Type` of an identifier declared within `for` or `with` statements.
 ///
 /// This function is called when `get_struct` identifies an identifier that is
@@ -640,7 +627,6 @@ fn get_type_from_declaration(
 
 /// Given a token that must be a closing (right) bracket, find the
 /// corresponding opening (left) bracket.
-#[cfg(feature = "full-compiler")]
 fn find_matching_left_bracket(
     token: &Token<Immutable>,
 ) -> Option<Token<Immutable>> {
@@ -666,7 +652,6 @@ fn find_matching_left_bracket(
     None
 }
 
-#[cfg(feature = "full-compiler")]
 fn ty_to_string(ty: &Type) -> String {
     match ty {
         Type::Integer => "integer".to_string(),

ls/src/features/diagnostics.rs

@@ -1,25 +1,23 @@
 use std::sync::Arc;
 
 use async_lsp::lsp_types::{Diagnostic, Range, Url};
-#[cfg(feature = "full-compiler")]
+
 use async_lsp::lsp_types::{
     DiagnosticRelatedInformation, DiagnosticSeverity, Location, NumberOrString,
 };
-#[cfg(feature = "full-compiler")]
+
 use dashmap::mapref::one::Ref;
 use serde::{Deserialize, Serialize};
 
 use chrono::NaiveDate;
 use regex::Regex;
 
 use crate::configuration::MetadataValidationRule;
-#[cfg(feature = "full-compiler")]
+
 use crate::documents::document::Document;
 use crate::documents::storage::DocumentStorage;
 
-#[cfg(feature = "full-compiler")]
 use yara_x::linters;
-#[cfg(feature = "full-compiler")]
 use yara_x::{Compiler, SourceCode};
 
 #[derive(Serialize, Deserialize)]
@@ -47,7 +45,6 @@ pub fn diagnostics(
     let doc = documents.get(&uri);
 
     if let Some(doc) = doc {
-        #[cfg(feature = "full-compiler")]
         diagnostics.extend(compiler_diagnostics(
             doc,
             metadata_validation,
@@ -64,7 +61,6 @@ pub fn diagnostics(
 /// and collects all errors and warnings as LSP diagnostics. This provides
 /// comprehensive feedback including type checking, semantic analysis,
 /// and pattern validation - not just syntax errors.
-#[cfg(feature = "full-compiler")]
 pub fn compiler_diagnostics(
     document: Ref<'_, Url, Document>,
     metadata_validation: &Vec<MetadataValidationRule>,

ls/src/tests/mod.rs

@@ -325,26 +325,12 @@ async fn document_highlights() {
 #[tokio::test]
 async fn document_diagnostics() {
     test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics1.yar").await;
-
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics2.yar").await;
-
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics3.yar").await;
-
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics4.yar").await;
-
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics5.yar").await;
-
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics6.yar").await;
-
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics7.yar").await;
-
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, DocumentDiagnosticRequest>("diagnostics8.yar").await;
 }
 
@@ -358,34 +344,25 @@ async fn completion() {
     test_lsp_request::<_, Completion>("completion6.yar").await;
     test_lsp_request::<_, Completion>("completion7.yar").await;
 
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, Completion>("completion8.yar").await;
 
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, Completion>("completion9.yar").await;
 
-    #[cfg(all(feature = "full-compiler", not(feature = "magic-module")))]
+    #[cfg(not(feature = "magic-module"))]
     test_lsp_request::<_, Completion>("completion10.yar").await;
 
-    #[cfg(all(feature = "full-compiler", not(feature = "magic-module")))]
+    #[cfg(not(feature = "magic-module"))]
     test_lsp_request::<_, Completion>("completion11.yar").await;
 
-    #[cfg(all(feature = "full-compiler", not(feature = "magic-module")))]
+    #[cfg(not(feature = "magic-module"))]
     test_lsp_request::<_, Completion>("completion12.yar").await;
 
-    #[cfg(all(feature = "full-compiler", not(feature = "magic-module")))]
+    #[cfg(not(feature = "magic-module"))]
     test_lsp_request::<_, Completion>("completion13.yar").await;
 
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, Completion>("completion14.yar").await;
-
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, Completion>("completion15.yar").await;
-
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, Completion>("completion16.yar").await;
-
-    #[cfg(feature = "full-compiler")]
     test_lsp_request::<_, Completion>("completion17.yar").await;
 }
 

py/Cargo.toml

@@ -64,7 +64,7 @@ serde_json = { workspace = true }
 strum = { workspace = true }
 strum_macros = { workspace = true }
 
-yara-x = { workspace = true, features = ["linkme", "exact-atoms", "constant-folding", "fast-regexp"] }
+yara-x = { workspace = true, default-features = true }
 yara-x-proto-json = { workspace = true }
 yara-x-fmt = { workspace = true }
 

site/content/docs/modules/math.md

@@ -73,6 +73,10 @@ Examples:
 
 Returns the serial correlation for the given string.
 
+Examples:
+
+`math.serial_correlation("BCA")` → `-0.5`
+
 ### mean(offset, size)
 
 Returns the mean for the size bytes starting at offset. When scanning a running
@@ -87,6 +91,10 @@ Examples:
 
 Returns the mean for the given string.
 
+Examples:
+
+`math.mean("ABCABC")` → `66.0`
+
 ### deviation(offset, size, mean)
 
 Returns the deviation from the mean for the size bytes starting at offset. When
@@ -111,7 +119,7 @@ comparisons are inclusive.
 
 Examples:
 
-`math.in_range(math.deviation(0, filesize, math.MEAN_BYTES), 63.9, 64,1)`
+`math.in_range(math.deviation(0, filesize, math.MEAN_BYTES), 63.9, 64.1)`
 
 ### max(int, int)
 
@@ -185,7 +193,9 @@ Converts the given integer to a string. Note: integers in YARA are signed.
 
 Examples:
 
-`math.to_string(10) == "10" Example: math.to_string(-1) == "-1"`
+`math.to_string(10) == "10"`
+
+`math.to_string(-1) == "-1"`
 
 ### to_string(int, base)