use nnue for Fairy-Stockfish

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "Fairy-Stockfish"]
 	path = Fairy-Stockfish
 	url = https://github.com/ianfab/Fairy-Stockfish.git
+[submodule "assets"]
+	path = assets
+	url = https://github.com/lichess-org/fishnet-assets

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fishnet"
-version = "2.11.1-dev" # pull, test, remove dev, publish, tag, bump dev, push
+version = "2.11.1-dev" # pull, remove dev, test, tag, bump dev, check, push
 description = "Distributed Stockfish analysis for lichess.org"
 repository = "https://github.com/lichess-org/fishnet"
 readme = "README.md"
@@ -10,16 +10,7 @@ categories = ["command-line-utilities", "games"]
 keywords = ["chess", "lichess"]
 rust-version = "1.85.1"
 edition = "2024"
-exclude = [
-    "Stockfish/**/*.o",
-    "Stockfish/**/*.s",
-    "Stockfish/**/.depend",
-    "Stockfish/**/*.nnue",
-    "Fairy-Stockfish/**/*.o",
-    "Fairy-Stockfish/**/*.s",
-    "Fairy-Stockfish/**/.depend",
-    "Fairy-Stockfish/**/*.nnue",
-]
+publish = false
 
 [profile.release]
 strip = true

build.rs

@@ -15,9 +15,6 @@ use zstd::stream::write::Encoder as ZstdEncoder;
 
 static OUT_PATH: LazyLock<PathBuf> = LazyLock::new(|| PathBuf::from(&env::var("OUT_DIR").unwrap()));
 
-const EVAL_FILE_NAME: &str = "nn-1c0000000000.nnue";
-const EVAL_FILE_SMALL_NAME: &str = "nn-37f18f62d772.nnue";
-
 static SF_SOURCE_FILES: LazyLock<Vec<PathBuf>> = LazyLock::new(|| {
     assert!(
         Path::new("Stockfish").join("src").is_dir(),
@@ -34,8 +31,6 @@ static SF_SOURCE_FILES: LazyLock<Vec<PathBuf>> = LazyLock::new(|| {
         "Stockfish/**/*.sh",
         "Stockfish/src/**/*.cpp",
         "Stockfish/src/**/*.h",
-        &format!("Stockfish/src/{}", EVAL_FILE_NAME),
-        &format!("Stockfish/src/{}", EVAL_FILE_SMALL_NAME),
         // Fairy-Stockfish
         "Fairy-Stockfish/src/Makefile",
         "Fairy-Stockfish/src/**/*.cpp",
@@ -65,22 +60,7 @@ fn main() {
         ZstdEncoder::new(File::create(OUT_PATH.join("assets.ar.zst")).unwrap(), 6).unwrap(),
     );
     stockfish_build(&mut archive);
-    append_file(
-        &mut archive,
-        SF_BUILD_PATH
-            .join("Stockfish")
-            .join("src")
-            .join(EVAL_FILE_NAME),
-        0o644,
-    );
-    append_file(
-        &mut archive,
-        SF_BUILD_PATH
-            .join("Stockfish")
-            .join("src")
-            .join(EVAL_FILE_SMALL_NAME),
-        0o644,
-    );
+    add_nnues(&mut archive);
     archive.into_inner().unwrap().finish().unwrap();
 
     add_favicon();
@@ -307,20 +287,8 @@ struct Target {
     sde: Option<String>,
 }
 
-#[derive(Debug, PartialEq, Eq)]
-enum Flavor {
-    Official,
-    MultiVariant,
-}
-
 impl Target {
-    fn build<W: Write>(
-        &self,
-        flavor: Flavor,
-        src_path: &Path,
-        name: &'static str,
-        archive: &mut ar::Builder<W>,
-    ) {
+    fn build<W: Write>(&self, src_path: &Path, name: &'static str, archive: &mut ar::Builder<W>) {
         let release = env::var("PROFILE").unwrap() == "release";
         let windows = env::var("CARGO_CFG_TARGET_FAMILY").unwrap() == "windows";
         let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
@@ -385,20 +353,6 @@ impl Target {
             "$(MAKE) clean"
         );
 
-        if flavor == Flavor::Official {
-            assert!(
-                Command::new(&make)
-                    .current_dir(src_path)
-                    .env("MAKEFLAGS", env::var("CARGO_MAKEFLAGS").unwrap())
-                    .arg("-B")
-                    .arg("net")
-                    .status()
-                    .unwrap()
-                    .success(),
-                "$(MAKE) net"
-            );
-        }
-
         assert!(
             Command::new(&make)
                 .current_dir(src_path)
@@ -445,7 +399,6 @@ impl Target {
 
     fn build_official<W: Write>(&self, archive: &mut ar::Builder<W>) {
         self.build(
-            Flavor::Official,
             &SF_BUILD_PATH.join("Stockfish").join("src"),
             "stockfish",
             archive,
@@ -454,7 +407,6 @@ impl Target {
 
     fn build_multi_variant<W: Write>(&self, archive: &mut ar::Builder<W>) {
         self.build(
-            Flavor::MultiVariant,
             &SF_BUILD_PATH.join("Fairy-Stockfish").join("src"),
             "fairy-stockfish",
             archive,
@@ -467,6 +419,34 @@ impl Target {
     }
 }
 
+fn add_nnues<W: Write>(archive: &mut ar::Builder<W>) {
+    assert!(
+        Path::new("assets").join("README.md").is_file(),
+        "assets/README.md does not exist. Try: git submodule update --init"
+    );
+
+    for nnue in glob("assets/**/*.nnue").unwrap() {
+        let nnue = nnue.unwrap();
+        println!("cargo:rerun-if-changed={}", nnue.display());
+        append_file(archive, nnue, 0o644);
+    }
+
+    println!(
+        "cargo:rustc-env=FISHNET_FAIRY_STOCKFISH_EVAL_FILES={}",
+        glob("assets/Fairy-Stockfish/*.nnue")
+            .unwrap()
+            .map(|path| path
+                .unwrap()
+                .file_name()
+                .unwrap()
+                .to_str()
+                .unwrap()
+                .to_owned())
+            .collect::<Vec<_>>()
+            .join(if cfg!(windows) { ";" } else { ":" })
+    );
+}
+
 fn append_file<W: Write, P: AsRef<Path>>(archive: &mut ar::Builder<W>, path: P, mode: u32) {
     let file = File::open(&path).unwrap();
     let metadata = file.metadata().unwrap();

src/api.rs

@@ -16,7 +16,6 @@ use tokio::{
 use url::Url;
 
 use crate::{
-    assets::EvalFlavor,
     configure::{Endpoint, Key, KeyError},
     ipc::Chunk,
     logger::Logger,
@@ -62,7 +61,6 @@ enum ApiMessage {
     },
     SubmitAnalysis {
         batch_id: BatchId,
-        flavor: EvalFlavor,
         analysis: Vec<Option<AnalysisPart>>,
     },
     SubmitMove {
@@ -117,7 +115,14 @@ impl Fishnet {
     }
 }
 
-#[derive(Debug, Serialize)]
+#[derive(Debug, Copy, Clone, Default, Serialize)]
+enum EvalFlavor {
+    #[default]
+    #[serde(rename = "nnue")]
+    Nnue,
+}
+
+#[derive(Debug, Clone, Default, Serialize)]
 struct Stockfish {
     flavor: EvalFlavor,
 }
@@ -213,22 +218,19 @@ impl fmt::Display for BatchId {
 
 #[derive(Debug, Copy, Clone, Deserialize)]
 pub struct NodeLimit {
-    classical: u32,
+    #[serde(rename = "classical")]
+    _classical: u32,
     sf16: u32,
 }
 
 impl NodeLimit {
-    pub fn get(&self, flavor: EvalFlavor) -> u64 {
+    pub fn get(self) -> u64 {
         // Adjust for nodes spent on overlap of chunks: Worst case is
         // Chunk::MAX_POSITIONS positions split into one chunk of
         // Chunk::MAX_POSITIONS - 1 real positions and one chunk of 1
         // real position and 1 overlap position, such that
         // Chunk::MAX_POSITIONS + 1 positions are analysed.
-        u64::from(match flavor {
-            EvalFlavor::Hce => self.classical,
-            EvalFlavor::Nnue => self.sf16,
-        }) * (Chunk::MAX_POSITIONS as u64)
-            / (Chunk::MAX_POSITIONS as u64 + 1)
+        u64::from(self.sf16) * (Chunk::MAX_POSITIONS as u64) / (Chunk::MAX_POSITIONS as u64 + 1)
     }
 }
 
@@ -446,18 +448,9 @@ impl ApiStub {
         res.await.ok()
     }
 
-    pub fn submit_analysis(
-        &mut self,
-        batch_id: BatchId,
-        flavor: EvalFlavor,
-        analysis: Vec<Option<AnalysisPart>>,
-    ) {
+    pub fn submit_analysis(&mut self, batch_id: BatchId, analysis: Vec<Option<AnalysisPart>>) {
         self.tx
-            .send(ApiMessage::SubmitAnalysis {
-                batch_id,
-                flavor,
-                analysis,
-            })
+            .send(ApiMessage::SubmitAnalysis { batch_id, analysis })
             .expect("api actor alive");
     }
 
@@ -680,11 +673,7 @@ impl ApiActor {
                     }
                 }
             }
-            ApiMessage::SubmitAnalysis {
-                batch_id,
-                flavor,
-                analysis,
-            } => {
+            ApiMessage::SubmitAnalysis { batch_id, analysis } => {
                 let url = format!("{}/analysis/{}", self.endpoint, batch_id);
                 let res = self
                     .client
@@ -696,7 +685,7 @@ impl ApiActor {
                     })
                     .json(&AnalysisRequestBody {
                         fishnet: Fishnet::authenticated(self.key.clone()),
-                        stockfish: Stockfish { flavor },
+                        stockfish: Stockfish::default(),
                         analysis,
                     })
                     .send()

src/assets.rs

@@ -8,7 +8,6 @@ use std::{
 
 use ar::Archive;
 use bitflags::bitflags;
-use serde::Serialize;
 use tempfile::TempDir;
 use zstd::stream::read::Decoder as ZstdDecoder;
 
@@ -151,11 +150,8 @@ pub enum EngineFlavor {
 }
 
 impl EngineFlavor {
-    pub fn eval_flavor(self) -> EvalFlavor {
-        match self {
-            EngineFlavor::Official => EvalFlavor::Nnue,
-            EngineFlavor::MultiVariant => EvalFlavor::Hce,
-        }
+    pub fn is_official(self) -> bool {
+        matches!(self, EngineFlavor::Official)
     }
 }
 
@@ -181,24 +177,6 @@ impl<T> ByEngineFlavor<T> {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize)]
-pub enum EvalFlavor {
-    #[serde(rename = "classical")]
-    Hce,
-    #[serde(rename = "nnue")]
-    Nnue,
-}
-
-impl EvalFlavor {
-    pub fn is_nnue(self) -> bool {
-        matches!(self, EvalFlavor::Nnue)
-    }
-
-    pub fn is_hce(self) -> bool {
-        matches!(self, EvalFlavor::Hce)
-    }
-}
-
 #[derive(Debug)]
 pub struct Stockfish {
     pub name: String,

src/main.rs

@@ -202,11 +202,11 @@ async fn run(opt: Opt, client: &Client, logger: &Logger) {
         // Print summary from time to time.
         if now.duration_since(summarized) >= Duration::from_secs(120) {
             summarized = now;
-            let (stats, nnue_nps) = queue.stats().await;
+            let (stats, nps) = queue.stats().await;
             logger.fishnet_info(&format!(
-                "v{}: {} (nnue), {} batches, {} positions, {} total nodes",
+                "v{}: {}, {} batches, {} positions, {} total nodes",
                 env!("CARGO_PKG_VERSION"),
-                nnue_nps,
+                nps,
                 dot_thousands(stats.total_batches),
                 dot_thousands(stats.total_positions),
                 dot_thousands(stats.total_nodes),
@@ -275,33 +275,35 @@ async fn worker(i: usize, assets: Arc<Assets>, tx: mpsc::Sender<Pull>, logger: L
             // Ensure engine process is ready.
             let flavor = chunk.flavor;
             let context = ProgressAt::from(&chunk);
-            let (mut sf, join_handle) = if let Some((sf, join_handle)) =
-                engine.get_mut(flavor).take()
-            {
-                (sf, join_handle)
-            } else {
-                // Backoff before starting engine.
-                let backoff = engine_backoff.next();
-                if backoff >= Duration::from_secs(5) {
-                    logger.info(&format!(
-                        "Waiting {backoff:?} before attempting to start engine"
-                    ));
+            let (mut sf, join_handle) =
+                if let Some((sf, join_handle)) = engine.get_mut(flavor).take() {
+                    (sf, join_handle)
                 } else {
-                    logger.debug(&format!(
-                        "Waiting {backoff:?} before attempting to start engine"
-                    ));
-                }
-                tokio::select! {
-                    _ = tx.closed() => break,
-                    _ = sleep(engine_backoff.next()) => (),
-                }
+                    // Backoff before starting engine.
+                    let backoff = engine_backoff.next();
+                    if backoff >= Duration::from_secs(5) {
+                        logger.info(&format!(
+                            "Waiting {backoff:?} before attempting to start engine"
+                        ));
+                    } else {
+                        logger.debug(&format!(
+                            "Waiting {backoff:?} before attempting to start engine"
+                        ));
+                    }
+                    tokio::select! {
+                        _ = tx.closed() => break,
+                        _ = sleep(engine_backoff.next()) => (),
+                    }
 
-                // Start engine and spawn actor.
-                let (sf, sf_actor) =
-                    stockfish::channel(assets.stockfish.get(flavor).path.clone(), logger.clone());
-                let join_handle = tokio::spawn(sf_actor.run());
-                (sf, join_handle)
-            };
+                    // Start engine and spawn actor.
+                    let (sf, sf_actor) = stockfish::channel(
+                        assets.stockfish.get(flavor).path.clone(),
+                        flavor,
+                        logger.clone(),
+                    );
+                    let join_handle = tokio::spawn(sf_actor.run());
+                    (sf, join_handle)
+                };
 
             // Analyse or play.
             let batch_id = chunk.work.id();