From 23909f5d55e7862312ce7c30122c873f86b9e0d0 Mon Sep 17 00:00:00 2001 From: Niklas Fiekas Date: Sun, 12 Oct 2025 12:38:25 +0200 Subject: [PATCH 1/3] use nnue for Fairy-Stockfish --- .gitmodules | 3 ++ Fairy-Stockfish | 2 +- assets | 1 + build.rs | 80 ++++++++++++++++++------------------------------ src/api.rs | 43 ++++++++++---------------- src/assets.rs | 26 ++-------------- src/main.rs | 52 ++++++++++++++++--------------- src/queue.rs | 35 ++++++--------------- src/stats.rs | 18 +++++------ src/stockfish.rs | 36 +++++++++++----------- 10 files changed, 116 insertions(+), 180 deletions(-) create mode 160000 assets diff --git a/.gitmodules b/.gitmodules index 0e5c564d..9636886f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "Fairy-Stockfish"] path = Fairy-Stockfish url = https://github.com/ianfab/Fairy-Stockfish.git +[submodule "assets"] + path = assets + url = https://github.com/lichess-org/fishnet-assets diff --git a/Fairy-Stockfish b/Fairy-Stockfish index f3e6969d..a8092108 160000 --- a/Fairy-Stockfish +++ b/Fairy-Stockfish @@ -1 +1 @@ -Subproject commit f3e6969d11d1bec17eba26e7ae0e629ad4af71dd +Subproject commit a809210877460579bc65ef6721365582e04e3008 diff --git a/assets b/assets new file mode 160000 index 00000000..9b66fb44 --- /dev/null +++ b/assets @@ -0,0 +1 @@ +Subproject commit 9b66fb44df9fa77b3c1223681806da629fb4160f diff --git a/build.rs b/build.rs index 715b0b21..312139aa 100644 --- a/build.rs +++ b/build.rs @@ -15,9 +15,6 @@ use zstd::stream::write::Encoder as ZstdEncoder; static OUT_PATH: LazyLock = LazyLock::new(|| PathBuf::from(&env::var("OUT_DIR").unwrap())); -const EVAL_FILE_NAME: &str = "nn-1c0000000000.nnue"; -const EVAL_FILE_SMALL_NAME: &str = "nn-37f18f62d772.nnue"; - static SF_SOURCE_FILES: LazyLock> = LazyLock::new(|| { assert!( Path::new("Stockfish").join("src").is_dir(), @@ -34,8 +31,6 @@ static SF_SOURCE_FILES: LazyLock> = LazyLock::new(|| { "Stockfish/**/*.sh", "Stockfish/src/**/*.cpp", "Stockfish/src/**/*.h", - &format!("Stockfish/src/{}", EVAL_FILE_NAME), - &format!("Stockfish/src/{}", EVAL_FILE_SMALL_NAME), // Fairy-Stockfish "Fairy-Stockfish/src/Makefile", "Fairy-Stockfish/src/**/*.cpp", @@ -65,22 +60,7 @@ fn main() { ZstdEncoder::new(File::create(OUT_PATH.join("assets.ar.zst")).unwrap(), 6).unwrap(), ); stockfish_build(&mut archive); - append_file( - &mut archive, - SF_BUILD_PATH - .join("Stockfish") - .join("src") - .join(EVAL_FILE_NAME), - 0o644, - ); - append_file( - &mut archive, - SF_BUILD_PATH - .join("Stockfish") - .join("src") - .join(EVAL_FILE_SMALL_NAME), - 0o644, - ); + add_nnues(&mut archive); archive.into_inner().unwrap().finish().unwrap(); add_favicon(); @@ -307,20 +287,8 @@ struct Target { sde: Option, } -#[derive(Debug, PartialEq, Eq)] -enum Flavor { - Official, - MultiVariant, -} - impl Target { - fn build( - &self, - flavor: Flavor, - src_path: &Path, - name: &'static str, - archive: &mut ar::Builder, - ) { + fn build(&self, src_path: &Path, name: &'static str, archive: &mut ar::Builder) { let release = env::var("PROFILE").unwrap() == "release"; let windows = env::var("CARGO_CFG_TARGET_FAMILY").unwrap() == "windows"; let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); @@ -385,20 +353,6 @@ impl Target { "$(MAKE) clean" ); - if flavor == Flavor::Official { - assert!( - Command::new(&make) - .current_dir(src_path) - .env("MAKEFLAGS", env::var("CARGO_MAKEFLAGS").unwrap()) - .arg("-B") - .arg("net") - .status() - .unwrap() - .success(), - "$(MAKE) net" - ); - } - assert!( Command::new(&make) .current_dir(src_path) @@ -445,7 +399,6 @@ impl Target { fn build_official(&self, archive: &mut ar::Builder) { self.build( - Flavor::Official, &SF_BUILD_PATH.join("Stockfish").join("src"), "stockfish", archive, @@ -454,7 +407,6 @@ impl Target { fn build_multi_variant(&self, archive: &mut ar::Builder) { self.build( - Flavor::MultiVariant, &SF_BUILD_PATH.join("Fairy-Stockfish").join("src"), "fairy-stockfish", archive, @@ -467,6 +419,34 @@ impl Target { } } +fn add_nnues(archive: &mut ar::Builder) { + assert!( + Path::new("assets").join("README.md").is_file(), + "assets/README.md does not exist. Try: git submodule update --init" + ); + + for nnue in glob("assets/**/*.nnue").unwrap() { + let nnue = nnue.unwrap(); + println!("cargo:rerun-if-changed={}", nnue.display()); + append_file(archive, nnue, 0o644); + } + + println!( + "cargo:rustc-env=FISHNET_FAIRY_STOCKFISH_EVAL_FILES={}", + glob("assets/Fairy-Stockfish/*.nnue") + .unwrap() + .map(|path| path + .unwrap() + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned()) + .collect::>() + .join(if cfg!(windows) { ";" } else { ":" }) + ); +} + fn append_file>(archive: &mut ar::Builder, path: P, mode: u32) { let file = File::open(&path).unwrap(); let metadata = file.metadata().unwrap(); diff --git a/src/api.rs b/src/api.rs index f7dc9fdf..f1030878 100644 --- a/src/api.rs +++ b/src/api.rs @@ -16,7 +16,6 @@ use tokio::{ use url::Url; use crate::{ - assets::EvalFlavor, configure::{Endpoint, Key, KeyError}, ipc::Chunk, logger::Logger, @@ -62,7 +61,6 @@ enum ApiMessage { }, SubmitAnalysis { batch_id: BatchId, - flavor: EvalFlavor, analysis: Vec>, }, SubmitMove { @@ -117,7 +115,14 @@ impl Fishnet { } } -#[derive(Debug, Serialize)] +#[derive(Debug, Copy, Clone, Default, Serialize)] +enum EvalFlavor { + #[default] + #[serde(rename = "nnue")] + Nnue, +} + +#[derive(Debug, Clone, Default, Serialize)] struct Stockfish { flavor: EvalFlavor, } @@ -213,22 +218,19 @@ impl fmt::Display for BatchId { #[derive(Debug, Copy, Clone, Deserialize)] pub struct NodeLimit { - classical: u32, + #[serde(rename = "classical")] + _classical: u32, sf16: u32, } impl NodeLimit { - pub fn get(&self, flavor: EvalFlavor) -> u64 { + pub fn get(self) -> u64 { // Adjust for nodes spent on overlap of chunks: Worst case is // Chunk::MAX_POSITIONS positions split into one chunk of // Chunk::MAX_POSITIONS - 1 real positions and one chunk of 1 // real position and 1 overlap position, such that // Chunk::MAX_POSITIONS + 1 positions are analysed. - u64::from(match flavor { - EvalFlavor::Hce => self.classical, - EvalFlavor::Nnue => self.sf16, - }) * (Chunk::MAX_POSITIONS as u64) - / (Chunk::MAX_POSITIONS as u64 + 1) + u64::from(self.sf16) * (Chunk::MAX_POSITIONS as u64) / (Chunk::MAX_POSITIONS as u64 + 1) } } @@ -446,18 +448,9 @@ impl ApiStub { res.await.ok() } - pub fn submit_analysis( - &mut self, - batch_id: BatchId, - flavor: EvalFlavor, - analysis: Vec>, - ) { + pub fn submit_analysis(&mut self, batch_id: BatchId, analysis: Vec>) { self.tx - .send(ApiMessage::SubmitAnalysis { - batch_id, - flavor, - analysis, - }) + .send(ApiMessage::SubmitAnalysis { batch_id, analysis }) .expect("api actor alive"); } @@ -680,11 +673,7 @@ impl ApiActor { } } } - ApiMessage::SubmitAnalysis { - batch_id, - flavor, - analysis, - } => { + ApiMessage::SubmitAnalysis { batch_id, analysis } => { let url = format!("{}/analysis/{}", self.endpoint, batch_id); let res = self .client @@ -696,7 +685,7 @@ impl ApiActor { }) .json(&AnalysisRequestBody { fishnet: Fishnet::authenticated(self.key.clone()), - stockfish: Stockfish { flavor }, + stockfish: Stockfish::default(), analysis, }) .send() diff --git a/src/assets.rs b/src/assets.rs index 563aa332..9ff45ad1 100644 --- a/src/assets.rs +++ b/src/assets.rs @@ -8,7 +8,6 @@ use std::{ use ar::Archive; use bitflags::bitflags; -use serde::Serialize; use tempfile::TempDir; use zstd::stream::read::Decoder as ZstdDecoder; @@ -151,11 +150,8 @@ pub enum EngineFlavor { } impl EngineFlavor { - pub fn eval_flavor(self) -> EvalFlavor { - match self { - EngineFlavor::Official => EvalFlavor::Nnue, - EngineFlavor::MultiVariant => EvalFlavor::Hce, - } + pub fn is_official(self) -> bool { + matches!(self, EngineFlavor::Official) } } @@ -181,24 +177,6 @@ impl ByEngineFlavor { } } -#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize)] -pub enum EvalFlavor { - #[serde(rename = "classical")] - Hce, - #[serde(rename = "nnue")] - Nnue, -} - -impl EvalFlavor { - pub fn is_nnue(self) -> bool { - matches!(self, EvalFlavor::Nnue) - } - - pub fn is_hce(self) -> bool { - matches!(self, EvalFlavor::Hce) - } -} - #[derive(Debug)] pub struct Stockfish { pub name: String, diff --git a/src/main.rs b/src/main.rs index 411f282c..e24b8513 100644 --- a/src/main.rs +++ b/src/main.rs @@ -275,33 +275,35 @@ async fn worker(i: usize, assets: Arc, tx: mpsc::Sender, logger: L // Ensure engine process is ready. let flavor = chunk.flavor; let context = ProgressAt::from(&chunk); - let (mut sf, join_handle) = if let Some((sf, join_handle)) = - engine.get_mut(flavor).take() - { - (sf, join_handle) - } else { - // Backoff before starting engine. - let backoff = engine_backoff.next(); - if backoff >= Duration::from_secs(5) { - logger.info(&format!( - "Waiting {backoff:?} before attempting to start engine" - )); + let (mut sf, join_handle) = + if let Some((sf, join_handle)) = engine.get_mut(flavor).take() { + (sf, join_handle) } else { - logger.debug(&format!( - "Waiting {backoff:?} before attempting to start engine" - )); - } - tokio::select! { - _ = tx.closed() => break, - _ = sleep(engine_backoff.next()) => (), - } + // Backoff before starting engine. + let backoff = engine_backoff.next(); + if backoff >= Duration::from_secs(5) { + logger.info(&format!( + "Waiting {backoff:?} before attempting to start engine" + )); + } else { + logger.debug(&format!( + "Waiting {backoff:?} before attempting to start engine" + )); + } + tokio::select! { + _ = tx.closed() => break, + _ = sleep(engine_backoff.next()) => (), + } - // Start engine and spawn actor. - let (sf, sf_actor) = - stockfish::channel(assets.stockfish.get(flavor).path.clone(), logger.clone()); - let join_handle = tokio::spawn(sf_actor.run()); - (sf, join_handle) - }; + // Start engine and spawn actor. + let (sf, sf_actor) = stockfish::channel( + assets.stockfish.get(flavor).path.clone(), + flavor, + logger.clone(), + ); + let join_handle = tokio::spawn(sf_actor.run()); + (sf, join_handle) + }; // Analyse or play. let batch_id = chunk.work.id(); diff --git a/src/queue.rs b/src/queue.rs index 34d1cfac..d63f6785 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -26,7 +26,7 @@ use crate::{ AcquireQuery, AcquireResponseBody, Acquired, AnalysisPart, ApiStub, BatchId, PositionIndex, Work, }, - assets::{EngineFlavor, EvalFlavor}, + assets::EngineFlavor, configure::{BacklogOpt, Endpoint, MaxBackoff, StatsOpt}, ipc::{Chunk, ChunkFailed, Position, PositionResponse, Pull}, logger::{Logger, ProgressAt, QueueStatusBar, short_variant_name}, @@ -117,7 +117,7 @@ impl QueueStub { let state = self.state.lock().await; ( state.stats_recorder.stats.clone(), - state.stats_recorder.nnue_nps.clone(), + state.stats_recorder.official_nps.clone(), ) } } @@ -250,20 +250,12 @@ impl QueueState { Ok(completed) => { let mut extra = Vec::new(); extra.extend(short_variant_name(completed.variant).map(|n| n.to_owned())); - if completed.flavor.eval_flavor().is_hce() { - extra.push("hce".to_owned()); - } extra.push(match completed.nps() { Some(nps) => { - let nnue_nps = if completed.flavor.eval_flavor() == EvalFlavor::Nnue { - Some(nps) - } else { - None - }; self.stats_recorder.record_batch( completed.total_positions(), completed.total_nodes, - nnue_nps, + completed.flavor.is_official().then_some(nps), ); format!("{} knps/core", nps / 1000) } @@ -286,11 +278,7 @@ impl QueueState { match completed.work { Work::Analysis { id, .. } => { self.logger.info(&log); - queue.api.submit_analysis( - id, - completed.flavor.eval_flavor(), - completed.into_analysis(), - ); + queue.api.submit_analysis(id, completed.into_analysis()); } Work::Move { id, .. } => { self.logger.debug(&log); @@ -305,11 +293,9 @@ impl QueueState { Err(pending) => { if !pending.work.matrix_wanted() { // Send partial analysis as progress report. - queue.api.submit_analysis( - pending.work.id(), - pending.flavor.eval_flavor(), - pending.progress_report(), - ); + queue + .api + .submit_analysis(pending.work.id(), pending.progress_report()); } self.pending.insert(pending.work.id(), pending); @@ -406,11 +392,8 @@ impl QueueActor { Err(IncomingError::AllSkipped(completed)) => { self.logger .warn(&format!("Completed empty batch {context}.")); - self.api.submit_analysis( - completed.work.id(), - completed.flavor.eval_flavor(), - completed.into_analysis(), - ); + self.api + .submit_analysis(completed.work.id(), completed.into_analysis()); } Err(err) if is_move => { self.logger diff --git a/src/stats.rs b/src/stats.rs index ec51af89..60010c04 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -19,7 +19,7 @@ fn default_stats_file() -> Option { pub struct StatsRecorder { pub stats: Stats, - pub nnue_nps: NpsRecorder, + pub official_nps: NpsRecorder, store: Option<(PathBuf, File)>, cores: NonZeroUsize, } @@ -60,13 +60,13 @@ impl Stats { impl StatsRecorder { pub fn new(opt: StatsOpt, cores: NonZeroUsize) -> StatsRecorder { - let nnue_nps = NpsRecorder::new(); + let official_nps = NpsRecorder::new(); if opt.no_stats_file { return StatsRecorder { stats: Stats::default(), store: None, - nnue_nps, + official_nps, cores, }; } @@ -78,7 +78,7 @@ impl StatsRecorder { return StatsRecorder { stats: Stats::default(), store: None, - nnue_nps, + official_nps, cores, }; }; @@ -116,18 +116,18 @@ impl StatsRecorder { StatsRecorder { stats, store, - nnue_nps, + official_nps, cores, } } - pub fn record_batch(&mut self, positions: u64, nodes: u64, nnue_nps: Option) { + pub fn record_batch(&mut self, positions: u64, nodes: u64, official_nps: Option) { self.stats.total_batches += 1; self.stats.total_positions += positions; self.stats.total_nodes += nodes; - if let Some(nnue_nps) = nnue_nps { - self.nnue_nps.record(nnue_nps); + if let Some(official_nps) = official_nps { + self.official_nps.record(official_nps); } if let Some((ref path, ref mut stats_file)) = self.store { @@ -142,7 +142,7 @@ impl StatsRecorder { // 60 positions at 1_450_000 nodes each. let estimated_batch_seconds = u64::from(min( 7 * 60, // deadline - 60 * 1_450_000 / self.cores.get() as u32 / max(1, self.nnue_nps.nps), + 60 * 1_450_000 / self.cores.get() as u32 / max(1, self.official_nps.nps), )); // Top end clients take no longer than 35 seconds. Its worth joining if diff --git a/src/stockfish.rs b/src/stockfish.rs index a8787e8e..8348c44b 100644 --- a/src/stockfish.rs +++ b/src/stockfish.rs @@ -9,19 +9,24 @@ use tokio::{ use crate::{ api::{Score, Work}, - assets::{EngineFlavor, EvalFlavor}, + assets::EngineFlavor, ipc::{Chunk, ChunkFailed, Matrix, Position, PositionResponse}, logger::Logger, util::NevermindExt as _, }; -pub fn channel(exe: PathBuf, logger: Logger) -> (StockfishStub, StockfishActor) { +pub fn channel( + exe: PathBuf, + engine_flavor: EngineFlavor, + logger: Logger, +) -> (StockfishStub, StockfishActor) { let (tx, rx) = mpsc::channel(1); ( StockfishStub { tx }, StockfishActor { rx, exe, + engine_flavor, initialized: false, logger, }, @@ -50,6 +55,7 @@ impl StockfishStub { pub struct StockfishActor { rx: mpsc::Receiver, exe: PathBuf, + engine_flavor: EngineFlavor, initialized: bool, logger: Logger, } @@ -213,6 +219,14 @@ impl StockfishActor { async fn init(&mut self, stdout: &mut Stdout, stdin: &mut Stdin) -> io::Result<()> { if !mem::replace(&mut self.initialized, true) { + if self.engine_flavor == EngineFlavor::MultiVariant { + stdin + .write_line(&format!( + "setoption name EvalFile value {}", + env!("FISHNET_FAIRY_STOCKFISH_EVAL_FILES"), + )) + .await?; + } stdin .write_line("setoption name UCI_Chess960 value true") .await?; @@ -250,12 +264,6 @@ impl StockfishActor { // Set basic options. if chunk.flavor == EngineFlavor::MultiVariant { - stdin - .write_line(&format!( - "setoption name Use NNUE value {}", - chunk.flavor.eval_flavor().is_nnue() - )) - .await?; stdin .write_line(&format!( "setoption name UCI_AnalyseMode value {}", @@ -288,10 +296,7 @@ impl StockfishActor { // Collect results for all positions of the chunk. let mut responses = Vec::with_capacity(chunk.positions.len()); for position in chunk.positions { - responses.push( - self.go(stdout, stdin, chunk.flavor.eval_flavor(), position) - .await?, - ); + responses.push(self.go(stdout, stdin, position).await?); } Ok(responses) } @@ -300,7 +305,6 @@ impl StockfishActor { &mut self, stdout: &mut Stdout, stdin: &mut Stdin, - eval_flavor: EvalFlavor, position: Position, ) -> io::Result { // Setup position. @@ -344,11 +348,7 @@ impl StockfishActor { go } Work::Analysis { nodes, depth, .. } => { - let mut go = vec![ - "go".to_owned(), - "nodes".to_owned(), - nodes.get(eval_flavor).to_string(), - ]; + let mut go = vec!["go".to_owned(), "nodes".to_owned(), nodes.get().to_string()]; if let Some(depth) = depth { go.extend_from_slice(&["depth".to_owned(), depth.to_string()]); From 9a3a779334858a2196a8328dcb76854eb3f16cc6 Mon Sep 17 00:00:00 2001 From: Niklas Fiekas Date: Sun, 12 Oct 2025 13:16:17 +0200 Subject: [PATCH 2/3] no need to mention nnue in stats reports now --- src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index e24b8513..b2d0642c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -202,11 +202,11 @@ async fn run(opt: Opt, client: &Client, logger: &Logger) { // Print summary from time to time. if now.duration_since(summarized) >= Duration::from_secs(120) { summarized = now; - let (stats, nnue_nps) = queue.stats().await; + let (stats, nps) = queue.stats().await; logger.fishnet_info(&format!( - "v{}: {} (nnue), {} batches, {} positions, {} total nodes", + "v{}: {}, {} batches, {} positions, {} total nodes", env!("CARGO_PKG_VERSION"), - nnue_nps, + nps, dot_thousands(stats.total_batches), dot_thousands(stats.total_positions), dot_thousands(stats.total_nodes), From 6dceeafc0f858eaec434c214edfea2c46ee7fa2d Mon Sep 17 00:00:00 2001 From: Niklas Fiekas Date: Sun, 12 Oct 2025 14:59:15 +0200 Subject: [PATCH 3/3] no longer publishing to crates.io --- Cargo.toml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a8d54dfe..d98f8cb2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fishnet" -version = "2.11.1-dev" # pull, test, remove dev, publish, tag, bump dev, push +version = "2.11.1-dev" # pull, remove dev, test, tag, bump dev, check, push description = "Distributed Stockfish analysis for lichess.org" repository = "https://github.com/lichess-org/fishnet" readme = "README.md" @@ -10,16 +10,7 @@ categories = ["command-line-utilities", "games"] keywords = ["chess", "lichess"] rust-version = "1.85.1" edition = "2024" -exclude = [ - "Stockfish/**/*.o", - "Stockfish/**/*.s", - "Stockfish/**/.depend", - "Stockfish/**/*.nnue", - "Fairy-Stockfish/**/*.o", - "Fairy-Stockfish/**/*.s", - "Fairy-Stockfish/**/.depend", - "Fairy-Stockfish/**/*.nnue", -] +publish = false [profile.release] strip = true