diff --git a/benches/bench.rs b/benches/bench.rs index 661359f8..3febb66b 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -28,6 +28,7 @@ use bench_source_map::{ }; use benchmark_repetitive_react_components::{ + benchmark_repetitive_react_components_index_map, benchmark_repetitive_react_components_map, benchmark_repetitive_react_components_source, }; @@ -188,6 +189,11 @@ fn bench_rspack_sources(criterion: &mut Criterion) { benchmark_repetitive_react_components_map, ); + group.bench_function( + "repetitive_react_components_index_map", + benchmark_repetitive_react_components_index_map, + ); + group.bench_function( "repetitive_react_components_source", benchmark_repetitive_react_components_source, diff --git a/benches/bench_complex_replace_source.rs b/benches/bench_complex_replace_source.rs index c3b27157..284f7b7d 100644 --- a/benches/bench_complex_replace_source.rs +++ b/benches/bench_complex_replace_source.rs @@ -9,7 +9,7 @@ pub use criterion::*; pub use codspeed_criterion_compat::*; use rspack_sources::{ - stream_chunks::StreamChunks, BoxSource, CachedSource, MapOptions, ObjectPool, + stream_chunks::ToStream, BoxSource, CachedSource, MapOptions, ObjectPool, OriginalSource, ReplaceSource, Source, SourceExt, }; @@ -36737,7 +36737,7 @@ pub fn benchmark_complex_replace_source_map_cached_source_stream_chunks( cached_source.map(&ObjectPool::default(), &MapOptions::default()); b.iter(|| { - black_box(cached_source.stream_chunks().stream( + black_box(cached_source.to_stream().chunks( &ObjectPool::default(), &MapOptions::default(), &mut |_chunk, _mapping| {}, diff --git a/benches/benchmark_repetitive_react_components.rs b/benches/benchmark_repetitive_react_components.rs index 094f107e..6f943cc7 100644 --- a/benches/benchmark_repetitive_react_components.rs +++ b/benches/benchmark_repetitive_react_components.rs @@ -3509,6 +3509,14 @@ pub fn benchmark_repetitive_react_components_map(b: &mut Bencher) { }); } +pub fn benchmark_repetitive_react_components_index_map(b: &mut Bencher) { + let source = REPETITIVE_1K_REACT_COMPONENTS_SOURCE.clone(); + + b.iter(|| { + black_box(source.index_map(&ObjectPool::default(), &MapOptions::default())); + }); +} + pub fn benchmark_repetitive_react_components_source(b: &mut Bencher) { let source = REPETITIVE_1K_REACT_COMPONENTS_SOURCE.clone(); diff --git a/src/cached_source.rs b/src/cached_source.rs index 2562c636..45e293ad 100644 --- a/src/cached_source.rs +++ b/src/cached_source.rs @@ -8,11 +8,12 @@ use rustc_hash::FxHasher; use crate::{ helpers::{ - stream_and_get_source_and_map, stream_chunks_of_raw_source, - stream_chunks_of_source_map, Chunks, GeneratedInfo, StreamChunks, + get_generated_source_info, stream_and_get_source_and_map, + stream_chunks_of_raw_source, stream_chunks_of_source_map, GeneratedInfo, + Stream, ToStream, }, object_pool::ObjectPool, - source::SourceValue, + source::{IndexSourceMap, Section, SectionOffset, SourceValue}, BoxSource, MapOptions, RawBufferSource, Source, SourceExt, SourceMap, }; @@ -23,6 +24,8 @@ struct CachedData { chunks: OnceLock>, columns_map: OnceLock>, line_only_map: OnceLock>, + columns_index_map: OnceLock>, + line_only_index_map: OnceLock>, } /// It tries to reused cached results from other methods to avoid calculations, @@ -155,31 +158,51 @@ impl Source for CachedSource { } } + fn index_map( + &self, + object_pool: &ObjectPool, + options: &MapOptions, + ) -> Option { + if options.columns { + self + .cache + .columns_index_map + .get_or_init(|| self.inner.index_map(object_pool, options)) + .clone() + } else { + self + .cache + .line_only_index_map + .get_or_init(|| self.inner.index_map(object_pool, options)) + .clone() + } + } + fn to_writer(&self, writer: &mut dyn std::io::Write) -> std::io::Result<()> { self.inner.to_writer(writer) } } -struct CachedSourceChunks<'source> { - chunks: Box, +struct CachedSourceStream<'source> { + stream: Box, cache: Arc, source: Cow<'source, str>, } -impl<'a> CachedSourceChunks<'a> { +impl<'a> CachedSourceStream<'a> { fn new(cache_source: &'a CachedSource) -> Self { let source = cache_source.source().into_string_lossy(); Self { - chunks: cache_source.inner.stream_chunks(), + stream: cache_source.inner.to_stream(), cache: cache_source.cache.clone(), source, } } } -impl Chunks for CachedSourceChunks<'_> { - fn stream<'a>( +impl Stream for CachedSourceStream<'_> { + fn chunks<'a>( &'a self, object_pool: &'a ObjectPool, options: &MapOptions, @@ -218,7 +241,7 @@ impl Chunks for CachedSourceChunks<'_> { let (generated_info, map) = stream_and_get_source_and_map( options, object_pool, - self.chunks.as_ref(), + self.stream.as_ref(), on_chunk, on_source, on_name, @@ -228,11 +251,75 @@ impl Chunks for CachedSourceChunks<'_> { } } } + + fn sections_size_hint(&self) -> usize { + if let Some(index_map) = self.cache.columns_index_map.get() { + index_map + .as_ref() + .map(|index_map| index_map.sections().len()) + .unwrap_or(0) + } else if let Some(index_map) = self.cache.line_only_index_map.get() { + index_map + .as_ref() + .map(|index_map| index_map.sections().len()) + .unwrap_or(0) + } else { + self.stream.sections_size_hint() + } + } + + fn sections<'a>( + &'a self, + object_pool: &'a ObjectPool, + columns: bool, + on_section: crate::helpers::OnSection<'_, 'a>, + ) -> GeneratedInfo { + let cell = if columns { + &self.cache.columns_index_map + } else { + &self.cache.line_only_index_map + }; + match cell.get() { + Some(index_map) => { + let generated_info = get_generated_source_info(self.source.as_ref()); + if let Some(index_map) = index_map { + for section in index_map.sections() { + on_section(section.offset, Some(section.map.clone())); + } + } else { + on_section(SectionOffset::default(), None); + } + generated_info + } + None => { + let mut sections = Vec::new(); + let generated_info = + self + .stream + .sections(object_pool, columns, &mut |offset, map| { + if let Some(ref map) = map { + sections.push(Section { + offset, + map: map.clone(), + }); + } + on_section(offset, map); + }); + let index_map = if sections.is_empty() { + None + } else { + Some(IndexSourceMap::new(sections)) + }; + cell.get_or_init(|| index_map); + generated_info + } + } + } } -impl StreamChunks for CachedSource { - fn stream_chunks<'a>(&'a self) -> Box { - Box::new(CachedSourceChunks::new(self)) +impl ToStream for CachedSource { + fn to_stream<'a>(&'a self) -> Box { + Box::new(CachedSourceStream::new(self)) } } @@ -387,8 +474,8 @@ mod tests { let mut on_name_count = 0; let generated_info = { let object_pool = ObjectPool::default(); - let chunks = source.stream_chunks(); - chunks.stream( + let stream = source.to_stream(); + stream.chunks( &object_pool, &map_options, &mut |_chunk, _mapping| { @@ -404,7 +491,7 @@ mod tests { }; let cached_source = CachedSource::new(source); - cached_source.stream_chunks().stream( + cached_source.to_stream().chunks( &ObjectPool::default(), &map_options, &mut |_chunk, _mapping| {}, @@ -415,7 +502,7 @@ mod tests { let mut cached_on_chunk_count = 0; let mut cached_on_source_count = 0; let mut cached_on_name_count = 0; - let cached_generated_info = cached_source.stream_chunks().stream( + let cached_generated_info = cached_source.to_stream().chunks( &ObjectPool::default(), &map_options, &mut |_chunk, _mapping| { @@ -477,4 +564,41 @@ mod tests { let cached_size = cached.size(); assert_eq!(raw_size, cached_size); } + + #[test] + fn index_map_should_be_cached() { + let original = OriginalSource::new("hello\nworld\n", "test.txt"); + let cached = CachedSource::new(original); + let pool = ObjectPool::default(); + let options = MapOptions::default(); + + let index_map1 = cached.index_map(&pool, &options); + let index_map2 = cached.index_map(&pool, &options); + assert!(index_map1.is_some()); + assert_eq!(index_map1, index_map2); + } + + #[test] + fn index_map_cached_matches_inner() { + let inner = ConcatSource::new([ + OriginalSource::new("a\n", "a.js").boxed(), + OriginalSource::new("b\n", "b.js").boxed(), + ]); + let pool = ObjectPool::default(); + let options = MapOptions::default(); + let expected = inner.index_map(&pool, &options); + let cached = CachedSource::new(inner); + let result = cached.index_map(&pool, &options); + assert_eq!(result, expected); + } + + #[test] + fn index_map_returns_none_for_raw_cached() { + let cached = CachedSource::new(RawStringSource::from("no map")); + let pool = ObjectPool::default(); + let options = MapOptions::default(); + assert!(cached.index_map(&pool, &options).is_none()); + // Second call also returns None (from cache) + assert!(cached.index_map(&pool, &options).is_none()); + } } diff --git a/src/concat_source.rs b/src/concat_source.rs index 8a7ce39a..ff18babe 100644 --- a/src/concat_source.rs +++ b/src/concat_source.rs @@ -8,10 +8,10 @@ use std::{ use rustc_hash::FxHashMap as HashMap; use crate::{ - helpers::{get_map, Chunks, GeneratedInfo, StreamChunks}, + helpers::{get_map, GeneratedInfo, Stream, ToStream}, linear_map::LinearMap, object_pool::ObjectPool, - source::{Mapping, OriginalLocation}, + source::{IndexSourceMap, Mapping, OriginalLocation, Section}, BoxSource, MapOptions, RawStringSource, Source, SourceExt, SourceMap, SourceValue, }; @@ -211,9 +211,27 @@ impl Source for ConcatSource { object_pool: &'a ObjectPool, options: &MapOptions, ) -> Option { - let chunks = self.stream_chunks(); - let result = get_map(object_pool, chunks.as_ref(), options); - result + let stream = self.to_stream(); + get_map(object_pool, stream.as_ref(), options).1 + } + + fn index_map( + &self, + object_pool: &ObjectPool, + options: &MapOptions, + ) -> Option { + let stream = self.to_stream(); + let mut sections = Vec::with_capacity(stream.sections_size_hint()); + stream.sections(object_pool, options.columns, &mut |offset, map| { + if let Some(map) = map { + sections.push(Section { offset, map }); + } + }); + if sections.is_empty() { + None + } else { + Some(IndexSourceMap::new(sections)) + } } fn to_writer(&self, writer: &mut dyn std::io::Write) -> std::io::Result<()> { @@ -240,23 +258,22 @@ impl PartialEq for ConcatSource { } impl Eq for ConcatSource {} -struct ConcatSourceChunks<'source> { - children_chunks: Vec>, +struct ConcatSourceStream<'source> { + children_streams: Vec>, } -impl<'source> ConcatSourceChunks<'source> { - fn new(concat_source: &'source ConcatSource) -> Self { - let children = concat_source.optimized_children(); - let children_chunks = children +impl<'source> ConcatSourceStream<'source> { + fn new(children: &'source [BoxSource]) -> Self { + let children_streams = children .iter() - .map(|child| child.stream_chunks()) + .map(|child| child.to_stream()) .collect::>(); - Self { children_chunks } + Self { children_streams } } } -impl Chunks for ConcatSourceChunks<'_> { - fn stream<'b>( +impl Stream for ConcatSourceStream<'_> { + fn chunks<'b>( &'b self, object_pool: &'b ObjectPool, options: &MapOptions, @@ -264,8 +281,8 @@ impl Chunks for ConcatSourceChunks<'_> { on_source: crate::helpers::OnSource<'_, 'b>, on_name: crate::helpers::OnName<'_, 'b>, ) -> GeneratedInfo { - if self.children_chunks.len() == 1 { - return self.children_chunks[0].stream( + if self.children_streams.len() == 1 { + return self.children_streams[0].chunks( object_pool, options, on_chunk, @@ -276,7 +293,7 @@ impl Chunks for ConcatSourceChunks<'_> { let mut current_line_offset = 0; let mut current_column_offset = 0; let mut source_mapping: HashMap, u32> = HashMap::default(); - let mut name_mapping: HashMap, u32> = HashMap::default(); + let mut name_mapping: HashMap<&str, u32> = HashMap::default(); let mut need_to_close_mapping = false; let source_index_mapping: RefCell> = @@ -284,14 +301,14 @@ impl Chunks for ConcatSourceChunks<'_> { let name_index_mapping: RefCell> = RefCell::new(LinearMap::default()); - for child_handle in &self.children_chunks { + for child_stream in &self.children_streams { source_index_mapping.borrow_mut().clear(); name_index_mapping.borrow_mut().clear(); let mut last_mapping_line = 0; let GeneratedInfo { generated_line, generated_column, - } = child_handle.stream( + } = child_stream.chunks( object_pool, options, &mut |chunk, mapping| { @@ -394,7 +411,7 @@ impl Chunks for ConcatSourceChunks<'_> { let mut global_index = name_mapping.get(&name).copied(); if global_index.is_none() { let len = name_mapping.len() as u32; - name_mapping.insert(name.clone(), len); + name_mapping.insert(name, len); on_name(len, name); global_index = Some(len); } @@ -429,11 +446,53 @@ impl Chunks for ConcatSourceChunks<'_> { generated_column: current_column_offset, } } + + fn sections_size_hint(&self) -> usize { + self + .children_streams + .iter() + .map(|child_stream| child_stream.sections_size_hint()) + .sum() + } + + fn sections<'a>( + &'a self, + object_pool: &'a ObjectPool, + columns: bool, + on_section: crate::helpers::OnSection<'_, 'a>, + ) -> GeneratedInfo { + let mut current_generated_info = GeneratedInfo { + generated_line: 1, + generated_column: 0, + }; + + for child_stream in &self.children_streams { + let generated_info = child_stream.sections( + object_pool, + columns, + &mut |mut offset, mapping| { + offset.line += current_generated_info.generated_line - 1; + offset.column += current_generated_info.generated_column; + on_section(offset, mapping); + }, + ); + current_generated_info.generated_line += + generated_info.generated_line - 1; + current_generated_info.generated_column = generated_info.generated_column; + } + current_generated_info + } } -impl StreamChunks for ConcatSource { - fn stream_chunks<'a>(&'a self) -> Box { - Box::new(ConcatSourceChunks::new(self)) +impl ToStream for ConcatSource { + fn to_stream<'a>(&'a self) -> Box { + let children = self.optimized_children(); + // Fast path: delegate directly to the single child's stream, + // avoiding ConcatSourceStream + Vec + extra Box allocations. + if children.len() == 1 { + return children[0].to_stream(); + } + Box::new(ConcatSourceStream::new(children)) } } @@ -464,6 +523,7 @@ fn optimize(children: &mut Vec) -> Vec { } /// Helper function to merge and flush pending raw sources. +#[inline(always)] fn merge_raw_sources( raw_sources: &mut Vec, new_children: &mut Vec, @@ -479,7 +539,7 @@ fn merge_raw_sources( let capacity = raw_sources.iter().map(|s| s.size()).sum(); let mut merged_content = String::with_capacity(capacity); for source in raw_sources.drain(..) { - merged_content.push_str(source.source().into_string_lossy().as_ref()); + source.rope(&mut |chunk| merged_content.push_str(chunk)); } let merged_source = RawStringSource::from(merged_content); new_children.push(merged_source.boxed()); @@ -493,6 +553,251 @@ mod tests { use super::*; + #[test] + fn index_map_returns_none_for_only_raw_sources() { + let source = ConcatSource::new([ + RawStringSource::from("Hello World\n").boxed(), + RawStringSource::from("Bye\n").boxed(), + ]); + let pool = ObjectPool::default(); + let options = MapOptions::default(); + assert!(source.index_map(&pool, &options).is_none()); + } + + #[test] + fn index_map_single_original_source_child() { + let source = ConcatSource::new([OriginalSource::new( + "console.log('test');\n", + "test.js", + ) + .boxed()]); + let pool = ObjectPool::default(); + let options = MapOptions::default(); + let index_map = source.index_map(&pool, &options).unwrap(); + // Single child -> delegates to child's index_map (1 section, offset 0,0) + assert_eq!(index_map.sections().len(), 1); + assert_eq!(index_map.sections()[0].offset.line, 0); + assert_eq!(index_map.sections()[0].offset.column, 0); + // The flattened source map should equal the child's map + let map = source.map(&pool, &options).unwrap(); + assert_eq!(index_map.to_source_map().unwrap(), map); + } + + #[test] + fn index_map_concat_two_original_sources() { + let source = ConcatSource::new([ + OriginalSource::new("line1\n", "a.js").boxed(), + OriginalSource::new("line2\n", "b.js").boxed(), + ]); + let pool = ObjectPool::default(); + let options = MapOptions::default(); + let index_map = source.index_map(&pool, &options).unwrap(); + assert_eq!(index_map.sections().len(), 2); + // First section at 0,0 + assert_eq!(index_map.sections()[0].offset.line, 0); + assert_eq!(index_map.sections()[0].offset.column, 0); + // Second section at line 1 (after "line1\n") + assert_eq!(index_map.sections()[1].offset.line, 1); + assert_eq!(index_map.sections()[1].offset.column, 0); + + // Flattened should match regular map + let flat = index_map.to_source_map().unwrap(); + let map = source.map(&pool, &options).unwrap(); + assert_eq!(flat.sources(), map.sources()); + assert_eq!(flat.sources_content(), map.sources_content()); + } + + #[test] + fn index_map_with_raw_prefix() { + // RawStringSource (no map) followed by OriginalSource (has map) + let source = ConcatSource::new([ + RawStringSource::from("// header\n").boxed(), + OriginalSource::new( + "console.log('test');\nconsole.log('test2');\n", + "console.js", + ) + .boxed(), + ]); + let pool = ObjectPool::default(); + let options = MapOptions::default(); + let index_map = source.index_map(&pool, &options).unwrap(); + // Only one section (from the OriginalSource), offset by 1 line + assert_eq!(index_map.sections().len(), 1); + assert_eq!(index_map.sections()[0].offset.line, 1); + assert_eq!(index_map.sections()[0].offset.column, 0); + } + + #[test] + fn index_map_with_raw_suffix() { + // OriginalSource followed by RawStringSource + let source = ConcatSource::new([ + OriginalSource::new("hello\n", "a.js").boxed(), + RawStringSource::from("// footer\n").boxed(), + ]); + let pool = ObjectPool::default(); + let options = MapOptions::default(); + let index_map = source.index_map(&pool, &options).unwrap(); + assert_eq!(index_map.sections().len(), 1); + assert_eq!(index_map.sections()[0].offset.line, 0); + assert_eq!(index_map.sections()[0].offset.column, 0); + } + + #[test] + fn index_map_same_line_concat() { + // Two sources on the same line (no trailing newline in first) + let source = ConcatSource::new([ + OriginalSource::new("hello", "a.js").boxed(), + OriginalSource::new(" world", "b.js").boxed(), + ]); + let pool = ObjectPool::default(); + let options = MapOptions::default(); + let index_map = source.index_map(&pool, &options).unwrap(); + assert_eq!(index_map.sections().len(), 2); + // First at 0,0 + assert_eq!(index_map.sections()[0].offset.line, 0); + assert_eq!(index_map.sections()[0].offset.column, 0); + // Second at 0,5 (same line, column 5 = length of "hello") + assert_eq!(index_map.sections()[1].offset.line, 0); + assert_eq!(index_map.sections()[1].offset.column, 5); + } + + #[test] + fn index_map_mixed_raw_and_original_sources() { + let source = ConcatSource::new([ + RawStringSource::from("Hello World\n").boxed(), + OriginalSource::new( + "console.log('test');\nconsole.log('test2');\n", + "console.js", + ) + .boxed(), + OriginalSource::new("Hello2\n", "hello.md").boxed(), + ]); + let pool = ObjectPool::default(); + let options = MapOptions::new(false); + let index_map = source.index_map(&pool, &options).unwrap(); + + // Two sections (from the two OriginalSources) + assert_eq!(index_map.sections().len(), 2); + + // First OriginalSource starts after "Hello World\n" -> line offset 1 + assert_eq!(index_map.sections()[0].offset.line, 1); + assert_eq!(index_map.sections()[0].offset.column, 0); + + // Second OriginalSource starts after the first one's 2 lines + // "Hello World\n" (1 line) + "console.log('test');\nconsole.log('test2');\n" (2 lines) = 3 lines + assert_eq!(index_map.sections()[1].offset.line, 3); + assert_eq!(index_map.sections()[1].offset.column, 0); + } + + #[test] + fn index_map_to_source_map_matches_regular_map() { + // Comprehensive test: the flattened IndexSourceMap should produce + // equivalent mappings to the regular map() method + let mut source = ConcatSource::new([ + RawStringSource::from("Hello World\n".to_string()).boxed(), + OriginalSource::new( + "console.log('test');\nconsole.log('test2');\n", + "console.js", + ) + .boxed(), + ]); + source.add(OriginalSource::new("Hello2\n", "hello.md")); + + let pool = ObjectPool::default(); + let options = MapOptions::new(false); + + let regular_map = source.map(&pool, &options).unwrap(); + let index_map = source.index_map(&pool, &options).unwrap(); + let flat_map = index_map.to_source_map().unwrap(); + + // Sources should match + assert_eq!(flat_map.sources(), regular_map.sources()); + assert_eq!(flat_map.sources_content(), regular_map.sources_content()); + + // Decoded mappings should match + let regular_mappings: Vec = + regular_map.decoded_mappings().collect(); + let flat_mappings: Vec = flat_map.decoded_mappings().collect(); + assert_eq!(regular_mappings.len(), flat_mappings.len()); + for (r, f) in regular_mappings.iter().zip(flat_mappings.iter()) { + assert_eq!(r.generated_line, f.generated_line); + assert_eq!(r.generated_column, f.generated_column); + assert_eq!( + r.original.as_ref().map(|o| o.source_index), + f.original.as_ref().map(|o| o.source_index) + ); + assert_eq!( + r.original.as_ref().map(|o| o.original_line), + f.original.as_ref().map(|o| o.original_line) + ); + assert_eq!( + r.original.as_ref().map(|o| o.original_column), + f.original.as_ref().map(|o| o.original_column) + ); + } + } + + #[test] + fn index_map_nested_concat_source() { + // Nested ConcatSource should flatten sections + let inner = ConcatSource::new([ + OriginalSource::new("a\n", "a.js").boxed(), + OriginalSource::new("b\n", "b.js").boxed(), + ]); + let outer = ConcatSource::new([ + inner.boxed(), + OriginalSource::new("c\n", "c.js").boxed(), + ]); + + let pool = ObjectPool::default(); + let options = MapOptions::default(); + let index_map = outer.index_map(&pool, &options).unwrap(); + + // Inner concat should contribute 2 sections, outer adds 1 = 3 total + assert_eq!(index_map.sections().len(), 3); + + // Verify offsets + assert_eq!(index_map.sections()[0].offset.line, 0); + assert_eq!(index_map.sections()[0].offset.column, 0); + assert_eq!(index_map.sections()[1].offset.line, 1); // after "a\n" + assert_eq!(index_map.sections()[1].offset.column, 0); + assert_eq!(index_map.sections()[2].offset.line, 2); // after "a\n" + "b\n" + assert_eq!(index_map.sections()[2].offset.column, 0); + + // Verify sources + assert_eq!(index_map.sections()[0].map.sources(), &["a.js".to_string()]); + assert_eq!(index_map.sections()[1].map.sources(), &["b.js".to_string()]); + assert_eq!(index_map.sections()[2].map.sources(), &["c.js".to_string()]); + + // Flattened should match regular map + let regular_map = outer.map(&pool, &options).unwrap(); + let flat_map = index_map.to_source_map().unwrap(); + assert_eq!(flat_map.sources(), regular_map.sources()); + let regular_mappings: Vec = + regular_map.decoded_mappings().collect(); + let flat_mappings: Vec = flat_map.decoded_mappings().collect(); + assert_eq!(regular_mappings.len(), flat_mappings.len()); + for (r, f) in regular_mappings.iter().zip(flat_mappings.iter()) { + assert_eq!(r.generated_line, f.generated_line); + assert_eq!(r.generated_column, f.generated_column); + } + } + + #[test] + fn index_map_with_empty_children() { + let source = ConcatSource::new([ + OriginalSource::new("hello\n", "a.js").boxed(), + RawStringSource::from("").boxed(), + OriginalSource::new("world\n", "b.js").boxed(), + ]); + let pool = ObjectPool::default(); + let options = MapOptions::default(); + let index_map = source.index_map(&pool, &options).unwrap(); + assert_eq!(index_map.sections().len(), 2); + assert_eq!(index_map.sections()[0].offset.line, 0); + assert_eq!(index_map.sections()[1].offset.line, 1); + } + #[test] fn should_concat_two_sources() { let mut source = ConcatSource::new([ diff --git a/src/helpers.rs b/src/helpers.rs index 4a793358..0a40e28c 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -15,20 +15,20 @@ use crate::{ source::{Mapping, OriginalLocation}, source_content_lines::SourceContentLines, with_utf16::WithUtf16, - MapOptions, SourceMap, + MapOptions, SectionOffset, SourceMap, }; pub fn get_map<'a>( object_pool: &'a ObjectPool, - chunks: &'a dyn Chunks, + stream: &'a dyn Stream, options: &MapOptions, -) -> Option { +) -> (GeneratedInfo, Option) { let mut mappings_encoder = create_encoder(options.columns); let mut sources: Vec = Vec::new(); let mut sources_content: Vec> = Vec::new(); let mut names: Vec = Vec::new(); - chunks.stream( + let generated_info = stream.chunks( object_pool, &MapOptions { columns: options.columns, @@ -61,9 +61,12 @@ pub fn get_map<'a>( names[name_index] = name.to_string(); }, ); + let mappings = mappings_encoder.drain(); - (!mappings.is_empty()) - .then(|| SourceMap::new(mappings, sources, sources_content, names)) + let map = (!mappings.is_empty()) + .then(|| SourceMap::new(mappings, sources, sources_content, names)); + + (generated_info, map) } /// A trait for processing source code chunks and generating source maps. @@ -72,13 +75,13 @@ pub fn get_map<'a>( /// while building source map information. It's designed to handle the transformation /// of source code into mappings that connect generated code positions to original /// source positions. -pub trait Chunks { +pub trait Stream { /// Streams through source code chunks and generates source map information. /// /// This method processes the source code in chunks, calling the provided callbacks /// for each chunk, source reference, and name reference encountered. It's the core /// method for building source maps during code transformation. - fn stream<'a>( + fn chunks<'a>( &'a self, object_pool: &'a ObjectPool, options: &MapOptions, @@ -86,12 +89,24 @@ pub trait Chunks { on_source: crate::helpers::OnSource<'_, 'a>, on_name: crate::helpers::OnName<'_, 'a>, ) -> crate::helpers::GeneratedInfo; + + /// Returns an estimated upper bound of sections that [`Stream::sections`] will produce. + fn sections_size_hint(&self) -> usize; + + /// Streams source map data as discrete sections, calling `on_section` for + /// each section with its offset and optional [`SourceMap`]. + fn sections<'a>( + &'a self, + object_pool: &'a ObjectPool, + columns: bool, + on_section: crate::helpers::OnSection<'_, 'a>, + ) -> crate::helpers::GeneratedInfo; } -/// [StreamChunks] abstraction, see [webpack-sources source.streamChunks](https://github.com/webpack/webpack-sources/blob/9f98066311d53a153fdc7c633422a1d086528027/lib/helpers/streamChunks.js#L13). -pub trait StreamChunks { - /// [StreamChunks] abstraction - fn stream_chunks<'a>(&'a self) -> Box; +/// [ToStream] abstraction, see [webpack-sources source.streamChunks](https://github.com/webpack/webpack-sources/blob/9f98066311d53a153fdc7c633422a1d086528027/lib/helpers/streamChunks.js#L13). +pub trait ToStream { + /// [ToStream] abstraction + fn to_stream<'a>(&'a self) -> Box; } /// [OnChunk] abstraction, see [webpack-sources onChunk](https://github.com/webpack/webpack-sources/blob/9f98066311d53a153fdc7c633422a1d086528027/lib/helpers/streamChunks.js#L13). @@ -103,7 +118,12 @@ pub type OnSource<'a, 'b> = &'a mut dyn FnMut(u32, Cow<'b, str>, Option<&'b Arc>); /// [OnName] abstraction, see [webpack-sources onName](https://github.com/webpack/webpack-sources/blob/9f98066311d53a153fdc7c633422a1d086528027/lib/helpers/streamChunks.js#L13). -pub type OnName<'a, 'b> = &'a mut dyn FnMut(u32, Cow<'b, str>); +pub type OnName<'a, 'b> = &'a mut dyn FnMut(u32, &'b str); + +/// Callback invoked for each section during [`Stream::sections`], receiving the +/// section's [`SectionOffset`] and an optional [`SourceMap`]. +pub type OnSection<'a, 'b> = + &'a mut dyn FnMut(SectionOffset, Option); /// Default stream chunks behavior impl, see [webpack-sources streamChunks](https://github.com/webpack/webpack-sources/blob/9f98066311d53a153fdc7c633422a1d086528027/lib/helpers/streamChunks.js#L15-L35). pub fn stream_chunks_default<'a>( @@ -131,7 +151,7 @@ pub fn stream_chunks_default<'a>( } /// `GeneratedSourceInfo` abstraction, see [webpack-sources GeneratedSourceInfo](https://github.com/webpack/webpack-sources/blob/9f98066311d53a153fdc7c633422a1d086528027/lib/helpers/getGeneratedSourceInfo.js) -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub struct GeneratedInfo { /// Generated line pub generated_line: u32, @@ -278,22 +298,34 @@ pub fn split_into_lines(source: &str) -> impl Iterator { split(source, b'\n') } +/// Computes the [`GeneratedInfo`] (line and column) for the end position of the given source string. +/// +/// See [webpack-sources getGeneratedSourceInfo](https://github.com/webpack/webpack-sources/blob/9f98066311d53a153fdc7c633422a1d086528027/lib/helpers/getGeneratedSourceInfo.js). pub fn get_generated_source_info(source: &str) -> GeneratedInfo { - let (generated_line, generated_column) = if source.ends_with('\n') { - (split_into_lines(source).count() + 1, 0) - } else { - let mut line_count = 0; - let mut last_line = ""; + let bytes = source.as_bytes(); - for line in split_into_lines(source) { - line_count += 1; - last_line = line; - } + let mut line_count = 0; + let mut last_newline_pos = None; - (line_count.max(1), last_line.encode_utf16().count()) + for pos in memchr::memchr_iter(b'\n', bytes) { + line_count += 1; + last_newline_pos = Some(pos); + } + + let generated_column = if let Some(pos) = last_newline_pos { + if pos == bytes.len() - 1 { + 0 + } else { + #[allow(unsafe_code)] + let last_line_slice = unsafe { source.get_unchecked(pos + 1..) }; + last_line_slice.chars().map(|c| c.len_utf16()).sum() + } + } else { + source.chars().map(|c| c.len_utf16()).sum() }; + GeneratedInfo { - generated_line: generated_line as u32, + generated_line: line_count + 1, generated_column: generated_column as u32, } } @@ -413,7 +445,7 @@ fn stream_chunks_of_source_map_final<'a>( ) } for (i, name) in source_map.names().iter().enumerate() { - on_name(i as u32, Cow::Borrowed(name)); + on_name(i as u32, name); } let mut mapping_active_line = 0; let mut on_mapping = |mapping: Mapping| { @@ -476,7 +508,7 @@ fn stream_chunks_of_source_map_full<'a>( ) } for (i, name) in source_map.names().iter().enumerate() { - on_name(i as u32, Cow::Borrowed(name)); + on_name(i as u32, name); } let last_line = &lines[lines.len() - 1].line; let last_new_line = last_line.ends_with('\n'); @@ -757,12 +789,12 @@ pub fn stream_chunks_of_combined_source_map<'a>( let inner_source: RefCell>> = RefCell::new(inner_source); let source_mapping: RefCell, u32>> = RefCell::new(HashMap::default()); - let mut name_mapping: HashMap, u32> = HashMap::default(); + let mut name_mapping: HashMap<&str, u32> = HashMap::default(); let source_index_mapping: RefCell> = RefCell::new(LinearMap::default()); let name_index_mapping: RefCell> = RefCell::new(LinearMap::default()); - let name_index_value_mapping: RefCell>> = + let name_index_value_mapping: RefCell> = RefCell::new(LinearMap::default()); let inner_source_index: RefCell = RefCell::new(-2); let inner_source_index_mapping: RefCell> = @@ -776,7 +808,7 @@ pub fn stream_chunks_of_combined_source_map<'a>( > = RefCell::new(LinearMap::default()); let inner_name_index_mapping: RefCell> = RefCell::new(LinearMap::default()); - let inner_name_index_value_mapping: RefCell>> = + let inner_name_index_value_mapping: RefCell> = RefCell::new(LinearMap::default()); let inner_source_map_line_data: RefCell> = RefCell::new(Vec::new()); @@ -939,8 +971,8 @@ pub fn stream_chunks_of_combined_source_map<'a>( let mut global_index = name_mapping.get(name).copied(); if global_index.is_none() { let len = name_mapping.len() as u32; - name_mapping.insert(name.clone(), len); - on_name(len, name.clone()); + name_mapping.insert(name, len); + on_name(len, name); global_index = Some(len); } final_name_index = global_index.unwrap() as i64; @@ -996,8 +1028,8 @@ pub fn stream_chunks_of_combined_source_map<'a>( let mut global_index = name_mapping.get(name).copied(); if global_index.is_none() { let len = name_mapping.len() as u32; - name_mapping.insert(name.clone(), len); - on_name(len, name.clone()); + name_mapping.insert(name, len); + on_name(len, name); global_index = Some(len); } final_name_index = global_index.unwrap() as i64; @@ -1096,8 +1128,8 @@ pub fn stream_chunks_of_combined_source_map<'a>( let mut global_index = name_mapping.get(name).copied(); if global_index.is_none() { let len = name_mapping.len() as u32; - name_mapping.borrow_mut().insert(name.clone(), len); - on_name(len, name.clone()); + name_mapping.borrow_mut().insert(name, len); + on_name(len, name); global_index = Some(len); } final_name_index = global_index.unwrap() as i64; @@ -1229,7 +1261,7 @@ pub fn stream_chunks_of_combined_source_map<'a>( pub fn stream_and_get_source_and_map<'a>( options: &MapOptions, object_pool: &'a ObjectPool, - chunks: &'a dyn Chunks, + stream: &'a dyn Stream, on_chunk: OnChunk<'_, 'a>, on_source: OnSource<'_, 'a>, on_name: OnName<'_, 'a>, @@ -1239,7 +1271,7 @@ pub fn stream_and_get_source_and_map<'a>( let mut sources_content: Vec> = Vec::new(); let mut names: Vec = Vec::new(); - let generated_info = chunks.stream( + let generated_info = stream.chunks( object_pool, options, &mut |chunk, mapping| { diff --git a/src/lib.rs b/src/lib.rs index c5dca46a..6493876d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,21 +23,23 @@ pub use original_source::OriginalSource; pub use raw_source::{RawBufferSource, RawStringSource}; pub use replace_source::{ReplaceSource, ReplacementEnforce}; pub use source::{ - BoxSource, MapOptions, Mapping, OriginalLocation, Source, SourceExt, - SourceMap, SourceValue, + BoxSource, IndexSourceMap, MapOptions, Mapping, OriginalLocation, Section, + SectionOffset, Source, SourceExt, SourceMap, SourceValue, }; pub use source_map_source::{ SourceMapSource, SourceMapSourceOptions, WithoutOriginalOptions, }; -/// Reexport `StreamChunks` related types. +/// Reexport `ToStream` related types. pub mod stream_chunks { pub use super::helpers::{ - stream_chunks_default, Chunks, GeneratedInfo, OnChunk, OnName, OnSource, - StreamChunks, + stream_chunks_default, GeneratedInfo, OnChunk, OnName, OnSection, OnSource, + Stream, ToStream, }; } -pub use helpers::{decode_mappings, encode_mappings}; +pub use helpers::{ + decode_mappings, encode_mappings, get_generated_source_info, +}; pub use object_pool::ObjectPool; diff --git a/src/original_source.rs b/src/original_source.rs index e800e7dc..dc882f5a 100644 --- a/src/original_source.rs +++ b/src/original_source.rs @@ -7,11 +7,11 @@ use std::{ use crate::{ helpers::{ get_generated_source_info, get_map, split_into_lines, - split_into_potential_tokens, Chunks, GeneratedInfo, StreamChunks, + split_into_potential_tokens, GeneratedInfo, Stream, ToStream, }, object_pool::ObjectPool, source::{Mapping, OriginalLocation}, - MapOptions, Source, SourceMap, SourceValue, + MapOptions, SectionOffset, Source, SourceMap, SourceValue, }; /// Represents source code, it will create source map for the source code, @@ -73,8 +73,8 @@ impl Source for OriginalSource { object_pool: &ObjectPool, options: &MapOptions, ) -> Option { - let chunks = self.stream_chunks(); - get_map(object_pool, chunks.as_ref(), options) + let stream = self.to_stream(); + get_map(object_pool, stream.as_ref(), options).1 } fn to_writer(&self, writer: &mut dyn std::io::Write) -> std::io::Result<()> { @@ -111,16 +111,16 @@ impl std::fmt::Debug for OriginalSource { } } -struct OriginalSourceChunks<'a>(&'a OriginalSource); +struct OriginalSourceStream<'a>(&'a OriginalSource); -impl<'source> OriginalSourceChunks<'source> { +impl<'source> OriginalSourceStream<'source> { pub fn new(source: &'source OriginalSource) -> Self { Self(source) } } -impl Chunks for OriginalSourceChunks<'_> { - fn stream<'b>( +impl Stream for OriginalSourceStream<'_> { + fn chunks<'b>( &'b self, _object_pool: &'b ObjectPool, options: &MapOptions, @@ -247,11 +247,33 @@ impl Chunks for OriginalSourceChunks<'_> { } } } + + fn sections_size_hint(&self) -> usize { + 1 + } + + fn sections<'a>( + &'a self, + object_pool: &'a ObjectPool, + columns: bool, + on_section: crate::helpers::OnSection<'_, 'a>, + ) -> GeneratedInfo { + let (generated_info, map) = get_map( + object_pool, + self, + &MapOptions { + columns, + final_source: true, + }, + ); + on_section(SectionOffset::default(), map); + generated_info + } } -impl StreamChunks for OriginalSource { - fn stream_chunks<'a>(&'a self) -> Box { - Box::new(OriginalSourceChunks::new(self)) +impl ToStream for OriginalSource { + fn to_stream<'a>(&'a self) -> Box { + Box::new(OriginalSourceStream::new(self)) } } @@ -365,8 +387,8 @@ mod tests { let source = OriginalSource::new(code, "test.js"); let mut chunks = vec![]; let object_pool = ObjectPool::default(); - let handle = source.stream_chunks(); - let generated_info = handle.stream( + let handle = source.to_stream(); + let generated_info = handle.chunks( &object_pool, &MapOptions::default(), &mut |chunk, mapping| { diff --git a/src/raw_source.rs b/src/raw_source.rs index c7fb29c9..71afb102 100644 --- a/src/raw_source.rs +++ b/src/raw_source.rs @@ -6,11 +6,11 @@ use std::{ use crate::{ helpers::{ - get_generated_source_info, stream_chunks_of_raw_source, Chunks, - GeneratedInfo, StreamChunks, + get_generated_source_info, stream_chunks_of_raw_source, GeneratedInfo, + Stream, ToStream, }, object_pool::ObjectPool, - MapOptions, Source, SourceMap, SourceValue, + MapOptions, SectionOffset, Source, SourceMap, SourceValue, }; /// A string variant of [RawStringSource]. @@ -107,16 +107,16 @@ impl Hash for RawStringSource { } } -struct RawStringChunks<'source>(&'source str); +struct RawStringStream<'source>(&'source str); -impl<'source> RawStringChunks<'source> { +impl<'source> RawStringStream<'source> { pub fn new(source: &'source RawStringSource) -> Self { - RawStringChunks(&source.0) + RawStringStream(&source.0) } } -impl Chunks for RawStringChunks<'_> { - fn stream<'a>( +impl Stream for RawStringStream<'_> { + fn chunks<'a>( &'a self, _object_pool: &'a ObjectPool, options: &MapOptions, @@ -130,11 +130,27 @@ impl Chunks for RawStringChunks<'_> { stream_chunks_of_raw_source(self.0, options, on_chunk, on_source, on_name) } } + + fn sections_size_hint(&self) -> usize { + 0 + } + + fn sections<'a>( + &'a self, + _object_pool: &'a ObjectPool, + _columns: bool, + on_section: crate::helpers::OnSection<'_, 'a>, + ) -> GeneratedInfo { + let generated_info = get_generated_source_info(self.0); + on_section(SectionOffset::default(), None); + generated_info + } } -impl StreamChunks for RawStringSource { - fn stream_chunks<'a>(&'a self) -> Box { - Box::new(RawStringChunks::new(self)) +impl ToStream for RawStringSource { + #[inline] + fn to_stream<'a>(&'a self) -> Box { + Box::new(RawStringStream::new(self)) } } @@ -253,10 +269,10 @@ impl Hash for RawBufferSource { } } -struct RawBufferSourceChunks<'a>(&'a RawBufferSource); +struct RawBufferSourceStream<'a>(&'a RawBufferSource); -impl Chunks for RawBufferSourceChunks<'_> { - fn stream<'a>( +impl Stream for RawBufferSourceStream<'_> { + fn chunks<'a>( &'a self, _object_pool: &'a ObjectPool, options: &MapOptions, @@ -271,11 +287,27 @@ impl Chunks for RawBufferSourceChunks<'_> { stream_chunks_of_raw_source(code, options, on_chunk, on_source, on_name) } } + + fn sections_size_hint(&self) -> usize { + 0 + } + + fn sections<'a>( + &'a self, + _object_pool: &'a ObjectPool, + _columns: bool, + on_section: crate::helpers::OnSection<'_, 'a>, + ) -> GeneratedInfo { + let code = self.0.get_or_init_value_as_string(); + let generated_info = get_generated_source_info(code); + on_section(SectionOffset::default(), None); + generated_info + } } -impl StreamChunks for RawBufferSource { - fn stream_chunks<'a>(&'a self) -> Box { - Box::new(RawBufferSourceChunks(self)) +impl ToStream for RawBufferSource { + fn to_stream<'a>(&'a self) -> Box { + Box::new(RawBufferSourceStream(self)) } } diff --git a/src/replace_source.rs b/src/replace_source.rs index c1a673e3..94cab4aa 100644 --- a/src/replace_source.rs +++ b/src/replace_source.rs @@ -8,12 +8,12 @@ use std::{ use rustc_hash::FxHashMap as HashMap; use crate::{ - helpers::{get_map, split_into_lines, Chunks, GeneratedInfo, StreamChunks}, + helpers::{get_map, split_into_lines, GeneratedInfo, Stream, ToStream}, linear_map::LinearMap, object_pool::ObjectPool, source_content_lines::SourceContentLines, - BoxSource, MapOptions, Mapping, OriginalLocation, OriginalSource, Source, - SourceExt, SourceMap, SourceValue, + BoxSource, MapOptions, Mapping, OriginalLocation, OriginalSource, + SectionOffset, Source, SourceExt, SourceMap, SourceValue, }; /// Decorates a Source with replacements and insertions of source code, @@ -325,8 +325,8 @@ impl Source for ReplaceSource { if replacements.is_empty() { return self.inner.map(&ObjectPool::default(), options); } - let chunks = self.stream_chunks(); - get_map(&ObjectPool::default(), chunks.as_ref(), options) + let stream = self.to_stream(); + get_map(&ObjectPool::default(), stream.as_ref(), options).1 } fn to_writer(&self, writer: &mut dyn std::io::Write) -> std::io::Result<()> { @@ -403,26 +403,26 @@ fn check_content_at_position( } } -struct ReplaceSourceChunks<'a> { +struct ReplaceSourceStream<'a> { is_original_source: bool, - chunks: Box, + stream: Box, replacements: &'a [Replacement], } -impl<'a> ReplaceSourceChunks<'a> { +impl<'a> ReplaceSourceStream<'a> { pub fn new(source: &'a ReplaceSource) -> Self { let is_original_source = source.inner.as_ref().as_any().is::(); Self { is_original_source, - chunks: source.inner.stream_chunks(), + stream: source.inner.to_stream(), replacements: &source.replacements, } } } -impl Chunks for ReplaceSourceChunks<'_> { - fn stream<'a>( +impl Stream for ReplaceSourceStream<'_> { + fn chunks<'a>( &'a self, object_pool: &'a ObjectPool, options: &MapOptions, @@ -431,17 +431,19 @@ impl Chunks for ReplaceSourceChunks<'_> { on_name: crate::helpers::OnName<'_, 'a>, ) -> crate::helpers::GeneratedInfo { let on_name = RefCell::new(on_name); - let repls = &self.replacements; + let replacements = &self.replacements; let mut pos: u32 = 0; let mut i: usize = 0; let mut replacement_end: Option = None; - let mut next_replacement = (i < repls.len()).then(|| repls[i].start); + let mut next_replacement = + (i < replacements.len()).then(|| replacements[i].start); let mut generated_line_offset: i64 = 0; let mut generated_column_offset: i64 = 0; let mut generated_column_offset_line = 0; let source_content_lines: RefCell>> = RefCell::new(LinearMap::default()); - let name_mapping: RefCell, u32>> = + + let name_mapping: RefCell> = RefCell::new(HashMap::default()); let name_index_mapping: RefCell> = RefCell::new(LinearMap::default()); @@ -501,7 +503,7 @@ impl Chunks for ReplaceSourceChunks<'_> { } }; - let result = self.chunks.stream( + let result = self.stream.chunks( object_pool, &MapOptions { columns: options.columns, @@ -612,7 +614,7 @@ impl Chunks for ReplaceSourceChunks<'_> { // Insert replacement content split into chunks by lines #[allow(unsafe_code)] // SAFETY: The safety of this operation relies on the fact that the `ReplaceSource` type will not delete the `replacements` during its entire lifetime. - let repl = &repls[i]; + let repl = &replacements[i]; let lines = split_into_lines(repl.content.as_str()).collect::>(); @@ -627,8 +629,8 @@ impl Chunks for ReplaceSourceChunks<'_> { let mut global_index = name_mapping.get(name.as_str()).copied(); if global_index.is_none() { let len = name_mapping.len() as u32; - name_mapping.insert(Cow::Borrowed(name), len); - on_name.borrow_mut()(len, Cow::Borrowed(name)); + name_mapping.insert(name, len); + on_name.borrow_mut()(len, name); global_index = Some(len); } replacement_name_index = global_index; @@ -683,8 +685,8 @@ impl Chunks for ReplaceSourceChunks<'_> { // Move to next replacement i += 1; - next_replacement = if i < repls.len() { - Some(repls[i].start) + next_replacement = if i < replacements.len() { + Some(replacements[i].start) } else { None }; @@ -785,10 +787,12 @@ impl Chunks for ReplaceSourceChunks<'_> { pos = end_pos; }, &mut |source_index, source, source_content| { - let mut source_content_lines = source_content_lines.borrow_mut(); - let lines = source_content - .map(|source_content| SourceContent::Raw(source_content.clone())); - source_content_lines.insert(source_index, lines); + if !self.is_original_source { + let mut source_content_lines = source_content_lines.borrow_mut(); + let lines = source_content + .map(|source_content| SourceContent::Raw(source_content.clone())); + source_content_lines.insert(source_index, lines); + } on_source(source_index, source, source_content); }, &mut |name_index, name| { @@ -796,7 +800,7 @@ impl Chunks for ReplaceSourceChunks<'_> { let mut global_index = name_mapping.get(&name).copied(); if global_index.is_none() { let len = name_mapping.len() as u32; - name_mapping.insert(name.clone(), len); + name_mapping.insert(name, len); on_name.borrow_mut()(len, name); global_index = Some(len); } @@ -808,8 +812,8 @@ impl Chunks for ReplaceSourceChunks<'_> { // Handle remaining replacements one by one let mut line = result.generated_line as i64 + generated_line_offset; - while i < repls.len() { - let content = &repls[i].content; + while i < replacements.len() { + let content = &replacements[i].content; let lines: Vec<&str> = split_into_lines(content).collect(); for (line_idx, content_line) in lines.iter().enumerate() { @@ -860,11 +864,33 @@ impl Chunks for ReplaceSourceChunks<'_> { }) as u32, } } + + fn sections_size_hint(&self) -> usize { + 1 + } + + fn sections<'a>( + &'a self, + object_pool: &'a ObjectPool, + columns: bool, + on_section: crate::helpers::OnSection<'_, 'a>, + ) -> GeneratedInfo { + let (generated_info, map) = get_map( + object_pool, + self, + &MapOptions { + columns, + final_source: true, + }, + ); + on_section(SectionOffset::default(), map); + generated_info + } } -impl StreamChunks for ReplaceSource { - fn stream_chunks<'a>(&'a self) -> Box { - Box::new(ReplaceSourceChunks::new(self)) +impl ToStream for ReplaceSource { + fn to_stream<'a>(&'a self) -> Box { + Box::new(ReplaceSourceStream::new(self)) } } @@ -1578,8 +1604,8 @@ return
{data.foo}
let mut chunks = vec![]; let object_pool = ObjectPool::default(); - let handle = source.stream_chunks(); - handle.stream( + let stream = source.to_stream(); + stream.chunks( &object_pool, &MapOptions::default(), &mut |chunk, mapping| { diff --git a/src/source.rs b/src/source.rs index 92ee366c..627a65c0 100644 --- a/src/source.rs +++ b/src/source.rs @@ -11,7 +11,7 @@ use dyn_clone::DynClone; use serde::{Deserialize, Serialize}; use crate::{ - helpers::{decode_mappings, Chunks, StreamChunks}, + helpers::{decode_mappings, encode_mappings, Stream, ToStream}, object_pool::ObjectPool, Result, }; @@ -109,7 +109,7 @@ impl<'a> SourceValue<'a> { /// [Source] abstraction, [webpack-sources docs](https://github.com/webpack/webpack-sources/#source). pub trait Source: - StreamChunks + DynHash + AsAny + DynEq + DynClone + fmt::Debug + Sync + Send + ToStream + DynHash + AsAny + DynEq + DynClone + fmt::Debug + Sync + Send { /// Get the source code. fn source(&self) -> SourceValue<'_>; @@ -130,6 +130,25 @@ pub trait Source: options: &MapOptions, ) -> Option; + /// Get the [IndexSourceMap]. + /// + /// Returns an index source map which uses sections to represent the mappings. + /// This is more efficient for concatenated sources as it avoids the expensive + /// mapping merge. The default implementation wraps the result of [Source::map] + /// into a single-section [IndexSourceMap]. + fn index_map( + &self, + object_pool: &ObjectPool, + options: &MapOptions, + ) -> Option { + self.map(object_pool, options).map(|map| { + IndexSourceMap::new(vec![Section { + offset: SectionOffset { line: 0, column: 0 }, + map, + }]) + }) + } + /// Update hash based on the source. fn update_hash(&self, state: &mut dyn Hasher) { self.dyn_hash(state); @@ -169,6 +188,15 @@ impl Source for BoxSource { self.as_ref().map(object_pool, options) } + #[inline] + fn index_map( + &self, + object_pool: &ObjectPool, + options: &MapOptions, + ) -> Option { + self.as_ref().index_map(object_pool, options) + } + #[inline] fn to_writer(&self, writer: &mut dyn std::io::Write) -> std::io::Result<()> { self.as_ref().to_writer(writer) @@ -177,9 +205,9 @@ impl Source for BoxSource { dyn_clone::clone_trait_object!(Source); -impl StreamChunks for BoxSource { - fn stream_chunks<'a>(&'a self) -> Box { - self.as_ref().stream_chunks() +impl ToStream for BoxSource { + fn to_stream<'a>(&'a self) -> Box { + self.as_ref().to_stream() } } @@ -573,6 +601,230 @@ impl TryFrom for SourceMap { } } +/// The offset of a section within the generated code. +/// +/// Both `line` and `column` are 0-based, as specified by the +/// [Index Source Map](https://tc39.es/ecma426/#sec-index-source-map) format. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Copy, Default)] +pub struct SectionOffset { + /// 0-based line offset in the generated code. + pub line: u32, + /// 0-based column offset in the generated code. + pub column: u32, +} + +/// A section within an [IndexSourceMap], pairing an [offset](SectionOffset) +/// with a regular [SourceMap]. +/// +/// See [Index Source Map § Section](https://tc39.es/ecma426/#sec-index-source-map). +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] +pub struct Section { + /// The offset in the generated code where this section begins. + pub offset: SectionOffset, + /// The source map for this section. + pub map: SourceMap, +} + +/// An [Index Source Map](https://tc39.es/ecma426/#sec-index-source-map) +/// that represents concatenated source maps as a list of [Section]s. +/// +/// Each section contains a regular [SourceMap] and an offset indicating +/// where that section starts in the generated output. This avoids the +/// need to merge mappings from multiple sources, improving performance +/// for concatenated sources like [ConcatSource](crate::ConcatSource). +/// +/// Use [IndexSourceMap::to_source_map] to flatten it into a regular [SourceMap]. +#[derive(Clone, PartialEq, Eq, Serialize)] +pub struct IndexSourceMap { + version: u8, + #[serde(skip_serializing_if = "Option::is_none")] + file: Option>, + sections: Vec
, +} + +impl std::fmt::Debug for IndexSourceMap { + fn fmt( + &self, + f: &mut std::fmt::Formatter<'_>, + ) -> std::result::Result<(), std::fmt::Error> { + write!( + f, + "IndexSourceMap {{ version: {}, file: {:?}, sections: {:?} }}", + self.version, self.file, self.sections + ) + } +} + +impl Hash for IndexSourceMap { + fn hash(&self, state: &mut H) { + self.file.hash(state); + self.sections.hash(state); + } +} + +impl IndexSourceMap { + /// Create a new [IndexSourceMap] from a list of [Section]s. + pub fn new(sections: Vec
) -> Self { + Self { + version: 3, + file: None, + sections, + } + } + + /// Get the file field. + pub fn file(&self) -> Option<&str> { + self.file.as_deref() + } + + /// Set the file field. + pub fn set_file>>(&mut self, file: Option) { + self.file = file.map(Into::into); + } + + /// Get the sections. + pub fn sections(&self) -> &[Section] { + &self.sections + } + + /// Flatten this [IndexSourceMap] into a regular [SourceMap] by merging + /// all sections, offsetting their mappings accordingly. + pub fn to_source_map(&self) -> Option { + if self.sections.is_empty() { + return None; + } + + // Single section with zero offset — return its map directly. + if self.sections.len() == 1 { + let section = &self.sections[0]; + if section.offset.line == 0 && section.offset.column == 0 { + let mut map = section.map.clone(); + if self.file.is_some() { + map.set_file(self.file.clone()); + } + return Some(map); + } + } + + let mut global_sources: Vec = Vec::new(); + let mut global_sources_content: Vec> = Vec::new(); + let mut global_names: Vec = Vec::new(); + let mut source_mapping: std::collections::HashMap = + std::collections::HashMap::new(); + let mut name_mapping: std::collections::HashMap = + std::collections::HashMap::new(); + + let mut all_mappings: Vec = Vec::new(); + + for section in &self.sections { + let map = §ion.map; + + // Build local-to-global source index mapping + let local_source_mapping: Vec = map + .sources() + .iter() + .enumerate() + .map(|(i, source)| { + if let Some(&idx) = source_mapping.get(source) { + // Update source content if we have better content + if let Some(content) = map.get_source_content(i) { + if (idx as usize) < global_sources_content.len() + && global_sources_content[idx as usize].is_empty() + { + global_sources_content[idx as usize] = content.clone(); + } + } + idx + } else { + let idx = global_sources.len() as u32; + source_mapping.insert(source.clone(), idx); + global_sources.push(source.clone()); + global_sources_content + .resize_with(global_sources.len(), || "".into()); + if let Some(content) = map.get_source_content(i) { + global_sources_content[idx as usize] = content.clone(); + } + idx + } + }) + .collect(); + + // Build local-to-global name index mapping + let local_name_mapping: Vec = map + .names() + .iter() + .map(|name| { + if let Some(&idx) = name_mapping.get(name) { + idx + } else { + let idx = global_names.len() as u32; + name_mapping.insert(name.clone(), idx); + global_names.push(name.clone()); + idx + } + }) + .collect(); + + // Decode, offset, and remap mappings + for mapping in map.decoded_mappings() { + // Offset the generated position. + // generated_line is 1-based; section.offset.line is 0-based. + let generated_line = mapping.generated_line + section.offset.line; + let generated_column = if mapping.generated_line == 1 { + mapping.generated_column + section.offset.column + } else { + mapping.generated_column + }; + + let original = mapping.original.map(|orig| OriginalLocation { + source_index: *local_source_mapping + .get(orig.source_index as usize) + .unwrap_or(&orig.source_index), + original_line: orig.original_line, + original_column: orig.original_column, + name_index: orig + .name_index + .map(|ni| *local_name_mapping.get(ni as usize).unwrap_or(&ni)), + }); + + all_mappings.push(Mapping { + generated_line, + generated_column, + original, + }); + } + } + + if all_mappings.is_empty() { + return None; + } + + let mappings_str = encode_mappings(all_mappings.into_iter()); + let mut result = SourceMap::new( + mappings_str, + global_sources, + global_sources_content, + global_names, + ); + if self.file.is_some() { + result.set_file(self.file.clone()); + } + Some(result) + } + + /// Generate index source map to a JSON string. + pub fn to_json(&self) -> Result { + let json = simd_json::serde::to_string(&self)?; + Ok(json) + } + + /// Generate index source map to writer. + pub fn to_writer(self, w: W) -> Result<()> { + simd_json::serde::to_writer(w, &self)?; + Ok(()) + } +} + /// Represent a [Mapping] information of source map. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Mapping { @@ -636,7 +888,7 @@ mod tests { use std::collections::HashMap; use crate::{ - CachedSource, ConcatSource, OriginalSource, RawBufferSource, + CachedSource, ConcatSource, ObjectPool, OriginalSource, RawBufferSource, RawStringSource, ReplaceSource, SourceMapSource, WithoutOriginalOptions, }; @@ -795,4 +1047,187 @@ mod tests { "ab" ); } + + #[test] + fn index_source_map_serialization() { + let map = SourceMap::new( + "AAAA;AACA", + vec!["file.js".into()], + vec!["line1\nline2\n".into()], + vec![], + ); + let index_map = IndexSourceMap::new(vec![Section { + offset: SectionOffset { line: 0, column: 0 }, + map, + }]); + let json = index_map.to_json().unwrap(); + assert!(json.contains("\"version\":3")); + assert!(json.contains("\"sections\"")); + assert!(json.contains("\"offset\"")); + assert!(json.contains("\"map\"")); + } + + #[test] + fn index_source_map_to_source_map_single_section() { + let map = SourceMap::new( + "AAAA;AACA", + vec!["file.js".into()], + vec!["line1\nline2\n".into()], + vec![], + ); + let index_map = IndexSourceMap::new(vec![Section { + offset: SectionOffset { line: 0, column: 0 }, + map: map.clone(), + }]); + let result = index_map.to_source_map().unwrap(); + assert_eq!(result, map); + } + + #[test] + fn index_source_map_to_source_map_empty_sections() { + let index_map = IndexSourceMap::new(vec![]); + assert!(index_map.to_source_map().is_none()); + } + + #[test] + fn index_source_map_to_source_map_with_offset() { + // First section at line 0, col 0 + let map1 = SourceMap::new( + "AAAA", + vec!["a.js".into()], + vec!["hello\n".into()], + vec![], + ); + // Second section at line 1, col 0 (after the first line) + let map2 = SourceMap::new( + "AAAA", + vec!["b.js".into()], + vec!["world\n".into()], + vec![], + ); + let index_map = IndexSourceMap::new(vec![ + Section { + offset: SectionOffset { line: 0, column: 0 }, + map: map1, + }, + Section { + offset: SectionOffset { line: 1, column: 0 }, + map: map2, + }, + ]); + let result = index_map.to_source_map().unwrap(); + assert_eq!(result.sources(), &["a.js".to_string(), "b.js".to_string()]); + assert_eq!( + result.sources_content(), + &[Arc::from("hello\n"), Arc::from("world\n")] + ); + // Verify mappings: first mapping at line 1 (1-based), second at line 2 + let mappings: Vec = result.decoded_mappings().collect(); + assert_eq!(mappings.len(), 2); + assert_eq!(mappings[0].generated_line, 1); + assert_eq!(mappings[0].generated_column, 0); + assert_eq!(mappings[0].original.as_ref().unwrap().source_index, 0); + assert_eq!(mappings[1].generated_line, 2); + assert_eq!(mappings[1].generated_column, 0); + assert_eq!(mappings[1].original.as_ref().unwrap().source_index, 1); + } + + #[test] + fn index_source_map_to_source_map_with_column_offset() { + // First section at line 0, col 0 + let map1 = + SourceMap::new("AAAA", vec!["a.js".into()], vec!["hello".into()], vec![]); + // Second section at line 0, col 5 (same line, after "hello") + let map2 = + SourceMap::new("AAAA", vec!["b.js".into()], vec!["world".into()], vec![]); + let index_map = IndexSourceMap::new(vec![ + Section { + offset: SectionOffset { line: 0, column: 0 }, + map: map1, + }, + Section { + offset: SectionOffset { line: 0, column: 5 }, + map: map2, + }, + ]); + let result = index_map.to_source_map().unwrap(); + let mappings: Vec = result.decoded_mappings().collect(); + assert_eq!(mappings.len(), 2); + assert_eq!(mappings[0].generated_line, 1); + assert_eq!(mappings[0].generated_column, 0); + assert_eq!(mappings[1].generated_line, 1); + assert_eq!(mappings[1].generated_column, 5); + } + + #[test] + fn index_map_default_impl_wraps_map() { + let source = OriginalSource::new("hello\nworld\n", "test.txt"); + let pool = ObjectPool::default(); + let options = MapOptions::default(); + let map = source.map(&pool, &options).unwrap(); + let index_map = source.index_map(&pool, &options).unwrap(); + + assert_eq!(index_map.sections().len(), 1); + assert_eq!(index_map.sections()[0].offset.line, 0); + assert_eq!(index_map.sections()[0].offset.column, 0); + assert_eq!(index_map.sections()[0].map, map); + } + + #[test] + fn index_map_returns_none_for_raw_source() { + let source = RawStringSource::from("hello world"); + let pool = ObjectPool::default(); + let options = MapOptions::default(); + assert!(source.index_map(&pool, &options).is_none()); + } + + #[test] + fn index_map_file_field_propagated() { + let map = + SourceMap::new("AAAA", vec!["a.js".into()], vec!["hello".into()], vec![]); + let mut index_map = IndexSourceMap::new(vec![Section { + offset: SectionOffset { line: 0, column: 0 }, + map, + }]); + index_map.set_file(Some("bundle.js")); + assert_eq!(index_map.file(), Some("bundle.js")); + + let result = index_map.to_source_map().unwrap(); + assert_eq!(result.file(), Some("bundle.js")); + } + + #[test] + fn index_source_map_shared_sources_across_sections() { + // Both sections reference the same source file + let map1 = SourceMap::new( + "AAAA", + vec!["shared.js".into()], + vec!["content".into()], + vec![], + ); + let map2 = SourceMap::new( + "AAAA", + vec!["shared.js".into()], + vec!["content".into()], + vec![], + ); + let index_map = IndexSourceMap::new(vec![ + Section { + offset: SectionOffset { line: 0, column: 0 }, + map: map1, + }, + Section { + offset: SectionOffset { line: 1, column: 0 }, + map: map2, + }, + ]); + let result = index_map.to_source_map().unwrap(); + // Should deduplicate sources + assert_eq!(result.sources().len(), 1); + assert_eq!(result.sources()[0], "shared.js"); + // Both mappings should reference source index 0 + let mappings: Vec = result.decoded_mappings().collect(); + assert_eq!(mappings[0].original.as_ref().unwrap().source_index, 0); + assert_eq!(mappings[1].original.as_ref().unwrap().source_index, 0); + } } diff --git a/src/source_map_source.rs b/src/source_map_source.rs index 357a26a2..48dcbbdc 100644 --- a/src/source_map_source.rs +++ b/src/source_map_source.rs @@ -7,10 +7,10 @@ use std::{ use crate::{ helpers::{ get_map, stream_chunks_of_combined_source_map, stream_chunks_of_source_map, - Chunks, StreamChunks, + GeneratedInfo, Stream, ToStream, }, object_pool::ObjectPool, - MapOptions, Source, SourceMap, SourceValue, + MapOptions, SectionOffset, Source, SourceMap, SourceValue, }; /// Options for [SourceMapSource::new]. @@ -114,8 +114,8 @@ impl Source for SourceMapSource { if self.inner_source_map.is_none() { return Some(self.source_map.clone()); } - let chunks = self.stream_chunks(); - get_map(object_pool, chunks.as_ref(), options) + let stream = self.to_stream(); + get_map(object_pool, stream.as_ref(), options).1 } fn to_writer(&self, writer: &mut dyn std::io::Write) -> std::io::Result<()> { @@ -188,10 +188,10 @@ impl std::fmt::Debug for SourceMapSource { } } -struct SourceMapSourceChunks<'source>(&'source SourceMapSource); +struct SourceMapSourceStream<'source>(&'source SourceMapSource); -impl Chunks for SourceMapSourceChunks<'_> { - fn stream<'a>( +impl Stream for SourceMapSourceStream<'_> { + fn chunks<'a>( &'a self, object_pool: &'a ObjectPool, options: &MapOptions, @@ -225,11 +225,33 @@ impl Chunks for SourceMapSourceChunks<'_> { ) } } + + fn sections_size_hint(&self) -> usize { + 1 + } + + fn sections<'a>( + &'a self, + object_pool: &'a ObjectPool, + columns: bool, + on_section: crate::helpers::OnSection<'_, 'a>, + ) -> GeneratedInfo { + let (generated_info, map) = get_map( + object_pool, + self, + &MapOptions { + columns, + final_source: true, + }, + ); + on_section(SectionOffset::default(), map); + generated_info + } } -impl StreamChunks for SourceMapSource { - fn stream_chunks<'a>(&'a self) -> Box { - Box::new(SourceMapSourceChunks(self)) +impl ToStream for SourceMapSource { + fn to_stream<'a>(&'a self) -> Box { + Box::new(SourceMapSourceStream(self)) } } diff --git a/tests/compat_source.rs b/tests/compat_source.rs index 20597052..13e8b454 100644 --- a/tests/compat_source.rs +++ b/tests/compat_source.rs @@ -3,12 +3,12 @@ use std::borrow::Cow; use std::hash::Hash; use rspack_sources::stream_chunks::{ - stream_chunks_default, Chunks, GeneratedInfo, OnChunk, OnName, OnSource, - StreamChunks, + stream_chunks_default, GeneratedInfo, OnChunk, OnName, OnSection, OnSource, + Stream, ToStream, }; use rspack_sources::{ - ConcatSource, MapOptions, ObjectPool, RawStringSource, Source, SourceExt, - SourceMap, SourceValue, + get_generated_source_info, ConcatSource, MapOptions, ObjectPool, + RawStringSource, SectionOffset, Source, SourceExt, SourceMap, SourceValue, }; #[derive(Debug, Eq)] @@ -44,16 +44,16 @@ impl Source for CompatSource { } } -struct CompatSourceChunks<'source>(&'static str, Option<&'source SourceMap>); +struct CompatSourceStream<'source>(&'static str, Option<&'source SourceMap>); -impl<'source> CompatSourceChunks<'source> { +impl<'source> CompatSourceStream<'source> { pub fn new(source: &'source CompatSource) -> Self { - CompatSourceChunks(&source.0, source.1.as_ref()) + CompatSourceStream(&source.0, source.1.as_ref()) } } -impl Chunks for CompatSourceChunks<'_> { - fn stream<'a>( +impl Stream for CompatSourceStream<'_> { + fn chunks<'a>( &'a self, object_pool: &'a ObjectPool, options: &MapOptions, @@ -71,11 +71,26 @@ impl Chunks for CompatSourceChunks<'_> { on_name, ) } + + fn sections_size_hint(&self) -> usize { + 1 + } + + fn sections<'a>( + &'a self, + _object_pool: &'a ObjectPool, + _columns: bool, + on_section: OnSection<'_, 'a>, + ) -> GeneratedInfo { + let generated_info = get_generated_source_info(self.0); + on_section(SectionOffset::default(), self.1.cloned()); + generated_info + } } -impl StreamChunks for CompatSource { - fn stream_chunks<'a>(&'a self) -> Box { - Box::new(CompatSourceChunks::new(self)) +impl ToStream for CompatSource { + fn to_stream<'a>(&'a self) -> Box { + Box::new(CompatSourceStream::new(self)) } }