Skip to content

Commit 073e8e7

Browse files
beinanclaude
andauthored
feat(java): add Dataset.sample() API (#6500)
## Summary - Expose the Rust core `Dataset::sample()` method through the Java JNI interface - Allows Java users to randomly sample `n` rows from a dataset, with optional fragment ID filtering - Returns results in row-id order for efficient underlying take operations ## Changes - **`java/lance-jni/src/blocking_dataset.rs`**: Add `nativeSample` JNI function that projects schema from column names, calls `dataset.sample()`, and serializes the result as Arrow IPC bytes - **`java/src/main/java/org/lance/Dataset.java`**: Add `sample(n, columns)` and `sample(n, columns, fragmentIds)` public methods with input validation - **`java/src/test/java/org/lance/DatasetTest.java`**: Add `testSample` covering basic sampling, fragment-filtered sampling, and over-sampling (n > total rows) ## Test plan - [ ] `testSample` — verifies correct row counts for basic sample, fragment-filtered sample, and over-sample cases - [ ] Rust JNI compiles cleanly with `cargo check` 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 423835f commit 073e8e7

3 files changed

Lines changed: 152 additions & 0 deletions

File tree

java/lance-jni/src/blocking_dataset.rs

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1978,6 +1978,64 @@ fn inner_take(
19781978
Ok(**byte_array)
19791979
}
19801980

1981+
#[unsafe(no_mangle)]
1982+
pub extern "system" fn Java_org_lance_Dataset_nativeSample(
1983+
mut env: JNIEnv,
1984+
java_dataset: JObject,
1985+
n: jlong,
1986+
columns_obj: JObject, // List<String>
1987+
fragment_ids_obj: JObject, // Optional<List<Integer>>
1988+
) -> jbyteArray {
1989+
match inner_sample(&mut env, java_dataset, n, columns_obj, fragment_ids_obj) {
1990+
Ok(byte_array) => byte_array,
1991+
Err(e) => {
1992+
let _ = env.throw_new("java/lang/RuntimeException", format!("{:?}", e));
1993+
std::ptr::null_mut()
1994+
}
1995+
}
1996+
}
1997+
1998+
fn inner_sample(
1999+
env: &mut JNIEnv,
2000+
java_dataset: JObject,
2001+
n: jlong,
2002+
columns_obj: JObject, // List<String>
2003+
fragment_ids_obj: JObject, // Optional<List<Integer>>
2004+
) -> Result<jbyteArray> {
2005+
let columns: Vec<String> = env.get_strings(&columns_obj)?;
2006+
let fragment_ids: Option<Vec<i32>> = env.get_ints_opt(&fragment_ids_obj)?;
2007+
let fragment_ids_u32: Option<Vec<u32>> =
2008+
fragment_ids.map(|ids| ids.iter().map(|&id| id as u32).collect());
2009+
2010+
let result = {
2011+
let dataset_guard =
2012+
unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?;
2013+
let dataset = &dataset_guard.inner;
2014+
2015+
let projection = dataset
2016+
.schema()
2017+
.project_preserve_system_columns(&columns)
2018+
.map_err(|e| Error::runtime_error(e.to_string()))?;
2019+
2020+
match RT.block_on(dataset.sample(n as usize, &projection, fragment_ids_u32.as_deref())) {
2021+
Ok(res) => res,
2022+
Err(e) => {
2023+
return Err(e.into());
2024+
}
2025+
}
2026+
};
2027+
2028+
let mut buffer = Vec::new();
2029+
{
2030+
let mut writer = StreamWriter::try_new(&mut buffer, &result.schema())?;
2031+
writer.write(&result)?;
2032+
writer.finish()?;
2033+
}
2034+
2035+
let byte_array = env.byte_array_from_slice(&buffer)?;
2036+
Ok(**byte_array)
2037+
}
2038+
19812039
#[unsafe(no_mangle)]
19822040
pub extern "system" fn Java_org_lance_Dataset_nativeDelete(
19832041
mut env: JNIEnv,

java/src/main/java/org/lance/Dataset.java

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -756,6 +756,57 @@ public void close() throws IOException {
756756

757757
private native byte[] nativeTake(List<Long> indices, List<String> columns);
758758

759+
/**
760+
* Randomly sample n rows from the dataset.
761+
*
762+
* <p>The returned rows are in row-id order (not random order), which allows the underlying take
763+
* operation to use an efficient sorted code path.
764+
*
765+
* @param n the number of rows to sample
766+
* @param columns the columns to include in the result
767+
* @return an ArrowReader containing the sampled rows
768+
* @throws IOException if an I/O error occurs
769+
*/
770+
public ArrowReader sample(long n, List<String> columns) throws IOException {
771+
return sample(n, columns, Optional.empty());
772+
}
773+
774+
/**
775+
* Randomly sample n rows from specific fragments of the dataset.
776+
*
777+
* <p>The returned rows are in row-id order (not random order), which allows the underlying take
778+
* operation to use an efficient sorted code path.
779+
*
780+
* @param n the number of rows to sample
781+
* @param columns the columns to include in the result
782+
* @param fragmentIds optional list of fragment IDs to restrict sampling to
783+
* @return an ArrowReader containing the sampled rows
784+
* @throws IOException if an I/O error occurs
785+
*/
786+
public ArrowReader sample(long n, List<String> columns, Optional<List<Integer>> fragmentIds)
787+
throws IOException {
788+
Preconditions.checkArgument(n > 0, "n must be greater than 0");
789+
Preconditions.checkNotNull(columns, "columns cannot be null");
790+
Preconditions.checkArgument(!columns.isEmpty(), "columns cannot be empty");
791+
Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed");
792+
try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
793+
byte[] arrowData = nativeSample(n, columns, fragmentIds);
794+
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(arrowData);
795+
ReadableByteChannel readChannel = Channels.newChannel(byteArrayInputStream);
796+
return new ArrowStreamReader(readChannel, allocator) {
797+
@Override
798+
public void close() throws IOException {
799+
super.close();
800+
readChannel.close();
801+
byteArrayInputStream.close();
802+
}
803+
};
804+
}
805+
}
806+
807+
private native byte[] nativeSample(
808+
long n, List<String> columns, Optional<List<Integer>> fragmentIds);
809+
759810
/**
760811
* Delete rows of data by predicate.
761812
*

java/src/test/java/org/lance/DatasetTest.java

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -839,6 +839,49 @@ void testTake(@TempDir Path tempDir) throws IOException, ClosedChannelException
839839
}
840840
}
841841

842+
@Test
843+
void testSample(@TempDir Path tempDir) throws IOException {
844+
String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
845+
String datasetPath = tempDir.resolve(testMethodName).toString();
846+
try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
847+
TestUtils.SimpleTestDataset testDataset =
848+
new TestUtils.SimpleTestDataset(allocator, datasetPath);
849+
dataset = testDataset.createEmptyDataset();
850+
851+
try (Dataset dataset2 = testDataset.write(1, 20)) {
852+
// Sample without fragment filter
853+
List<String> columns = Arrays.asList("id", "name");
854+
try (ArrowReader reader = dataset2.sample(5, columns)) {
855+
assertTrue(reader.loadNextBatch());
856+
VectorSchemaRoot result = reader.getVectorSchemaRoot();
857+
assertNotNull(result);
858+
assertEquals(5, result.getRowCount());
859+
assertEquals(2, result.getSchema().getFields().size());
860+
}
861+
862+
// Sample with fragment filter
863+
List<Fragment> fragments = dataset2.getFragments();
864+
assertFalse(fragments.isEmpty());
865+
List<Integer> fragmentIds =
866+
fragments.stream().map(f -> f.getId()).collect(Collectors.toList());
867+
try (ArrowReader reader = dataset2.sample(3, columns, Optional.of(fragmentIds))) {
868+
assertTrue(reader.loadNextBatch());
869+
VectorSchemaRoot result = reader.getVectorSchemaRoot();
870+
assertNotNull(result);
871+
assertEquals(3, result.getRowCount());
872+
}
873+
874+
// Sample more than available rows returns all rows
875+
try (ArrowReader reader = dataset2.sample(100, columns)) {
876+
assertTrue(reader.loadNextBatch());
877+
VectorSchemaRoot result = reader.getVectorSchemaRoot();
878+
assertNotNull(result);
879+
assertEquals(20, result.getRowCount());
880+
}
881+
}
882+
}
883+
}
884+
842885
@Test
843886
void testCountRows(@TempDir Path tempDir) {
844887
String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();

0 commit comments

Comments
 (0)