Skip to content

Commit 0c74b4c

Browse files
dfa1claude
andauthored
chore: dedup the ZDICT sample-flattening to clear new-code duplication (#24)
train / optimize / finalizeFrom each repeated the same block that packs the samples into one native buffer plus a parallel size_t[] of lengths, and the same produced-size error-check/copy-out tail. The recent null-validation edits marked these methods as "new code", so Sonar's quality gate failed on new_duplicated_lines_density (9.5%). Extract flatten() -> FlatSamples, toDictionary(), and requireNonEmpty(); the three trainers now share them. Behaviour and messages unchanged. Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent f59bee2 commit 0c74b4c

1 file changed

Lines changed: 49 additions & 70 deletions

File tree

zstd/src/main/java/io/github/dfa1/zstd/ZstdDictionary.java

Lines changed: 49 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -96,38 +96,18 @@ public static ZstdDictionary of(byte[] raw) {
9696
/// @throws ZstdException if training fails (commonly: not enough sample data)
9797
public static ZstdDictionary train(List<byte[]> samples, int maxDictBytes) {
9898
Objects.requireNonNull(samples, SAMPLES);
99-
if (samples.isEmpty()) {
100-
throw new ZstdException("cannot train a dictionary from zero samples");
101-
}
102-
long total = 0;
103-
for (byte[] s : samples) {
104-
total += s.length;
105-
}
99+
requireNonEmpty(samples, "train");
106100
try (Arena arena = Arena.ofConfined()) {
107-
// flatten all samples into one buffer + a parallel size_t[] of lengths
108-
MemorySegment flat = arena.allocate(Math.max(total, 1));
109-
MemorySegment sizes = arena.allocate(JAVA_LONG, samples.size());
110-
long offset = 0;
111-
for (int i = 0; i < samples.size(); i++) {
112-
byte[] s = samples.get(i);
113-
MemorySegment.copy(s, 0, flat, JAVA_BYTE, offset, s.length);
114-
sizes.setAtIndex(JAVA_LONG, i, s.length);
115-
offset += s.length;
116-
}
101+
FlatSamples in = flatten(arena, samples);
117102
MemorySegment dictBuf = arena.allocate(maxDictBytes);
118103
long produced;
119104
try {
120105
produced = (long) Bindings.ZDICT_TRAIN.invokeExact(
121-
dictBuf, (long) maxDictBytes, flat, sizes, samples.size());
106+
dictBuf, (long) maxDictBytes, in.data(), in.sizes(), in.count());
122107
} catch (Throwable t) {
123108
throw NativeCall.rethrow(t);
124109
}
125-
if (zdictIsError(produced)) {
126-
throw new ZstdException("dictionary training failed: " + zdictErrorName(produced));
127-
}
128-
byte[] out = new byte[Math.toIntExact(produced)];
129-
MemorySegment.copy(dictBuf, JAVA_BYTE, 0, out, 0, out.length);
130-
return new ZstdDictionary(out);
110+
return toDictionary(dictBuf, produced, "dictionary training");
131111
}
132112
}
133113

@@ -180,23 +160,9 @@ public static ZstdDictionary trainFastCover(List<byte[]> samples, int maxDictByt
180160
private static ZstdDictionary optimize(List<byte[]> samples, int maxDictBytes,
181161
int compressionLevel, boolean fast) {
182162
Objects.requireNonNull(samples, SAMPLES);
183-
if (samples.isEmpty()) {
184-
throw new ZstdException("cannot train a dictionary from zero samples");
185-
}
163+
requireNonEmpty(samples, "train");
186164
try (Arena arena = Arena.ofConfined()) {
187-
long total = 0;
188-
for (byte[] s : samples) {
189-
total += s.length;
190-
}
191-
MemorySegment flat = arena.allocate(Math.max(total, 1));
192-
MemorySegment sizes = arena.allocate(JAVA_LONG, samples.size());
193-
long offset = 0;
194-
for (int i = 0; i < samples.size(); i++) {
195-
byte[] s = samples.get(i);
196-
MemorySegment.copy(s, 0, flat, JAVA_BYTE, offset, s.length);
197-
sizes.setAtIndex(JAVA_LONG, i, s.length);
198-
offset += s.length;
199-
}
165+
FlatSamples in = flatten(arena, samples);
200166
// zeroed params (auto-tune k/d/steps); set single-threaded + target level.
201167
MemoryLayout layout = fast ? FASTCOVER_PARAMS : COVER_PARAMS;
202168
MemorySegment params = arena.allocate(layout);
@@ -207,16 +173,11 @@ private static ZstdDictionary optimize(List<byte[]> samples, int maxDictBytes,
207173
long produced;
208174
try {
209175
produced = (long) handle.invokeExact(
210-
dictBuf, (long) maxDictBytes, flat, sizes, samples.size(), params);
176+
dictBuf, (long) maxDictBytes, in.data(), in.sizes(), in.count(), params);
211177
} catch (Throwable t) {
212178
throw NativeCall.rethrow(t);
213179
}
214-
if (zdictIsError(produced)) {
215-
throw new ZstdException("dictionary training failed: " + zdictErrorName(produced));
216-
}
217-
byte[] out = new byte[Math.toIntExact(produced)];
218-
MemorySegment.copy(dictBuf, JAVA_BYTE, 0, out, 0, out.length);
219-
return new ZstdDictionary(out);
180+
return toDictionary(dictBuf, produced, "dictionary training");
220181
}
221182
}
222183

@@ -235,23 +196,9 @@ public static ZstdDictionary finalizeFrom(byte[] content, List<byte[]> samples,
235196
int maxDictBytes, int compressionLevel) {
236197
Objects.requireNonNull(content, "content");
237198
Objects.requireNonNull(samples, SAMPLES);
238-
if (samples.isEmpty()) {
239-
throw new ZstdException("cannot finalise a dictionary from zero samples");
240-
}
199+
requireNonEmpty(samples, "finalise");
241200
try (Arena arena = Arena.ofConfined()) {
242-
long total = 0;
243-
for (byte[] s : samples) {
244-
total += s.length;
245-
}
246-
MemorySegment flat = arena.allocate(Math.max(total, 1));
247-
MemorySegment sizes = arena.allocate(JAVA_LONG, samples.size());
248-
long offset = 0;
249-
for (int i = 0; i < samples.size(); i++) {
250-
byte[] s = samples.get(i);
251-
MemorySegment.copy(s, 0, flat, JAVA_BYTE, offset, s.length);
252-
sizes.setAtIndex(JAVA_LONG, i, s.length);
253-
offset += s.length;
254-
}
201+
FlatSamples in = flatten(arena, samples);
255202
MemorySegment contentSeg = Zstd.copyIn(arena, content);
256203
MemorySegment params = arena.allocate(Bindings.ZDICT_PARAMS_LAYOUT);
257204
params.set(JAVA_INT, 0, compressionLevel); // compressionLevel; notificationLevel/dictID = 0
@@ -260,17 +207,49 @@ public static ZstdDictionary finalizeFrom(byte[] content, List<byte[]> samples,
260207
try {
261208
produced = (long) Bindings.ZDICT_FINALIZE_DICTIONARY.invokeExact(
262209
dictBuf, (long) maxDictBytes, contentSeg, (long) content.length,
263-
flat, sizes, samples.size(), params);
210+
in.data(), in.sizes(), in.count(), params);
264211
} catch (Throwable t) {
265212
throw NativeCall.rethrow(t);
266213
}
267-
if (zdictIsError(produced)) {
268-
throw new ZstdException("dictionary finalisation failed: " + zdictErrorName(produced));
269-
}
270-
byte[] out = new byte[Math.toIntExact(produced)];
271-
MemorySegment.copy(dictBuf, JAVA_BYTE, 0, out, 0, out.length);
272-
return new ZstdDictionary(out);
214+
return toDictionary(dictBuf, produced, "dictionary finalisation");
215+
}
216+
}
217+
218+
/// One native buffer holding all samples back to back, plus a parallel
219+
/// `size_t[]` of their lengths — the shape the ZDICT trainers consume.
220+
private record FlatSamples(MemorySegment data, MemorySegment sizes, int count) {
221+
}
222+
223+
private static FlatSamples flatten(Arena arena, List<byte[]> samples) {
224+
long total = 0;
225+
for (byte[] s : samples) {
226+
total += s.length;
227+
}
228+
MemorySegment data = arena.allocate(Math.max(total, 1));
229+
MemorySegment sizes = arena.allocate(JAVA_LONG, samples.size());
230+
long offset = 0;
231+
for (int i = 0; i < samples.size(); i++) {
232+
byte[] s = samples.get(i);
233+
MemorySegment.copy(s, 0, data, JAVA_BYTE, offset, s.length);
234+
sizes.setAtIndex(JAVA_LONG, i, s.length);
235+
offset += s.length;
236+
}
237+
return new FlatSamples(data, sizes, samples.size());
238+
}
239+
240+
private static void requireNonEmpty(List<byte[]> samples, String verb) {
241+
if (samples.isEmpty()) {
242+
throw new ZstdException("cannot " + verb + " a dictionary from zero samples");
243+
}
244+
}
245+
246+
private static ZstdDictionary toDictionary(MemorySegment dictBuf, long produced, String what) {
247+
if (zdictIsError(produced)) {
248+
throw new ZstdException(what + " failed: " + zdictErrorName(produced));
273249
}
250+
byte[] out = new byte[Math.toIntExact(produced)];
251+
MemorySegment.copy(dictBuf, JAVA_BYTE, 0, out, 0, out.length);
252+
return new ZstdDictionary(out);
274253
}
275254

276255
/// The dictionary id zstd stamps into frames compressed with this dictionary,

0 commit comments

Comments
 (0)