Skip to content
This repository was archived by the owner on Jan 9, 2025. It is now read-only.

Commit 2a0eef5

Browse files
fh-msfh-ms
authored
Binary import enhancement (#478)
1 parent 053af31 commit 2a0eef5

24 files changed

Lines changed: 895 additions & 428 deletions

‎base/src/main/java/one/microstream/memory/XMemory.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1143,6 +1143,14 @@ public static final byte[] toArray(final ByteBuffer[] sources)
11431143
}
11441144
return bytes;
11451145
}
1146+
1147+
public static final ByteBuffer toDirectByteBuffer(final byte[] bytes)
1148+
{
1149+
final ByteBuffer buffer = allocateDirectNative(bytes.length);
1150+
buffer.put(bytes);
1151+
buffer.flip();
1152+
return buffer;
1153+
}
11461154

11471155
public static final long getPositionLimit(final ByteBuffer buffer)
11481156
{
@@ -1181,6 +1189,12 @@ public static final ByteBuffer clearForLimit(final ByteBuffer buffer, final long
11811189
return buffer;
11821190
}
11831191

1192+
public static final ByteBuffer slice(final ByteBuffer source, final long position, final long limit)
1193+
{
1194+
final ByteBuffer tmp = source.duplicate();
1195+
tmp.limit((int)(position + limit)).position((int)position);
1196+
return tmp.slice();
1197+
}
11841198

11851199

11861200
///////////////////////////////////////////////////////////////////////////

‎integrations/cdi/src/main/java/one/microstream/integrations/cdi/types/config/StorageManagerProxy.java‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11
package one.microstream.integrations.cdi.types.config;
22

3+
import java.nio.ByteBuffer;
4+
import java.util.Optional;
5+
import java.util.function.Predicate;
6+
7+
import javax.enterprise.inject.spi.CDI;
8+
9+
import org.eclipse.microprofile.config.ConfigProvider;
10+
311
/*-
412
* #%L
513
* MicroStream Integrations CDI
@@ -44,11 +52,6 @@
4452
import one.microstream.storage.types.StorageManager;
4553
import one.microstream.storage.types.StorageRawFileStatistics;
4654
import one.microstream.storage.types.StorageTypeDictionary;
47-
import org.eclipse.microprofile.config.ConfigProvider;
48-
49-
import javax.enterprise.inject.spi.CDI;
50-
import java.util.Optional;
51-
import java.util.function.Predicate;
5255

5356
/**
5457
* For MicroProfile Config, at deployment time, we need to validate if @ConfigProperty is valid by
@@ -72,7 +75,7 @@ public StorageManagerProxy(final String value)
7275
// location.
7376
final EmbeddedStorageConfigurationBuilder configurationBuilder = EmbeddedStorageConfiguration.load(value);
7477

75-
Configuration configuration = configurationBuilder.buildConfiguration();
78+
final Configuration configuration = configurationBuilder.buildConfiguration();
7679
final String name = Optional.ofNullable(configuration.get("database-name")).orElse(value);
7780

7881
this.foundation = configurationBuilder
@@ -274,6 +277,12 @@ public void importFiles(final XGettingEnum<AFile> importFiles)
274277
{
275278
this.getStorageManager().importFiles(importFiles);
276279
}
280+
281+
@Override
282+
public void importData(final XGettingEnum<ByteBuffer> importData)
283+
{
284+
this.getStorageManager().importData(importData);
285+
}
277286

278287
@Override
279288
public PersistenceManager<Binary> persistenceManager()

‎persistence/binary/src/main/java/one/microstream/persistence/binary/types/Binary.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1679,6 +1679,10 @@ public abstract long iterateReferences(
16791679
BinaryReferenceTraverser[] traversers,
16801680
PersistenceObjectIdAcceptor acceptor
16811681
);
1682+
1683+
public abstract void mark();
1684+
1685+
public abstract void reset();
16821686

16831687
public final long storeCharsAsList(
16841688
final long memoryOffset,

‎persistence/binary/src/main/java/one/microstream/persistence/binary/types/BinaryLoadItem.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,5 +186,17 @@ public void iterateChannelChunks(final Consumer<? super Binary> logic)
186186
// technically, the single data set could be iterated, but designwise, it's not the task, here.
187187
throw new UnsupportedOperationException();
188188
}
189+
190+
@Override
191+
public void mark()
192+
{
193+
throw new UnsupportedOperationException();
194+
}
195+
196+
@Override
197+
public void reset()
198+
{
199+
throw new UnsupportedOperationException();
200+
}
189201

190202
}

‎persistence/binary/src/main/java/one/microstream/persistence/binary/types/ChunksBuffer.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,5 +343,23 @@ public long iterateReferences(
343343
{
344344
throw new UnsupportedOperationException();
345345
}
346+
347+
@Override
348+
public void mark()
349+
{
350+
for(int i = 0; i <= this.currentBuffersIndex; i++)
351+
{
352+
this.buffers[i].mark();
353+
}
354+
}
355+
356+
@Override
357+
public void reset()
358+
{
359+
for(int i = 0; i <= this.currentBuffersIndex; i++)
360+
{
361+
this.buffers[i].reset();
362+
}
363+
}
346364

347365
}

‎persistence/binary/src/main/java/one/microstream/persistence/binary/types/ChunksWrapper.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,5 +170,27 @@ public long iterateReferences(
170170
{
171171
throw new UnsupportedOperationException();
172172
}
173+
174+
@Override
175+
public void mark()
176+
{
177+
final ByteBuffer[] buffers = this.buffers;
178+
179+
for(int i = 0; i < buffers.length; i++)
180+
{
181+
buffers[i].mark();
182+
}
183+
}
184+
185+
@Override
186+
public void reset()
187+
{
188+
final ByteBuffer[] buffers = this.buffers;
189+
190+
for(int i = 0; i < buffers.length; i++)
191+
{
192+
buffers[i].reset();
193+
}
194+
}
173195

174196
}

‎storage/embedded/src/main/java/one/microstream/storage/embedded/types/EmbeddedStorageManager.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
import static one.microstream.X.notNull;
2424

25+
import java.nio.ByteBuffer;
2526
import java.util.Arrays;
2627
import java.util.function.Predicate;
2728

@@ -555,7 +556,12 @@ public final void importFiles(final XGettingEnum<AFile> importFiles)
555556
this.singletonConnection().importFiles(importFiles);
556557
}
557558

558-
559+
@Override
560+
public void importData(final XGettingEnum<ByteBuffer> importData)
561+
{
562+
this.singletonConnection().importData(importData);
563+
}
564+
559565
/**
560566
* @deprecated will be removed in version 8
561567
*/

‎storage/storage/src/main/java/one/microstream/storage/types/StorageChannel.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ public void postStoreUpdateEntityCache(ByteBuffer[] chunks, long[] chunksStorage
7979
// (19.07.2014 TM)TODO: refactor storage typing to avoid classes in public API
8080
public StorageEntityCache.Default prepareImportData();
8181

82-
public void importData(StorageImportSourceFile importFile);
82+
public void importData(StorageImportSource importSource);
8383

8484
public void rollbackImportData(Throwable cause);
8585

@@ -659,9 +659,9 @@ public StorageEntityCache.Default prepareImportData()
659659
}
660660

661661
@Override
662-
public void importData(final StorageImportSourceFile importFile)
662+
public void importData(final StorageImportSource importSource)
663663
{
664-
this.fileManager.copyData(importFile);
664+
this.fileManager.copyData(importSource);
665665
}
666666

667667
@Override

‎storage/storage/src/main/java/one/microstream/storage/types/StorageChannelImportBatch.java‎

Lines changed: 84 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,93 @@
2424

2525
public interface StorageChannelImportBatch
2626
{
27-
public long fileOffset();
27+
public long batchOffset();
2828

29-
public long fileLength();
29+
public long batchLength();
3030

3131
public void iterateEntities(Consumer<? super StorageChannelImportEntity> iterator);
3232

3333
public StorageChannelImportEntity first();
34+
35+
36+
/* to optimize the common case where a batch contains only one entity, the batch itself is the first entity.
37+
* The additional memory is insignificant if there are many entities and few large batches.
38+
* If there are many short batches, then memory is saved by incorporating the first entity.
39+
*/
40+
public static class Default
41+
extends StorageChannelImportEntity.Default
42+
implements StorageChannelImportBatch
43+
{
44+
///////////////////////////////////////////////////////////////////////////
45+
// instance fields //
46+
////////////////////
47+
48+
long batchOffset;
49+
long batchLength;
50+
StorageChannelImportBatch.Default batchNext ;
51+
52+
53+
54+
///////////////////////////////////////////////////////////////////////////
55+
// constructors //
56+
/////////////////
57+
58+
Default()
59+
{
60+
super(0, 0, null);
61+
}
62+
63+
Default(
64+
final long batchOffset ,
65+
final int entityLength,
66+
final long objectId ,
67+
final StorageEntityType.Default type
68+
)
69+
{
70+
super(entityLength, objectId, type);
71+
this.batchOffset = batchOffset ;
72+
this.batchLength = entityLength;
73+
}
74+
75+
76+
77+
///////////////////////////////////////////////////////////////////////////
78+
// methods //
79+
////////////
80+
81+
@Override
82+
public long batchOffset()
83+
{
84+
return this.batchOffset;
85+
}
86+
87+
@Override
88+
public final long batchLength()
89+
{
90+
return this.batchLength;
91+
}
92+
93+
@Override
94+
public final void iterateEntities(final Consumer<? super StorageChannelImportEntity> iterator)
95+
{
96+
for(StorageChannelImportEntity.Default e = this.first(); e != null; e = e.next)
97+
{
98+
iterator.accept(e);
99+
}
100+
}
101+
102+
@Override
103+
public final StorageChannelImportEntity.Default first()
104+
{
105+
return this.type != null ? this : this.batchNext;
106+
}
107+
108+
@Override
109+
public final String toString()
110+
{
111+
return "batch" + "[" + this.length + "]" + (this.batchNext == null ? "" : " " + this.batchNext.toString());
112+
}
113+
114+
}
115+
34116
}

‎storage/storage/src/main/java/one/microstream/storage/types/StorageChannelImportEntity.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,64 @@ public interface StorageChannelImportEntity
2929
public long objectId();
3030

3131
public StorageChannelImportEntity next();
32+
33+
34+
35+
public static class Default implements StorageChannelImportEntity
36+
{
37+
///////////////////////////////////////////////////////////////////////////
38+
// instance fields //
39+
////////////////////
40+
41+
final int length ;
42+
final long objectId;
43+
final StorageEntityType.Default type ;
44+
StorageChannelImportEntity.Default next ;
45+
46+
///////////////////////////////////////////////////////////////////////////
47+
// constructors //
48+
/////////////////
49+
50+
Default(
51+
final int length ,
52+
final long objectId,
53+
final StorageEntityType.Default type
54+
)
55+
{
56+
super();
57+
this.length = length ;
58+
this.objectId = objectId;
59+
this.type = type ;
60+
}
61+
62+
///////////////////////////////////////////////////////////////////////////
63+
// methods //
64+
////////////
65+
66+
@Override
67+
public final int length()
68+
{
69+
return this.length;
70+
}
71+
72+
@Override
73+
public final StorageEntityType.Default type()
74+
{
75+
return this.type;
76+
}
77+
78+
@Override
79+
public final long objectId()
80+
{
81+
return this.objectId;
82+
}
83+
84+
@Override
85+
public final StorageChannelImportEntity.Default next()
86+
{
87+
return this.next;
88+
}
89+
90+
}
91+
3292
}

0 commit comments

Comments
 (0)