Skip to content

Commit 242c1fc

Browse files
epughepugh
authored
SOLR-17852: Migrate schema designer to filestoreapi part deux (#3031)
Now using the FileStore API for persisting working documents used in the Schema Designer.
1 parent bef3353 commit 242c1fc

9 files changed

Lines changed: 105 additions & 79 deletions

File tree

‎solr/CHANGES.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ Improvements
4949

5050
* SOLR-17739: Uploading a config sets with forbidden file types will produces a Exception. Previously it was just ignoring those files and logging a WARN. (Abhishek Umarjikar via Eric Pugh)
5151

52+
* SOLR-17852: Migrate Schema Designer to use FileStore API instead of BlobHandler for persisting working data. (Eric Pugh)
53+
5254
Optimizations
5355
---------------------
5456
* SOLR-17568: The CLI bin/solr export tool now contacts the appropriate nodes directly for data instead of proxying through one.

‎solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -344,18 +344,19 @@ public void put(FileEntry entry) throws IOException {
344344
private void distribute(FileInfo info) {
345345
try {
346346
String dirName = info.path.substring(0, info.path.lastIndexOf('/'));
347+
347348
coreContainer
348349
.getZkController()
349350
.getZkClient()
350-
.makePath(ZK_PACKAGESTORE + dirName, false, true);
351-
coreContainer
352-
.getZkController()
353-
.getZkClient()
354-
.create(
351+
.makePath(
355352
ZK_PACKAGESTORE + info.path,
356353
info.getDetails().getMetaData().sha512.getBytes(UTF_8),
357354
CreateMode.PERSISTENT,
358-
true);
355+
null,
356+
false,
357+
true,
358+
0);
359+
359360
} catch (Exception e) {
360361
throw new SolrException(SERVER_ERROR, "Unable to create an entry in ZK", e);
361362
}
@@ -452,7 +453,7 @@ public boolean fetch(String path, String from) {
452453
}
453454

454455
@Override
455-
public void get(String path, Consumer<FileEntry> consumer, boolean fetchmissing)
456+
public void get(String path, Consumer<FileEntry> consumer, boolean fetchMissing)
456457
throws IOException {
457458
Path file = getRealPath(path);
458459
String simpleName = file.getFileName().toString();
@@ -573,7 +574,6 @@ public void refresh(String path) {
573574
@SuppressWarnings({"rawtypes"})
574575
List myFiles = list(path, s -> true);
575576
for (Object f : l) {
576-
// TODO: https://issues.apache.org/jira/browse/SOLR-15426
577577
// l should be a List<String> and myFiles should be a List<FileDetails>, so contains
578578
// should always return false!
579579
if (!myFiles.contains(f)) {

‎solr/core/src/java/org/apache/solr/filestore/FileStore.java‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,11 @@
2929
import org.apache.solr.filestore.FileStoreAPI.MetaData;
3030
import org.apache.zookeeper.server.ByteBufferInputStream;
3131

32-
/** The interface to be implemented by any package store provider * @lucene.experimental */
32+
/**
33+
* The interface to be implemented by any package store provider
34+
*
35+
* @lucene.experimental
36+
*/
3337
public interface FileStore {
3438

3539
/**
@@ -38,8 +42,13 @@ public interface FileStore {
3842
*/
3943
void put(FileEntry fileEntry) throws IOException;
4044

41-
/** read file content from a given path */
42-
void get(String path, Consumer<FileEntry> filecontent, boolean getMissing) throws IOException;
45+
/**
46+
* Read file content from a given path.
47+
*
48+
* <p>TODO: Is fetchMissing actually used? I don't see it being used, but the IDE doesn't flag it
49+
* not being used!
50+
*/
51+
void get(String path, Consumer<FileEntry> consumer, boolean fetchMissing) throws IOException;
4352

4453
/** Fetch a resource from another node internal API */
4554
boolean fetch(String path, String from);
@@ -59,7 +68,7 @@ public interface FileStore {
5968
Map<String, byte[]> getKeys() throws IOException;
6069

6170
/**
62-
* Refresh the files in a path. May be this node does not have all files
71+
* Refresh the files in a path. Maybe this node does not have all files?
6372
*
6473
* @param path the path to be refreshed.
6574
*/
@@ -71,12 +80,12 @@ public interface FileStore {
7180
/** Delete file from local file system */
7281
void deleteLocal(String path);
7382

74-
public class FileEntry {
83+
class FileEntry {
7584
final ByteBuffer buf;
7685
final MetaData meta;
7786
final String path;
7887

79-
FileEntry(ByteBuffer buf, MetaData meta, String path) {
88+
public FileEntry(ByteBuffer buf, MetaData meta, String path) {
8089
this.buf = buf;
8190
this.meta = meta;
8291
this.path = path;

‎solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,7 @@
8686
/** All V2 APIs have a prefix of /api/schema-designer/ */
8787
public class SchemaDesignerAPI implements SchemaDesignerConstants {
8888

89-
private static final Set<String> excludeConfigSetNames =
90-
new HashSet<>(Arrays.asList(DEFAULT_CONFIGSET_NAME, BLOB_STORE_ID));
89+
private static final Set<String> excludeConfigSetNames = Set.of(DEFAULT_CONFIGSET_NAME);
9190

9291
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
9392

@@ -172,7 +171,7 @@ public void getInfo(SolrQueryRequest req, SolrQueryResponse rsp) throws IOExcept
172171

173172
// don't fail if loading sample docs fails
174173
try {
175-
responseMap.put("numDocs", configSetHelper.getStoredSampleDocs(configSet).size());
174+
responseMap.put("numDocs", configSetHelper.retrieveSampleDocs(configSet).size());
176175
} catch (Exception exc) {
177176
log.warn("Failed to load sample docs from blob store for {}", configSet, exc);
178177
}
@@ -282,7 +281,7 @@ public void updateFileContents(SolrQueryRequest req, SolrQueryResponse rsp)
282281
ManagedIndexSchema schema = loadLatestSchema(mutableId);
283282
Map<Object, Throwable> errorsDuringIndexing = null;
284283
SolrException solrExc = null;
285-
List<SolrInputDocument> docs = configSetHelper.getStoredSampleDocs(configSet);
284+
List<SolrInputDocument> docs = configSetHelper.retrieveSampleDocs(configSet);
286285
String[] analysisErrorHolder = new String[1];
287286
if (!docs.isEmpty()) {
288287
String idField = schema.getUniqueKeyField().getName();
@@ -318,7 +317,7 @@ public void getSampleValue(SolrQueryRequest req, SolrQueryResponse rsp) throws I
318317
final String idField = getRequiredParam(UNIQUE_KEY_FIELD_PARAM, req);
319318
String docId = req.getParams().get(DOC_ID_PARAM);
320319

321-
final List<SolrInputDocument> docs = configSetHelper.getStoredSampleDocs(configSet);
320+
final List<SolrInputDocument> docs = configSetHelper.retrieveSampleDocs(configSet);
322321
String textValue = null;
323322
if (StrUtils.isNullOrEmpty(docId)) {
324323
// no doc ID from client ... find the first doc with a non-empty string value for fieldName
@@ -429,7 +428,7 @@ public void addSchemaObject(SolrQueryRequest req, SolrQueryResponse rsp)
429428

430429
ManagedIndexSchema schema = loadLatestSchema(mutableId);
431430
Map<String, Object> response =
432-
buildResponse(configSet, schema, null, configSetHelper.getStoredSampleDocs(configSet));
431+
buildResponse(configSet, schema, null, configSetHelper.retrieveSampleDocs(configSet));
433432
response.put(action, objectName);
434433
rsp.getValues().addAll(response);
435434
}
@@ -468,7 +467,7 @@ public void updateSchemaObject(SolrQueryRequest req, SolrQueryResponse rsp)
468467

469468
// re-index the docs if no error to this point
470469
final ManagedIndexSchema schema = loadLatestSchema(mutableId);
471-
List<SolrInputDocument> docs = configSetHelper.getStoredSampleDocs(configSet);
470+
List<SolrInputDocument> docs = configSetHelper.retrieveSampleDocs(configSet);
472471
Map<Object, Throwable> errorsDuringIndexing = null;
473472
String[] analysisErrorHolder = new String[1];
474473
if (solrExc == null && !docs.isEmpty()) {
@@ -571,7 +570,7 @@ && zkStateReader().getClusterState().hasCollection(newCollection)) {
571570
int rf = req.getParams().getInt("replicationFactor", 1);
572571
configSetHelper.createCollection(newCollection, configSet, numShards, rf);
573572
if (req.getParams().getBool(INDEX_TO_COLLECTION_PARAM, false)) {
574-
List<SolrInputDocument> docs = configSetHelper.getStoredSampleDocs(configSet);
573+
List<SolrInputDocument> docs = configSetHelper.retrieveSampleDocs(configSet);
575574
if (!docs.isEmpty()) {
576575
ManagedIndexSchema schema = loadLatestSchema(mutableId);
577576
errorsDuringIndexing =
@@ -773,7 +772,7 @@ public void query(SolrQueryRequest req, SolrQueryResponse rsp)
773772
mutableId,
774773
version,
775774
currentVersion);
776-
List<SolrInputDocument> docs = configSetHelper.getStoredSampleDocs(configSet);
775+
List<SolrInputDocument> docs = configSetHelper.retrieveSampleDocs(configSet);
777776
ManagedIndexSchema schema = loadLatestSchema(mutableId);
778777
errorsDuringIndexing =
779778
indexSampleDocsWithRebuildOnAnalysisError(
@@ -829,7 +828,7 @@ protected SampleDocuments loadSampleDocuments(SolrQueryRequest req, String confi
829828
if (!docs.isEmpty()) {
830829
// user posted in some docs, if there are already docs stored in the blob store, then add
831830
// these to the existing set
832-
List<SolrInputDocument> stored = configSetHelper.getStoredSampleDocs(configSet);
831+
List<SolrInputDocument> stored = configSetHelper.retrieveSampleDocs(configSet);
833832
if (!stored.isEmpty()) {
834833
// keep the docs in the request as newest
835834
ManagedIndexSchema latestSchema = loadLatestSchema(getMutableId(configSet));
@@ -838,14 +837,14 @@ protected SampleDocuments loadSampleDocuments(SolrQueryRequest req, String confi
838837
latestSchema.getUniqueKeyField().getName(), stored, MAX_SAMPLE_DOCS);
839838
}
840839

841-
// store in the blob store so that we always have access to these docs
840+
// store in the Filestore so that we always have access to these docs
842841
configSetHelper.storeSampleDocs(configSet, docs);
843842
}
844843
}
845844

846845
if (docs == null || docs.isEmpty()) {
847846
// no sample docs in the request ... find in blob store (or fail if no docs previously stored)
848-
docs = configSetHelper.getStoredSampleDocs(configSet);
847+
docs = configSetHelper.retrieveSampleDocs(configSet);
849848

850849
// no docs? but if this schema has already been published, it's OK, we can skip the docs part
851850
if (docs.isEmpty() && !configExists(configSet)) {

‎solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java‎

Lines changed: 45 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,11 @@
2727
import static org.apache.solr.schema.ManagedIndexSchemaFactory.DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME;
2828

2929
import java.io.ByteArrayOutputStream;
30+
import java.io.FileNotFoundException;
3031
import java.io.IOException;
3132
import java.io.InputStream;
3233
import java.lang.invoke.MethodHandles;
34+
import java.nio.ByteBuffer;
3335
import java.nio.charset.StandardCharsets;
3436
import java.nio.file.FileVisitResult;
3537
import java.nio.file.Files;
@@ -49,6 +51,7 @@
4951
import java.util.Set;
5052
import java.util.concurrent.TimeUnit;
5153
import java.util.concurrent.TimeoutException;
54+
import java.util.concurrent.atomic.AtomicReference;
5255
import java.util.stream.Collectors;
5356
import java.util.zip.ZipEntry;
5457
import java.util.zip.ZipOutputStream;
@@ -59,8 +62,6 @@
5962
import org.apache.solr.client.solrj.SolrResponse;
6063
import org.apache.solr.client.solrj.SolrServerException;
6164
import org.apache.solr.client.solrj.impl.CloudSolrClient;
62-
import org.apache.solr.client.solrj.impl.InputStreamResponseParser;
63-
import org.apache.solr.client.solrj.impl.JavaBinResponseParser;
6465
import org.apache.solr.client.solrj.impl.JsonMapResponseParser;
6566
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
6667
import org.apache.solr.client.solrj.request.GenericSolrRequest;
@@ -78,13 +79,16 @@
7879
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
7980
import org.apache.solr.common.cloud.ZkStateReader;
8081
import org.apache.solr.common.params.ModifiableSolrParams;
81-
import org.apache.solr.common.util.IOUtils;
8282
import org.apache.solr.common.util.NamedList;
8383
import org.apache.solr.common.util.SimpleOrderedMap;
8484
import org.apache.solr.common.util.Utils;
8585
import org.apache.solr.core.CoreContainer;
8686
import org.apache.solr.core.SolrConfig;
8787
import org.apache.solr.core.SolrResourceLoader;
88+
import org.apache.solr.filestore.ClusterFileStore;
89+
import org.apache.solr.filestore.DistribFileStore;
90+
import org.apache.solr.filestore.FileStore;
91+
import org.apache.solr.filestore.FileStoreAPI;
8892
import org.apache.solr.handler.admin.CollectionsHandler;
8993
import org.apache.solr.schema.CopyField;
9094
import org.apache.solr.schema.FieldType;
@@ -446,7 +450,7 @@ boolean updateField(
446450

447451
protected void validateMultiValuedChange(String configSet, SchemaField field, Boolean multiValued)
448452
throws IOException {
449-
List<SolrInputDocument> docs = getStoredSampleDocs(configSet);
453+
List<SolrInputDocument> docs = retrieveSampleDocs(configSet);
450454
if (!docs.isEmpty()) {
451455
boolean isMV = schemaSuggester.isMultiValued(field.getName(), docs);
452456
if (isMV && !multiValued) {
@@ -466,44 +470,58 @@ protected void validateTypeChange(String configSet, SchemaField field, FieldType
466470
SolrException.ErrorCode.BAD_REQUEST,
467471
"Cannot change type of the _version_ field; it must be a plong.");
468472
}
469-
List<SolrInputDocument> docs = getStoredSampleDocs(configSet);
473+
List<SolrInputDocument> docs = retrieveSampleDocs(configSet);
470474
if (!docs.isEmpty()) {
471475
schemaSuggester.validateTypeChange(field, toType, docs);
472476
}
473477
}
474478

479+
String getSampleDocsPathFromConfigSet(String configSet) {
480+
return "schemadesigner" + "/" + configSet + "_sampledocs.javabin";
481+
}
482+
475483
void deleteStoredSampleDocs(String configSet) {
476-
try {
477-
cloudClient().deleteByQuery(BLOB_STORE_ID, "id:" + configSet + "_sample/*", 10);
478-
} catch (IOException | SolrServerException | SolrException exc) {
479-
final String excStr = exc.toString();
480-
log.warn("Failed to delete sample docs from blob store for {} due to: {}", configSet, excStr);
481-
}
484+
String path = getSampleDocsPathFromConfigSet(configSet);
485+
// why do I have to do this in two stages?
486+
DistribFileStore.deleteZKFileEntry(cc.getZkController().getZkClient(), path);
487+
cc.getFileStore().delete(path);
482488
}
483489

484490
@SuppressWarnings("unchecked")
485-
List<SolrInputDocument> getStoredSampleDocs(final String configSet) throws IOException {
486-
var request = new GenericSolrRequest(SolrRequest.METHOD.GET, "/blob/" + configSet + "_sample");
487-
request.setRequiresCollection(true);
488-
request.setResponseParser(new InputStreamResponseParser("filestream"));
489-
InputStream inputStream = null;
491+
List<SolrInputDocument> retrieveSampleDocs(final String configSet) throws IOException {
492+
AtomicReference<List<SolrInputDocument>> docs = new AtomicReference<>(List.of());
493+
String path = getSampleDocsPathFromConfigSet(configSet);
494+
490495
try {
491-
var resp = request.process(cloudClient(), BLOB_STORE_ID).getResponse();
492-
inputStream = (InputStream) resp.get("stream");
493-
var bytes = inputStream.readAllBytes();
494-
if (bytes.length > 0) {
495-
return (List<SolrInputDocument>) Utils.fromJavabin(bytes);
496-
} else return Collections.emptyList();
497-
} catch (SolrServerException e) {
498-
throw new IOException("Failed to lookup stored docs for " + configSet + " due to: " + e);
499-
} finally {
500-
IOUtils.closeQuietly(inputStream);
496+
cc.getFileStore()
497+
.get(
498+
path,
499+
entry -> {
500+
try (InputStream is = entry.getInputStream()) {
501+
docs.set((List<SolrInputDocument>) Utils.fromJavabin(is));
502+
} catch (IOException e) {
503+
log.error("Error reading file content at path {}", path, e);
504+
}
505+
},
506+
true);
507+
} catch (FileNotFoundException e) {
508+
log.info("File at path {} not found.", path);
501509
}
510+
511+
return docs.get();
502512
}
503513

504514
void storeSampleDocs(final String configSet, List<SolrInputDocument> docs) throws IOException {
505515
docs.forEach(d -> d.removeField(VERSION_FIELD)); // remove _version_ field before storing ...
506-
postDataToBlobStore(cloudClient(), configSet + "_sample", readAllBytes(() -> toJavabin(docs)));
516+
storeSampleDocs(configSet, readAllBytes(() -> toJavabin(docs)));
517+
}
518+
519+
protected void storeSampleDocs(String configSet, byte[] bytes) throws IOException {
520+
String path = getSampleDocsPathFromConfigSet(configSet);
521+
522+
FileStoreAPI.MetaData meta = ClusterFileStore._createJsonMetaData(bytes, null);
523+
524+
cc.getFileStore().put(new FileStore.FileEntry(ByteBuffer.wrap(bytes), meta, path));
507525
}
508526

509527
/** Gets the stream, reads all the bytes, closes the stream. */
@@ -513,18 +531,6 @@ static byte[] readAllBytes(IOSupplier<InputStream> hasStream) throws IOException
513531
}
514532
}
515533

516-
protected void postDataToBlobStore(CloudSolrClient cloudClient, String blobName, byte[] bytes)
517-
throws IOException {
518-
var request = new GenericSolrRequest(SolrRequest.METHOD.POST, "/blob/" + blobName);
519-
request.withContent(bytes, JavaBinResponseParser.JAVABIN_CONTENT_TYPE);
520-
request.setRequiresCollection(true);
521-
try {
522-
request.process(cloudClient, BLOB_STORE_ID);
523-
} catch (SolrServerException e) {
524-
throw new SolrException(ErrorCode.SERVER_ERROR, e);
525-
}
526-
}
527-
528534
private String getBaseUrl(final String collection) {
529535
String baseUrl = null;
530536
try {

‎solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConstants.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ public interface SchemaDesignerConstants {
4040
String DESIGNER_KEY = "_designer.";
4141
String LANGUAGES_PARAM = "languages";
4242
String CONFIGOVERLAY_JSON = "configoverlay.json";
43-
String BLOB_STORE_ID = ".system";
4443
String UPDATE_ERROR = "updateError";
4544
String ANALYSIS_ERROR = "analysisError";
4645
String ERROR_DETAILS = "errorDetails";

‎solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import java.util.Optional;
3636
import java.util.stream.Stream;
3737
import org.apache.solr.client.solrj.SolrQuery;
38-
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
3938
import org.apache.solr.client.solrj.response.QueryResponse;
4039
import org.apache.solr.cloud.SolrCloudTestCase;
4140
import org.apache.solr.common.SolrDocumentList;
@@ -72,9 +71,6 @@ public static void createCluster() throws Exception {
7271
configureCluster(1)
7372
.addConfig(DEFAULT_CONFIGSET_NAME, ExternalPaths.DEFAULT_CONFIGSET)
7473
.configure();
75-
// SchemaDesignerAPI depends on the blob store ".system" collection existing.
76-
CollectionAdminRequest.createCollection(BLOB_STORE_ID, 1, 1).process(cluster.getSolrClient());
77-
cluster.waitForActiveCollection(BLOB_STORE_ID, 1, 1);
7874
}
7975

8076
@AfterClass

0 commit comments

Comments
 (0)