diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..99b2416 --- /dev/null +++ b/.classpath @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ae3c172 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/bin/ diff --git a/.project b/.project new file mode 100644 index 0000000..fae508f --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + solr-vector-scoring + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + diff --git a/VectorPlugin8.6.jar b/VectorPlugin8.6.jar new file mode 100644 index 0000000..e1485eb Binary files /dev/null and b/VectorPlugin8.6.jar differ diff --git a/src/com/github/saaay71/solr/CustomVectorValueSource.java b/src/com/github/saaay71/solr/CustomVectorValueSource.java new file mode 100644 index 0000000..40ccbcd --- /dev/null +++ b/src/com/github/saaay71/solr/CustomVectorValueSource.java @@ -0,0 +1,121 @@ +package com.github.saaay71.solr; + +import org.apache.lucene.analysis.payloads.PayloadHelper; +import org.apache.lucene.index.*; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DoubleValues; +import org.apache.lucene.search.DoubleValuesSource; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.BytesRef; +import org.apache.solr.common.SolrException; + +import java.io.IOException; +import java.util.*; + +public class CustomVectorValueSource extends DoubleValuesSource { + private List vector; + private String field; + double queryVectorNorm = 0; + + public CustomVectorValueSource(String Vector, String field) { + this.field = field; + this.vector = new ArrayList(); + String[] vectorArray = Vector.split(","); + for (int i = 0; i < vectorArray.length; i++) { + double v = Double.parseDouble(vectorArray[i]); + vector.add(v); + + queryVectorNorm += Math.pow(v, 2.0); + + } + + } + + @Override + public DoubleValues getValues(LeafReaderContext ctx, DoubleValues doubleValues) throws IOException { + + return new DoubleValues() { + + double val = 0; + + @Override + public double doubleValue() throws IOException { + return val; + } + + @Override + public boolean advanceExact(int docId) throws IOException { + float score = 0; + double docVectorNorm = 0; + LeafReader reader = ctx.reader(); + Terms terms = reader.getTermVector(docId, field); + + if (vector.size() != terms.size()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "indexed and input vector array must have same length"); + } + + TermsEnum iter = terms.iterator(); + BytesRef text; + while ((text = iter.next()) != null) { + String term = text.utf8ToString(); + float payloadValue = 0f; + PostingsEnum postings = iter.postings(null, PostingsEnum.ALL); + while (postings.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { + int freq = postings.freq(); + while (freq-- > 0) + postings.nextPosition(); + + BytesRef payload = postings.getPayload(); + payloadValue = PayloadHelper.decodeFloat(payload.bytes, payload.offset); + + docVectorNorm += Math.pow(payloadValue, 2.0); + } + + score = (float) (score + payloadValue * (vector.get(Integer.parseInt(term)))); + } + + if ((docVectorNorm == 0) || (queryVectorNorm == 0)) + val = 0; + val = (float) (score / (Math.sqrt(docVectorNorm) * Math.sqrt(queryVectorNorm))); + + return true; + } + }; + } + + @Override + public boolean needsScores() { + return false; + } + + @Override + public DoubleValuesSource rewrite(IndexSearcher indexSearcher) throws IOException { + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + CustomVectorValueSource that = (CustomVectorValueSource) o; + return vector.equals(that.vector) && field.equals(that.field); + } + + @Override + public int hashCode() { + return Objects.hash(vector, field); + } + + @Override + public String toString() { + return "CustomVectorValuesSource{" + "vector=" + vector + ", Field=" + field + '}'; + } + + @Override + public boolean isCacheable(LeafReaderContext leafReaderContext) { + return false; + } +} \ No newline at end of file diff --git a/src/com/github/saaay71/solr/VectorQParserPlugin.java b/src/com/github/saaay71/solr/VectorQParserPlugin.java index 22a8cb0..db131ca 100644 --- a/src/com/github/saaay71/solr/VectorQParserPlugin.java +++ b/src/com/github/saaay71/solr/VectorQParserPlugin.java @@ -1,5 +1,6 @@ package com.github.saaay71.solr; +import org.apache.lucene.queries.function.FunctionScoreQuery; import org.apache.lucene.search.Query; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.SolrParams; @@ -18,7 +19,7 @@ public QParser createParser(String qstr, SolrParams localParams, SolrParams para public Query parse() throws SyntaxError { String field = localParams.get(QueryParsing.F); String vector = localParams.get("vector"); - boolean cosine = localParams.getBool("cosine", true); + // boolean cosine = localParams.getBool("cosine", true); if (field == null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'f' not specified"); @@ -31,18 +32,17 @@ public Query parse() throws SyntaxError { Query subQuery = subQuery(localParams.get(QueryParsing.V), null).getQuery(); FieldType ft = req.getCore().getLatestSchema().getFieldType(field); - if(ft != null) { + if (ft != null) { VectorQuery q = new VectorQuery(subQuery); - q.setQueryString(localParams.toLocalParamsString()); + q.setQueryString(localParams.toLocalParamsString()); query = q; } - if (query == null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Query is null"); } - return new VectorScoreQuery(query, vector, field, cosine); + return new FunctionScoreQuery(query, new CustomVectorValueSource(vector, field)); } }; diff --git a/src/com/github/saaay71/solr/VectorQuery.java b/src/com/github/saaay71/solr/VectorQuery.java index f907403..bd3f425 100644 --- a/src/com/github/saaay71/solr/VectorQuery.java +++ b/src/com/github/saaay71/solr/VectorQuery.java @@ -2,13 +2,7 @@ import java.io.IOException; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.ConstantScoreScorer; -import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; +import org.apache.lucene.search.*; public class VectorQuery extends Query { String queryStr = ""; @@ -22,18 +16,22 @@ public void setQueryString(String queryString){ this.queryStr = queryString; } - @Override - public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException { + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { Weight w; if(q == null){ - w = new ConstantScoreWeight(this) { + w = new ConstantScoreWeight(this, boost) { + @Override + public boolean isCacheable(LeafReaderContext lf) { + return false; + } + @Override public Scorer scorer(LeafReaderContext context) throws IOException { - return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc())); + return new ConstantScoreScorer(this, score(), scoreMode, DocIdSetIterator.all(context.reader().maxDoc())); } }; }else{ - w = searcher.createWeight(q, needsScores); + w = searcher.createWeight(q, scoreMode, boost); } return w; } @@ -54,4 +52,4 @@ public int hashCode() { return classHash() ^ queryStr.hashCode(); } -} +} \ No newline at end of file diff --git a/src/com/github/saaay71/solr/VectorScoreQuery.java b/src/com/github/saaay71/solr/VectorScoreQuery.java deleted file mode 100644 index 7c32fbe..0000000 --- a/src/com/github/saaay71/solr/VectorScoreQuery.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.github.saaay71.solr; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import org.apache.lucene.analysis.payloads.PayloadHelper; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.queries.CustomScoreProvider; -import org.apache.lucene.queries.CustomScoreQuery; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Query; -import org.apache.lucene.util.BytesRef; -import org.apache.solr.common.SolrException; - -public class VectorScoreQuery extends CustomScoreQuery { - List vector; - String field; - boolean cosine = true; - double queryVectorNorm = 0; - - public VectorScoreQuery(Query subQuery, String Vector, String field, boolean cosine) { - super(subQuery); - this.field = field; - this.cosine = cosine; - this.vector = new ArrayList(); - String[] vectorArray = Vector.split(","); - for(int i=0;i 0) postings.nextPosition(); - - BytesRef payload = postings.getPayload(); - payloadValue = PayloadHelper.decodeFloat(payload.bytes, payload.offset); - - if (cosine) - docVectorNorm += Math.pow(payloadValue, 2.0); - } - - score = (float)(score + payloadValue * (vector.get(Integer.parseInt(term)))); - } - - if (cosine) { - if ((docVectorNorm == 0) || (queryVectorNorm == 0)) return 0f; - return (float)(score / (Math.sqrt(docVectorNorm) * Math.sqrt(queryVectorNorm))); - } - - return score; - } - }; - } -} \ No newline at end of file