Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .classpath
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="src" path="src"/>
<classpathentry kind="lib" path="VectorPlugin.jar"/>
<classpathentry kind="lib" path="D:/Alexsei/apps/jars/8.6/required_jars/lucene-analyzers-common-8.6.0.jar"/>
<classpathentry kind="lib" path="D:/Alexsei/apps/jars/8.6/required_jars/lucene-core-8.6.0.jar"/>
<classpathentry kind="lib" path="D:/Alexsei/apps/jars/8.6/required_jars/lucene-expressions-8.6.0.jar"/>
<classpathentry kind="lib" path="D:/Alexsei/apps/jars/8.6/required_jars/lucene-queries-8.6.0.jar"/>
<classpathentry kind="lib" path="D:/Alexsei/apps/jars/8.6/required_jars/solr-core-8.6.0.jar"/>
<classpathentry kind="lib" path="D:/Alexsei/apps/jars/8.6/required_jars/solr-solrj-8.6.0.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/bin/
17 changes: 17 additions & 0 deletions .project
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>solr-vector-scoring</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
Binary file added VectorPlugin8.6.jar
Binary file not shown.
121 changes: 121 additions & 0 deletions src/com/github/saaay71/solr/CustomVectorValueSource.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package com.github.saaay71.solr;

import org.apache.lucene.analysis.payloads.PayloadHelper;
import org.apache.lucene.index.*;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;

import java.io.IOException;
import java.util.*;

public class CustomVectorValueSource extends DoubleValuesSource {
private List<Double> vector;
private String field;
double queryVectorNorm = 0;

public CustomVectorValueSource(String Vector, String field) {
this.field = field;
this.vector = new ArrayList<Double>();
String[] vectorArray = Vector.split(",");
for (int i = 0; i < vectorArray.length; i++) {
double v = Double.parseDouble(vectorArray[i]);
vector.add(v);

queryVectorNorm += Math.pow(v, 2.0);

}

}

@Override
public DoubleValues getValues(LeafReaderContext ctx, DoubleValues doubleValues) throws IOException {

return new DoubleValues() {

double val = 0;

@Override
public double doubleValue() throws IOException {
return val;
}

@Override
public boolean advanceExact(int docId) throws IOException {
float score = 0;
double docVectorNorm = 0;
LeafReader reader = ctx.reader();
Terms terms = reader.getTermVector(docId, field);

if (vector.size() != terms.size()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"indexed and input vector array must have same length");
}

TermsEnum iter = terms.iterator();
BytesRef text;
while ((text = iter.next()) != null) {
String term = text.utf8ToString();
float payloadValue = 0f;
PostingsEnum postings = iter.postings(null, PostingsEnum.ALL);
while (postings.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
int freq = postings.freq();
while (freq-- > 0)
postings.nextPosition();

BytesRef payload = postings.getPayload();
payloadValue = PayloadHelper.decodeFloat(payload.bytes, payload.offset);

docVectorNorm += Math.pow(payloadValue, 2.0);
}

score = (float) (score + payloadValue * (vector.get(Integer.parseInt(term))));
}

if ((docVectorNorm == 0) || (queryVectorNorm == 0))
val = 0;
val = (float) (score / (Math.sqrt(docVectorNorm) * Math.sqrt(queryVectorNorm)));

return true;
}
};
}

@Override
public boolean needsScores() {
return false;
}

@Override
public DoubleValuesSource rewrite(IndexSearcher indexSearcher) throws IOException {
return this;
}

@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
CustomVectorValueSource that = (CustomVectorValueSource) o;
return vector.equals(that.vector) && field.equals(that.field);
}

@Override
public int hashCode() {
return Objects.hash(vector, field);
}

@Override
public String toString() {
return "CustomVectorValuesSource{" + "vector=" + vector + ", Field=" + field + '}';
}

@Override
public boolean isCacheable(LeafReaderContext leafReaderContext) {
return false;
}
}
10 changes: 5 additions & 5 deletions src/com/github/saaay71/solr/VectorQParserPlugin.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.github.saaay71.solr;

import org.apache.lucene.queries.function.FunctionScoreQuery;
import org.apache.lucene.search.Query;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
Expand All @@ -18,7 +19,7 @@ public QParser createParser(String qstr, SolrParams localParams, SolrParams para
public Query parse() throws SyntaxError {
String field = localParams.get(QueryParsing.F);
String vector = localParams.get("vector");
boolean cosine = localParams.getBool("cosine", true);
// boolean cosine = localParams.getBool("cosine", true);

if (field == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'f' not specified");
Expand All @@ -31,18 +32,17 @@ public Query parse() throws SyntaxError {
Query subQuery = subQuery(localParams.get(QueryParsing.V), null).getQuery();

FieldType ft = req.getCore().getLatestSchema().getFieldType(field);
if(ft != null) {
if (ft != null) {
VectorQuery q = new VectorQuery(subQuery);
q.setQueryString(localParams.toLocalParamsString());
q.setQueryString(localParams.toLocalParamsString());
query = q;
}


if (query == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Query is null");
}

return new VectorScoreQuery(query, vector, field, cosine);
return new FunctionScoreQuery(query, new CustomVectorValueSource(vector, field));

}
};
Expand Down
24 changes: 11 additions & 13 deletions src/com/github/saaay71/solr/VectorQuery.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,7 @@

import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.*;

public class VectorQuery extends Query {
String queryStr = "";
Expand All @@ -22,18 +16,22 @@ public void setQueryString(String queryString){
this.queryStr = queryString;
}

@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
Weight w;
if(q == null){
w = new ConstantScoreWeight(this) {
w = new ConstantScoreWeight(this, boost) {
@Override
public boolean isCacheable(LeafReaderContext lf) {
return false;
}

@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc()));
return new ConstantScoreScorer(this, score(), scoreMode, DocIdSetIterator.all(context.reader().maxDoc()));
}
};
}else{
w = searcher.createWeight(q, needsScores);
w = searcher.createWeight(q, scoreMode, boost);
}
return w;
}
Expand All @@ -54,4 +52,4 @@ public int hashCode() {
return classHash() ^ queryStr.hashCode();
}

}
}
80 changes: 0 additions & 80 deletions src/com/github/saaay71/solr/VectorScoreQuery.java

This file was deleted.