Skip to content

GlobalOrdinalsWithScoreCollector to accumulate scores in double precision #16428

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
javanna wants to merge 1 commit into
base: branch_10x
Choose a base branch
from
+40 −34
Open

GlobalOrdinalsWithScoreCollector to accumulate scores in double precision #16428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions lucene/CHANGES.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,10 @@ Bug Fixes
including the default codec. Since 10.4.0 the merger tested the per-field wrapper instead
of the unwrapped reader, so every merge rebuilt the graph from scratch. (Jeho Jeong)

* GITHUB#16408: Accumulate join Total/Avg scores in double precision in
GlobalOrdinalsWithScoreCollector, fixing an intermittent failure caused by
non-associative float addition that #16378 missed. (Luca Cavanna)

Other
---------------------
* GITHUB#16266: Remove deprecated search(Query, Collector) calls in QueryUtils by replacing
Expand Down
70 changes: 36 additions & 34 deletions lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,12 @@ public boolean match(int globalOrd) {
}

public float score(int globalOrdinal) {
return scores.getScore(globalOrdinal);
return (float) scores.getScore(globalOrdinal);
}

protected abstract void doScore(int globalOrd, float existingScore, float newScore);
protected abstract void doScore(int globalOrd, double existingScore, double newScore);

protected abstract float unset();
protected abstract double unset();

@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
Expand Down Expand Up @@ -116,8 +116,8 @@ public void collect(int doc) throws IOException {
if (docTermOrds.advanceExact(doc)) {
final int globalOrd = (int) segmentOrdToGlobalOrdLookup.get(docTermOrds.ordValue());
collectedOrds.set(globalOrd);
float existingScore = scores.getScore(globalOrd);
float newScore = scorer.score();
double existingScore = scores.getScore(globalOrd);
double newScore = scorer.score();
doScore(globalOrd, existingScore, newScore);
if (occurrences != null) {
occurrences.increment(globalOrd);
Expand Down Expand Up @@ -145,8 +145,8 @@ public void collect(int doc) throws IOException {
if (docTermOrds.advanceExact(doc)) {
int segmentOrd = docTermOrds.ordValue();
collectedOrds.set(segmentOrd);
float existingScore = scores.getScore(segmentOrd);
float newScore = scorer.score();
double existingScore = scores.getScore(segmentOrd);
double newScore = scorer.score();
doScore(segmentOrd, existingScore, newScore);
if (occurrences != null) {
occurrences.increment(segmentOrd);
Expand All @@ -167,13 +167,13 @@ public Min(String field, OrdinalMap ordinalMap, long valueCount, int min, int ma
}

@Override
protected void doScore(int globalOrd, float existingScore, float newScore) {
protected void doScore(int globalOrd, double existingScore, double newScore) {
scores.setScore(globalOrd, Math.min(existingScore, newScore));
}

@Override
protected float unset() {
return Float.POSITIVE_INFINITY;
protected double unset() {
return Double.POSITIVE_INFINITY;
}
}

Expand All @@ -184,13 +184,13 @@ public Max(String field, OrdinalMap ordinalMap, long valueCount, int min, int ma
}

@Override
protected void doScore(int globalOrd, float existingScore, float newScore) {
protected void doScore(int globalOrd, double existingScore, double newScore) {
scores.setScore(globalOrd, Math.max(existingScore, newScore));
}

@Override
protected float unset() {
return Float.NEGATIVE_INFINITY;
protected double unset() {
return Double.NEGATIVE_INFINITY;
}
}

Expand All @@ -201,13 +201,13 @@ public Sum(String field, OrdinalMap ordinalMap, long valueCount, int min, int ma
}

@Override
protected void doScore(int globalOrd, float existingScore, float newScore) {
protected void doScore(int globalOrd, double existingScore, double newScore) {
scores.setScore(globalOrd, existingScore + newScore);
}

@Override
protected float unset() {
return 0f;
protected double unset() {
return 0d;
}
}

Expand All @@ -218,18 +218,18 @@ public Avg(String field, OrdinalMap ordinalMap, long valueCount, int min, int ma
}

@Override
protected void doScore(int globalOrd, float existingScore, float newScore) {
protected void doScore(int globalOrd, double existingScore, double newScore) {
scores.setScore(globalOrd, existingScore + newScore);
}

@Override
public float score(int globalOrdinal) {
return scores.getScore(globalOrdinal) / occurrences.getOccurrence(globalOrdinal);
return (float) (scores.getScore(globalOrdinal) / occurrences.getOccurrence(globalOrdinal));
}

@Override
protected float unset() {
return 0f;
protected double unset() {
return 0d;
}
}

Expand Down Expand Up @@ -276,16 +276,16 @@ public void collect(int doc) throws IOException {
}

@Override
protected void doScore(int globalOrd, float existingScore, float newScore) {}
protected void doScore(int globalOrd, double existingScore, double newScore) {}

@Override
public float score(int globalOrdinal) {
return 1f;
}

@Override
protected float unset() {
return 0f;
protected double unset() {
return 0d;
}

@Override
Expand All @@ -309,33 +309,35 @@ public org.apache.lucene.search.ScoreMode scoreMode() {

static final class Scores {

final float[][] blocks;
final float unset;
// Accumulated in double precision because float addition isn't associative, so summing the
// same per-document scores in a different order may round to a different float.
final double[][] blocks;
final double unset;

private Scores(long valueCount, float unset) {
private Scores(long valueCount, double unset) {
long blockSize = valueCount + arraySize - 1;
blocks = new float[(int) ((blockSize) / arraySize)][];
blocks = new double[(int) ((blockSize) / arraySize)][];
this.unset = unset;
}

public void setScore(int globalOrdinal, float score) {
public void setScore(int globalOrdinal, double score) {
int block = globalOrdinal / arraySize;
int offset = globalOrdinal % arraySize;
float[] scores = blocks[block];
double[] scores = blocks[block];
if (scores == null) {
blocks[block] = scores = new float[arraySize];
if (unset != 0f) {
blocks[block] = scores = new double[arraySize];
if (unset != 0d) {
Arrays.fill(scores, unset);
}
}
scores[offset] = score;
}

public float getScore(int globalOrdinal) {
public double getScore(int globalOrdinal) {
int block = globalOrdinal / arraySize;
int offset = globalOrdinal % arraySize;
float[] scores = blocks[block];
float score;
double[] scores = blocks[block];
double score;
if (scores != null) {
score = scores[offset];
} else {
Expand Down