From 001d3cb9629e99520f9057264b21e6e16d6d55aa Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Mon, 9 Dec 2019 21:48:46 -0800 Subject: [PATCH 1/2] Fix condition. --- .../main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java index b15365fe54ad6..9d23a64d7ebc5 100644 --- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java +++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java @@ -741,7 +741,7 @@ public boolean append(Object kbase, long koff, int klen, Object vbase, long voff longArray.set(pos * 2 + 1, keyHashcode); isDefined = true; - if (numKeys >= growthThreshold && longArray.size() < MAX_CAPACITY) { + if (numKeys >= growthThreshold && longArray.size() / 2 < MAX_CAPACITY) { try { growAndRehash(); } catch (SparkOutOfMemoryError oom) { From 5ac203e43c836bee295461035c9adacdf9ca1d6f Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Tue, 10 Dec 2019 11:56:50 -0800 Subject: [PATCH 2/2] Add comment. --- .../main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java index 9d23a64d7ebc5..7590b72112c54 100644 --- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java +++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java @@ -741,6 +741,8 @@ public boolean append(Object kbase, long koff, int klen, Object vbase, long voff longArray.set(pos * 2 + 1, keyHashcode); isDefined = true; + // We use two array entries per key, so the array size is twice the capacity. + // We should compare the current capacity of the array, instead of its size. if (numKeys >= growthThreshold && longArray.size() / 2 < MAX_CAPACITY) { try { growAndRehash();