From ad710dd8efc7bf577c196e95e39a31407d8ff3e7 Mon Sep 17 00:00:00 2001 From: jinxing Date: Wed, 26 Jul 2017 08:43:16 +0800 Subject: [PATCH 1/3] Update description of spark.shuffle.maxChunksBeingTransferred --- .../main/java/org/apache/spark/network/util/TransportConf.java | 2 ++ docs/configuration.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java index ea52e9fe6c1c1..7751e009dab7f 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java @@ -259,6 +259,8 @@ public Properties cryptoConf() { /** * The max number of chunks allowed to being transferred at the same time on shuffle service. + * Note that new coming connections will be closed when the max number is hit. Client should + * have retry mechanism, otherwise it will end up with fetch failure. */ public long maxChunksBeingTransferred() { return conf.getLong("spark.shuffle.maxChunksBeingTransferred", Long.MAX_VALUE); diff --git a/docs/configuration.md b/docs/configuration.md index f4b6f46db5b66..c5094a3120a96 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -636,6 +636,8 @@ Apart from these, the following properties are also available, and may be useful Long.MAX_VALUE The max number of chunks allowed to being transferred at the same time on shuffle service. + Note that new coming connections will be closed when the max number is hit. Client should + have retry mechanism, otherwise it will end up with fetch failure. From 14f857e76694fd944fe2caa5c7e4cad98dbc6836 Mon Sep 17 00:00:00 2001 From: jinxing Date: Wed, 26 Jul 2017 23:30:58 +0800 Subject: [PATCH 2/3] refine according to tgravescs's comments --- .../java/org/apache/spark/network/util/TransportConf.java | 7 ++++--- docs/configuration.md | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java index 7751e009dab7f..7a741b54f927f 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java @@ -258,9 +258,10 @@ public Properties cryptoConf() { } /** - * The max number of chunks allowed to being transferred at the same time on shuffle service. - * Note that new coming connections will be closed when the max number is hit. Client should - * have retry mechanism, otherwise it will end up with fetch failure. + * The max number of chunks allowed to be transferred at the same time on shuffle service. + * Note that new incoming connections will be closed when the max number is hit. The client will + * retry according to the shuffle retry configs (see spark.shuffle.io.maxRetries and + * spark.shuffle.io.retryWait), if those limits are reached the task will fail with fetch failure. */ public long maxChunksBeingTransferred() { return conf.getLong("spark.shuffle.maxChunksBeingTransferred", Long.MAX_VALUE); diff --git a/docs/configuration.md b/docs/configuration.md index c5094a3120a96..9b6b93878ad8b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -635,9 +635,10 @@ Apart from these, the following properties are also available, and may be useful spark.shuffle.maxChunksBeingTransferred Long.MAX_VALUE - The max number of chunks allowed to being transferred at the same time on shuffle service. - Note that new coming connections will be closed when the max number is hit. Client should - have retry mechanism, otherwise it will end up with fetch failure. + The max number of chunks allowed to be transferred at the same time on shuffle service. + Note that new incoming connections will be closed when the max number is hit. The client will + retry according to the shuffle retry configs (see spark.shuffle.io.maxRetries and + spark.shuffle.io.retryWait), if those limits are reached the task will fail with fetch failure. From 7db7cc706ca8d22dbdecf36ed4d55150aaa62967 Mon Sep 17 00:00:00 2001 From: jinxing Date: Thu, 27 Jul 2017 08:41:11 +0800 Subject: [PATCH 3/3] refine description --- .../java/org/apache/spark/network/util/TransportConf.java | 5 +++-- docs/configuration.md | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java index 7a741b54f927f..88256b810bf04 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java @@ -260,8 +260,9 @@ public Properties cryptoConf() { /** * The max number of chunks allowed to be transferred at the same time on shuffle service. * Note that new incoming connections will be closed when the max number is hit. The client will - * retry according to the shuffle retry configs (see spark.shuffle.io.maxRetries and - * spark.shuffle.io.retryWait), if those limits are reached the task will fail with fetch failure. + * retry according to the shuffle retry configs (see `spark.shuffle.io.maxRetries` and + * `spark.shuffle.io.retryWait`), if those limits are reached the task will fail with fetch + * failure. */ public long maxChunksBeingTransferred() { return conf.getLong("spark.shuffle.maxChunksBeingTransferred", Long.MAX_VALUE); diff --git a/docs/configuration.md b/docs/configuration.md index 9b6b93878ad8b..500f980455b0e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -637,8 +637,9 @@ Apart from these, the following properties are also available, and may be useful The max number of chunks allowed to be transferred at the same time on shuffle service. Note that new incoming connections will be closed when the max number is hit. The client will - retry according to the shuffle retry configs (see spark.shuffle.io.maxRetries and - spark.shuffle.io.retryWait), if those limits are reached the task will fail with fetch failure. + retry according to the shuffle retry configs (see spark.shuffle.io.maxRetries and + spark.shuffle.io.retryWait), if those limits are reached the task will fail with + fetch failure.