From 6703d13e1c129948158993435b182cf9e12f2e91 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Tue, 19 Sep 2023 10:41:21 +0800 Subject: [PATCH] feat(api): add http/grpc interface rate limit --- .../org/tron/common/parameter/CommonParameter.java | 3 +++ common/src/main/java/org/tron/core/Constant.java | 2 ++ .../src/main/java/org/tron/core/config/args/Args.java | 6 ++++++ .../tron/core/services/http/RateLimiterServlet.java | 5 +++-- .../services/ratelimiter/RateLimiterInterceptor.java | 4 +++- .../services/ratelimiter/strategy/QpsStrategy.java | 3 ++- .../java/org/tron/common/config/args/ArgsTest.java | 10 ++++++++++ framework/src/test/resources/config-test.conf | 1 + 8 files changed, 30 insertions(+), 4 deletions(-) diff --git a/common/src/main/java/org/tron/common/parameter/CommonParameter.java b/common/src/main/java/org/tron/common/parameter/CommonParameter.java index aa9f4e40f14..94d398c4b86 100644 --- a/common/src/main/java/org/tron/common/parameter/CommonParameter.java +++ b/common/src/main/java/org/tron/common/parameter/CommonParameter.java @@ -415,6 +415,9 @@ public class CommonParameter { @Setter public int rateLimiterGlobalIpQps; @Getter + @Setter + public int rateLimiterGlobalApiQps; + @Getter public DbBackupConfig dbBackupConfig; @Getter public RocksDbSettings rocksDBCustomSettings; diff --git a/common/src/main/java/org/tron/core/Constant.java b/common/src/main/java/org/tron/core/Constant.java index 3e859f6fd96..54ff30d9e65 100644 --- a/common/src/main/java/org/tron/core/Constant.java +++ b/common/src/main/java/org/tron/core/Constant.java @@ -256,6 +256,8 @@ public class Constant { public static final String RATE_LIMITER_GLOBAL_IP_QPS = "rate.limiter.global.ip.qps"; + public static final String RATE_LIMITER_GLOBAL_API_QPS = "rate.limiter.global.api.qps"; + public static final String COMMITTEE_CHANGED_DELEGATION = "committee.changedDelegation"; public static final String CRYPTO_ENGINE = "crypto.engine"; diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 74a755aca7f..b3a6821f3b2 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -224,6 +224,7 @@ public static void clearParam() { PARAMETER.memoFee = 0; PARAMETER.rateLimiterGlobalQps = 50000; PARAMETER.rateLimiterGlobalIpQps = 10000; + PARAMETER.rateLimiterGlobalApiQps = 1000; PARAMETER.p2pDisable = false; PARAMETER.dynamicConfigEnable = false; PARAMETER.dynamicConfigCheckInterval = 600; @@ -957,6 +958,10 @@ public static void setParam(final String[] args, final String confFileName) { config.hasPath(Constant.RATE_LIMITER_GLOBAL_IP_QPS) ? config .getInt(Constant.RATE_LIMITER_GLOBAL_IP_QPS) : 10000; + PARAMETER.rateLimiterGlobalApiQps = + config.hasPath(Constant.RATE_LIMITER_GLOBAL_API_QPS) ? config + .getInt(Constant.RATE_LIMITER_GLOBAL_API_QPS) : 1000; + PARAMETER.rateLimiterInitialization = getRateLimiterFromConfig(config); PARAMETER.changedDelegation = @@ -1627,6 +1632,7 @@ public static void logConfig() { logger.info("Node effective check enable: {}", parameter.isNodeEffectiveCheckEnable()); logger.info("Rate limiter global qps: {}", parameter.getRateLimiterGlobalQps()); logger.info("Rate limiter global ip qps: {}", parameter.getRateLimiterGlobalIpQps()); + logger.info("Rate limiter global api qps: {}", parameter.getRateLimiterGlobalApiQps()); logger.info("************************ Backup config ************************"); logger.info("Backup priority: {}", parameter.getBackupPriority()); logger.info("Backup listen port: {}", parameter.getBackupPort()); diff --git a/framework/src/main/java/org/tron/core/services/http/RateLimiterServlet.java b/framework/src/main/java/org/tron/core/services/http/RateLimiterServlet.java index eb8b7b86257..062750f914e 100644 --- a/framework/src/main/java/org/tron/core/services/http/RateLimiterServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/RateLimiterServlet.java @@ -31,6 +31,7 @@ public abstract class RateLimiterServlet extends HttpServlet { private static final String KEY_PREFIX_HTTP = "http_"; private static final String ADAPTER_PREFIX = "org.tron.core.services.ratelimiter.adapter."; + private static final int QPS = Args.getInstance().getRateLimiterGlobalApiQps(); @Autowired private RateLimiterContainer container; @@ -62,7 +63,7 @@ private void addRateContainer() { } else { constructor = c.getConstructor(); - obj = constructor.newInstance(); + obj = constructor.newInstance("qps=" + QPS); container.add(KEY_PREFIX_HTTP, getClass().getSimpleName(), (IRateLimiter) obj); } success = true; @@ -78,7 +79,7 @@ private void addRateContainer() { // if the specific rate limiter strategy of servlet is not defined or fail to add, // then add a default Strategy. try { - IRateLimiter rateLimiter = new DefaultBaseQqsAdapter("qps=1000"); + IRateLimiter rateLimiter = new DefaultBaseQqsAdapter("qps=" + QPS); container.add(KEY_PREFIX_HTTP, getClass().getSimpleName(), rateLimiter); } catch (Exception e) { logger.warn( diff --git a/framework/src/main/java/org/tron/core/services/ratelimiter/RateLimiterInterceptor.java b/framework/src/main/java/org/tron/core/services/ratelimiter/RateLimiterInterceptor.java index d629ae05abd..d6803dc76d7 100644 --- a/framework/src/main/java/org/tron/core/services/ratelimiter/RateLimiterInterceptor.java +++ b/framework/src/main/java/org/tron/core/services/ratelimiter/RateLimiterInterceptor.java @@ -31,6 +31,8 @@ public class RateLimiterInterceptor implements ServerInterceptor { private static final String KEY_PREFIX_RPC = "rpc_"; + private static final int QPS = Args.getInstance().getRateLimiterGlobalApiQps(); + @Autowired private RateLimiterContainer container; @@ -41,7 +43,7 @@ public void init(Server server) { for (ServerServiceDefinition service : server.getServices()) { for (ServerMethodDefinition method : service.getMethods()) { container.add(KEY_PREFIX_RPC, method.getMethodDescriptor().getFullMethodName(), - new DefaultBaseQqsAdapter("qps=1000")); + new DefaultBaseQqsAdapter("qps=" + QPS)); } } diff --git a/framework/src/main/java/org/tron/core/services/ratelimiter/strategy/QpsStrategy.java b/framework/src/main/java/org/tron/core/services/ratelimiter/strategy/QpsStrategy.java index a4d56af8ba2..8c18af34da9 100644 --- a/framework/src/main/java/org/tron/core/services/ratelimiter/strategy/QpsStrategy.java +++ b/framework/src/main/java/org/tron/core/services/ratelimiter/strategy/QpsStrategy.java @@ -4,12 +4,13 @@ import java.util.HashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.tron.core.config.args.Args; @Slf4j public class QpsStrategy extends Strategy { public static final String STRATEGY_PARAM_QPS = "qps"; - public static final Double DEFAULT_QPS = 100D; + public static final int DEFAULT_QPS = Args.getInstance().getRateLimiterGlobalApiQps(); private RateLimiter rateLimiter; public QpsStrategy(String paramString) { diff --git a/framework/src/test/java/org/tron/common/config/args/ArgsTest.java b/framework/src/test/java/org/tron/common/config/args/ArgsTest.java index 2e29b38824f..aa9ac14964d 100644 --- a/framework/src/test/java/org/tron/common/config/args/ArgsTest.java +++ b/framework/src/test/java/org/tron/common/config/args/ArgsTest.java @@ -1,13 +1,21 @@ package org.tron.common.config.args; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URL; +import java.net.URLConnection; + import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.tron.common.client.WalletGrpcClient; import org.tron.common.parameter.RateLimiterInitialization; +import org.tron.common.utils.client.GrpcClient; +import org.tron.common.utils.client.WalletClient; import org.tron.core.Constant; import org.tron.core.config.args.Args; @@ -31,6 +39,7 @@ public void destroy() { @Test public void testConfig() { + Args.logConfig(); Assert.assertEquals(Args.getInstance().getMaxTransactionPendingSize(), 2000); Assert.assertEquals(Args.getInstance().getPendingTransactionTimeout(), 60_000); Assert.assertEquals(Args.getInstance().getMaxFastForwardNum(), 3); @@ -39,6 +48,7 @@ public void testConfig() { Assert.assertFalse(Args.getInstance().isNodeEffectiveCheckEnable()); Assert.assertEquals(Args.getInstance().getRateLimiterGlobalQps(), 1000); Assert.assertEquals(Args.getInstance().getRateLimiterGlobalIpQps(), 1000); + Assert.assertEquals(Args.getInstance().getRateLimiterGlobalApiQps(), 100); Assert.assertEquals(Args.getInstance().p2pDisable, true); Assert.assertEquals(Args.getInstance().getMaxTps(), 1000); RateLimiterInitialization rateLimiter = Args.getInstance().getRateLimiterInitialization(); diff --git a/framework/src/test/resources/config-test.conf b/framework/src/test/resources/config-test.conf index d506521965f..236502d3360 100644 --- a/framework/src/test/resources/config-test.conf +++ b/framework/src/test/resources/config-test.conf @@ -348,6 +348,7 @@ committee = { rate.limiter.global.qps = 1000 rate.limiter.global.ip.qps = 1000 +rate.limiter.global.api.qps = 100 rate.limiter.http = [ { component = "GetNowBlockServlet",