Skip to content

Commit 0194f56

Browse files
IanMeyersIanMeyers
authored
Merge pull request #89 from IanMeyers/master
Version .9.7.0
2 parents 264ec43 + bb07114 commit 0194f56

17 files changed

+369
-350
lines changed

‎README.md‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -44,24 +44,24 @@ The Kinesis Autoscaling WAR can be deployed as an Elastic Beanstalk application,
4444

4545
![AutoscalingGraph](https://s3-eu-west-1.amazonaws.com/meyersi-ire-aws/KinesisScalingUtility/img/KinesisAutoscalingGraph.png)
4646

47-
To get started, create a new Elastic Beanstalk application which is a Web Server with a Tomcat predefined configuration. Deploy the WAR by uploading from your local GitHub copy of [dist/KinesisAutoscaling-.9.6.0.war](dist/KinesisAutoscaling-.9.6.0.war), or using the following S3 URLs:
47+
To get started, create a new Elastic Beanstalk application which is a Web Server with a Tomcat predefined configuration. Deploy the WAR by uploading from your local GitHub copy of [dist/KinesisAutoscaling-.9.7.0.war](dist/KinesisAutoscaling-.9.7.0.war), or using the following S3 URLs:
4848

4949
| region| S3 Path |
5050
| ----- | ------- |
51-
| ap-northeast-1 | https://s3.ap-northeast-1.amazonaws.com/awslabs-code-ap-northeast-1/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
52-
| ap-northeast-2 | https://s3.ap-northeast-2.amazonaws.com/awslabs-code-ap-northeast-2/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
53-
| ap-south-1 | https://s3.ap-south-1.amazonaws.com/awslabs-code-ap-south-1/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
54-
| ap-southeast-1 | https://s3.ap-southeast-1.amazonaws.com/awslabs-code-ap-southeast-1/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
55-
| ap-southeast-2 | https://s3.ap-southeast-2.amazonaws.com/awslabs-code-ap-southeast-2/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
56-
| ca-central-1 | https://s3.ca-central-1.amazonaws.com/awslabs-code-ca-central-1/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
57-
| eu-central-1 | https://s3.eu-central-1.amazonaws.com/awslabs-code-eu-central-1/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
58-
| eu-west-1 | https://s3.eu-west-1.amazonaws.com/awslabs-code-eu-west-1/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
59-
| eu-west-2 | https://s3.eu-west-2.amazonaws.com/awslabs-code-eu-west-2/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
60-
| sa-east-1 | https://s3.sa-east-1.amazonaws.com/awslabs-code-sa-east-1/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
61-
| us-east-1 | https://s3.us-east-1.amazonaws.com/awslabs-code-us-east-1/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
62-
| us-east-2 | https://s3.us-east-2.amazonaws.com/awslabs-code-us-east-2/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
63-
| us-west-1 | https://s3.us-west-1.amazonaws.com/awslabs-code-us-west-1/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
64-
| us-west-2 | https://s3.us-west-2.amazonaws.com/awslabs-code-us-west-2/KinesisAutoscaling/KinesisAutoscaling-.9.6.0.war |
51+
| ap-northeast-1 | https://s3.ap-northeast-1.amazonaws.com/awslabs-code-ap-northeast-1/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
52+
| ap-northeast-2 | https://s3.ap-northeast-2.amazonaws.com/awslabs-code-ap-northeast-2/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
53+
| ap-south-1 | https://s3.ap-south-1.amazonaws.com/awslabs-code-ap-south-1/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
54+
| ap-southeast-1 | https://s3.ap-southeast-1.amazonaws.com/awslabs-code-ap-southeast-1/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
55+
| ap-southeast-2 | https://s3.ap-southeast-2.amazonaws.com/awslabs-code-ap-southeast-2/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
56+
| ca-central-1 | https://s3.ca-central-1.amazonaws.com/awslabs-code-ca-central-1/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
57+
| eu-central-1 | https://s3.eu-central-1.amazonaws.com/awslabs-code-eu-central-1/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
58+
| eu-west-1 | https://s3.eu-west-1.amazonaws.com/awslabs-code-eu-west-1/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
59+
| eu-west-2 | https://s3.eu-west-2.amazonaws.com/awslabs-code-eu-west-2/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
60+
| sa-east-1 | https://s3.sa-east-1.amazonaws.com/awslabs-code-sa-east-1/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
61+
| us-east-1 | https://s3.us-east-1.amazonaws.com/awslabs-code-us-east-1/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
62+
| us-east-2 | https://s3.us-east-2.amazonaws.com/awslabs-code-us-east-2/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
63+
| us-west-1 | https://s3.us-west-1.amazonaws.com/awslabs-code-us-west-1/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
64+
| us-west-2 | https://s3.us-west-2.amazonaws.com/awslabs-code-us-west-2/KinesisAutoscaling/KinesisAutoscaling-.9.7.0.war |
6565

6666
Once deployed, you must configure the Autoscaling engine by providing a JSON configuration file on an HTTP or S3 URL. The structure of this configuration file is as follows:
6767

‎bin/distribute.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22
# set -x
33

4-
ver=.9.6.0
4+
ver=.9.7.0
55

66
# publish war to regional buckets
77
for r in `aws ec2 describe-regions --query Regions[*].RegionName --output text`; do aws s3 cp ../dist/KinesisAutoscaling-$ver.war s3://awslabs-code-$r/KinesisAutoscaling/ --acl public-read --region $r; done

‎pom.xml‎

Lines changed: 55 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1-
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
1+
<project xmlns="http://maven.apache.org/POM/4.0.0"
2+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
23
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
34
<modelVersion>4.0.0</modelVersion>
45
<groupId>com.amazonaws</groupId>
56
<artifactId>kinesis-scaling-utils</artifactId>
6-
<version>.9.6.0</version>
7+
<version>.9.7.0</version>
78
<properties>
8-
<sdk-version>1.11.452</sdk-version>
9+
<sdk-version>2.13.39</sdk-version>
910
</properties>
1011
<licenses>
1112
<license>
@@ -22,6 +23,7 @@
2223
<includes>
2324
<include>README</include>
2425
<include>*.properties</include>
26+
<include>*.xml</include>
2527
</includes>
2628
</resource>
2729
</resources>
@@ -95,28 +97,34 @@
9597
</build>
9698
<dependencies>
9799
<dependency>
98-
<groupId>com.amazonaws</groupId>
99-
<artifactId>aws-java-sdk-core</artifactId>
100+
<groupId>software.amazon.awssdk</groupId>
101+
<artifactId>sdk-core</artifactId>
100102
<version>${sdk-version}</version>
101103
</dependency>
102104
<dependency>
103-
<groupId>com.amazonaws</groupId>
104-
<artifactId>aws-java-sdk-kinesis</artifactId>
105+
<groupId>software.amazon.awssdk</groupId>
106+
<artifactId>kinesis</artifactId>
105107
<version>${sdk-version}</version>
108+
<exclusions>
109+
<exclusion>
110+
<groupId>commons-logging</groupId>
111+
<artifactId>commons-logging</artifactId>
112+
</exclusion>
113+
</exclusions>
106114
</dependency>
107115
<dependency>
108-
<groupId>com.amazonaws</groupId>
109-
<artifactId>aws-java-sdk-s3</artifactId>
116+
<groupId>software.amazon.awssdk</groupId>
117+
<artifactId>s3</artifactId>
110118
<version>${sdk-version}</version>
111119
</dependency>
112120
<dependency>
113-
<groupId>com.amazonaws</groupId>
114-
<artifactId>aws-java-sdk-sns</artifactId>
121+
<groupId>software.amazon.awssdk</groupId>
122+
<artifactId>sns</artifactId>
115123
<version>${sdk-version}</version>
116124
</dependency>
117125
<dependency>
118-
<groupId>com.amazonaws</groupId>
119-
<artifactId>aws-java-sdk-cloudwatch</artifactId>
126+
<groupId>software.amazon.awssdk</groupId>
127+
<artifactId>cloudwatch</artifactId>
120128
<version>${sdk-version}</version>
121129
</dependency>
122130
<dependency>
@@ -133,7 +141,7 @@
133141
<dependency>
134142
<groupId>commons-io</groupId>
135143
<artifactId>commons-io</artifactId>
136-
<version>LATEST</version>
144+
<version>2.7</version>
137145
</dependency>
138146
<dependency>
139147
<groupId>joda-time</groupId>
@@ -152,9 +160,39 @@
152160
<version>2.7.4</version>
153161
</dependency>
154162
<dependency>
155-
<groupId>log4j</groupId>
156-
<artifactId>log4j</artifactId>
157-
<version>1.2.12</version>
163+
<groupId>org.slf4j</groupId>
164+
<artifactId>slf4j-api</artifactId>
165+
<version>1.7.30</version>
166+
</dependency>
167+
<dependency>
168+
<groupId>ch.qos.logback</groupId>
169+
<artifactId>logback-core</artifactId>
170+
<version>1.2.3</version>
171+
</dependency>
172+
<dependency>
173+
<groupId>ch.qos.logback</groupId>
174+
<artifactId>logback-classic</artifactId>
175+
<version>1.2.3</version>
176+
</dependency>
177+
<dependency>
178+
<groupId>software.amazon.awssdk</groupId>
179+
<artifactId>http-client-spi</artifactId>
180+
<version>2.13.39</version>
181+
</dependency>
182+
<dependency>
183+
<groupId>com.fasterxml.jackson.core</groupId>
184+
<artifactId>jackson-annotations</artifactId>
185+
<version>2.11.0</version>
186+
</dependency>
187+
<dependency>
188+
<groupId>org.slf4j</groupId>
189+
<artifactId>jcl-over-slf4j</artifactId>
190+
<version>1.7.30</version>
191+
</dependency>
192+
<dependency>
193+
<groupId>org.apache.commons</groupId>
194+
<artifactId>commons-lang3</artifactId>
195+
<version>3.10</version>
158196
</dependency>
159197
</dependencies>
160198
<url>https://github.com/awslabs/amazon-kinesis-scaling-utils</url>

‎src/main/java/ScalingClient.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@
66
*
77
* SPDX-License-Identifier: Apache-2.0
88
*/
9-
import com.amazonaws.regions.Region;
10-
import com.amazonaws.regions.Regions;
119
import com.amazonaws.services.kinesis.scaling.ScaleDirection;
1210
import com.amazonaws.services.kinesis.scaling.ScalingCompletionStatus;
1311
import com.amazonaws.services.kinesis.scaling.ScalingOperationReport;
1412
import com.amazonaws.services.kinesis.scaling.StreamScaler;
1513
import com.amazonaws.services.kinesis.scaling.StreamScaler.ScalingAction;
1614

15+
import software.amazon.awssdk.regions.Region;
16+
1717
/**
1818
* Class which provides a host environment interface to working with the Kinesis
1919
* Scaling Utility<br>
@@ -76,7 +76,7 @@ public class ScalingClient {
7676

7777
private String shardId;
7878

79-
private Region region = Region.getRegion(Regions.US_EAST_1);
79+
private Region region = Region.US_EAST_1;
8080

8181
private ScalingAction scalingAction;
8282

@@ -112,7 +112,7 @@ private void loadParams() throws Exception {
112112
}
113113

114114
if (System.getProperty(REGION_PARAM) != null) {
115-
this.region = Region.getRegion(Regions.fromName(System.getProperty(REGION_PARAM)));
115+
this.region = Region.of(System.getProperty(REGION_PARAM));
116116
}
117117

118118
if (System.getProperty(WAIT_FOR_COMPLETION) != null) {

‎src/main/java/com/amazonaws/services/kinesis/scaling/AdjacentShards.java‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,7 @@
1010
import java.math.BigInteger;
1111
import java.util.Map;
1212

13-
import com.amazonaws.services.kinesis.AmazonKinesisClient;
14-
import com.amazonaws.services.kinesis.model.LimitExceededException;
15-
import com.amazonaws.services.kinesis.model.ResourceInUseException;
13+
import software.amazon.awssdk.services.kinesis.KinesisClient;
1614

1715
/**
1816
* AdjacentShards are a transfer object for maintaining references between an
@@ -27,8 +25,8 @@ public class AdjacentShards {
2725

2826
public AdjacentShards(String streamName, ShardHashInfo lower, ShardHashInfo higher) throws Exception {
2927
// ensure that the shards are adjacent
30-
if (!new BigInteger(higher.getShard().getHashKeyRange().getStartingHashKey())
31-
.subtract(new BigInteger(lower.getShard().getHashKeyRange().getEndingHashKey()))
28+
if (!new BigInteger(higher.getShard().hashKeyRange().startingHashKey())
29+
.subtract(new BigInteger(lower.getShard().hashKeyRange().endingHashKey()))
3230
.equals(new BigInteger("1"))) {
3331
throw new Exception("Shards are not Adjacent");
3432
}
@@ -52,15 +50,15 @@ protected ShardHashInfo getHigherShard() {
5250
* @return
5351
* @throws Exception
5452
*/
55-
protected ShardHashInfo doMerge(AmazonKinesisClient kinesisClient, String currentHighestShardId) throws Exception {
53+
protected ShardHashInfo doMerge(KinesisClient kinesisClient, String currentHighestShardId) throws Exception {
5654
StreamScalingUtils.mergeShards(kinesisClient, streamName, this.lowerShard, this.higherShard, true);
5755

5856
Map<String, ShardHashInfo> openShards = StreamScalingUtils.getOpenShards(kinesisClient, streamName,
5957
currentHighestShardId);
6058

6159
for (ShardHashInfo info : openShards.values()) {
62-
if (lowerShard.getShardId().equals(info.getShard().getParentShardId())
63-
&& higherShard.getShardId().equals(info.getShard().getAdjacentParentShardId())) {
60+
if (lowerShard.getShardId().equals(info.getShard().parentShardId())
61+
&& higherShard.getShardId().equals(info.getShard().adjacentParentShardId())) {
6462
return new ShardHashInfo(streamName, info.getShard());
6563
}
6664
}

‎src/main/java/com/amazonaws/services/kinesis/scaling/ShardHashInfo.java‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,12 @@
1313
import java.text.NumberFormat;
1414
import java.util.Map;
1515

16-
import com.amazonaws.services.kinesis.AmazonKinesisClient;
17-
import com.amazonaws.services.kinesis.model.Shard;
1816
import com.fasterxml.jackson.annotation.JsonProperty;
1917
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
2018

19+
import software.amazon.awssdk.services.kinesis.KinesisClient;
20+
import software.amazon.awssdk.services.kinesis.model.Shard;
21+
2122
/**
2223
* Immutable transfer object containing enhanced metadata about Shards in a
2324
* Stream, as well as utility methods for working with a Stream of Shards
@@ -53,8 +54,8 @@ public ShardHashInfo(String streamName, Shard shard) {
5354
}
5455
this.shard = shard;
5556
this.streamName = streamName;
56-
this.endHash = new BigInteger(shard.getHashKeyRange().getEndingHashKey());
57-
this.startHash = new BigInteger(shard.getHashKeyRange().getStartingHashKey());
57+
this.endHash = new BigInteger(shard.hashKeyRange().endingHashKey());
58+
this.startHash = new BigInteger(shard.hashKeyRange().startingHashKey());
5859
this.hashWidth = getWidth(this.startHash, this.endHash);
5960
this.pctOfKeyspace = getPctOfKeyspace(this.hashWidth);
6061
}
@@ -74,7 +75,7 @@ public static Double getPctOfKeyspace(BigInteger hashWidth) {
7475

7576
@JsonProperty("shardID")
7677
protected String getShardId() {
77-
return this.shard.getShardId();
78+
return this.shard.shardId();
7879
}
7980

8081
protected Shard getShard() {
@@ -125,7 +126,7 @@ public String getStreamName() {
125126
* @return
126127
* @throws Exception
127128
*/
128-
public AdjacentShards doSplit(AmazonKinesisClient kinesisClient, double targetPct, String currentHighestShardId)
129+
public AdjacentShards doSplit(KinesisClient kinesisClient, double targetPct, String currentHighestShardId)
129130
throws Exception {
130131
BigInteger targetHash = getHashAtPctOffset(targetPct);
131132

@@ -140,8 +141,8 @@ public AdjacentShards doSplit(AmazonKinesisClient kinesisClient, double targetPc
140141
currentHighestShardId);
141142

142143
for (ShardHashInfo info : openShards.values()) {
143-
if (!info.getShard().getShardId().equals(this.shard.getShardId())) {
144-
if (info.getShard().getHashKeyRange().getStartingHashKey().equals(targetHash.toString())) {
144+
if (!info.getShard().shardId().equals(this.shard.shardId())) {
145+
if (info.getShard().hashKeyRange().startingHashKey().equals(targetHash.toString())) {
145146
higherShard = new ShardHashInfo(this.streamName, info.getShard());
146147
break;
147148
} else {

0 commit comments

Comments
 (0)