From be9cf41d4a090182427c2bc4834aa909558809f1 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Wed, 7 Jun 2023 20:10:57 +0000 Subject: [PATCH 1/3] This change includes: 1. Adding TSERV_GROUP_NAME property to add resource groups to TabletServers like has been done for Compactors and ScanServers 2. Modified TServerInstance to include the group in the location information. 3. Modified TabletMetadata to remove the session from the location columns colq and put all of the location information in the column value. 4. Modified MetricsUtil to add the servers resource group as a tag to all emitted metrics 5. Modified CachedTablet to use the TServerInstance instead of interned strings for the server and session. --- .../core/clientImpl/ClientContext.java | 2 +- .../core/clientImpl/ClientTabletCache.java | 42 ++--- .../clientImpl/ClientTabletCacheImpl.java | 10 +- .../clientImpl/RootClientTabletCache.java | 13 +- .../core/clientImpl/TableOperationsImpl.java | 2 +- .../apache/accumulo/core/conf/Property.java | 14 +- .../accumulo/core/lock/ServiceLockData.java | 9 - .../manager/balancer/TabletServerIdImpl.java | 11 +- .../MetadataCachedTabletObtainer.java | 18 +- .../core/metadata/TServerInstance.java | 70 +++++--- .../core/metadata/schema/TabletMetadata.java | 54 ++---- .../accumulo/core/metrics/MetricsUtil.java | 18 +- .../core/spi/balancer/GroupBalancer.java | 5 +- .../core/spi/balancer/SimpleLoadBalancer.java | 6 +- .../spi/balancer/data/TabletServerId.java | 2 + .../accumulo/core/summary/Gatherer.java | 6 +- .../clientImpl/ClientTabletCacheImplTest.java | 110 ++++++------ .../core/lock/ServiceLockDataTest.java | 9 +- .../metadata/schema/TabletMetadataTest.java | 55 +++--- .../BaseHostRegexTableLoadBalancerTest.java | 47 ++--- .../core/spi/balancer/GroupBalancerTest.java | 10 +- .../HostRegexTableLoadBalancerTest.java | 166 ++++++------------ .../spi/balancer/SimpleLoadBalancerTest.java | 19 +- .../spi/balancer/TableLoadBalancerTest.java | 3 +- .../accumulo/server/AbstractServer.java | 12 ++ .../constraints/MetadataConstraints.java | 17 ++ .../accumulo/server/log/WalStateManager.java | 2 +- .../server/manager/LiveTServerSet.java | 7 +- .../state/LoggingTabletStateStore.java | 2 +- .../state/TabletManagementIterator.java | 14 +- .../ConditionalTabletMutatorImpl.java | 4 +- .../server/metadata/TabletMutatorBase.java | 6 +- .../server/util/ManagerMetadataUtil.java | 22 ++- .../constraints/MetadataConstraintsTest.java | 7 +- .../server/manager/LiveTServerSetTest.java | 6 +- .../state/RootTabletStateStoreTest.java | 5 +- .../manager/state/TabletManagementTest.java | 7 +- .../server/util/ManagerMetadataUtilTest.java | 7 +- .../apache/accumulo/compactor/Compactor.java | 25 +-- .../accumulo/gc/SimpleGarbageCollector.java | 7 +- .../gc/GarbageCollectWriteAheadLogsTest.java | 6 +- .../org/apache/accumulo/manager/Manager.java | 7 +- .../accumulo/manager/TabletGroupWatcher.java | 5 +- .../compaction/CompactionFinalizer.java | 5 +- .../tableOps/bulkVer2/RefreshTablets.java | 4 +- .../manager/tserverOps/ShutdownTServer.java | 4 +- .../compaction/CompactionCoordinatorTest.java | 8 +- .../compaction/QueueSummariesTest.java | 3 +- .../manager/tableOps/ShutdownTServerTest.java | 4 +- .../org/apache/accumulo/monitor/Monitor.java | 4 +- .../monitor/rest/tables/TablesResource.java | 2 +- .../apache/accumulo/tserver/ScanServer.java | 14 +- .../apache/accumulo/tserver/TabletServer.java | 17 +- .../accumulo/tserver/UnloadTabletHandler.java | 4 +- .../tserver/tablet/DatafileManager.java | 3 +- .../accumulo/tserver/tablet/Tablet.java | 2 +- .../tserver/CheckTabletMetadataTest.java | 26 ++- .../shell/commands/ListTabletsCommand.java | 2 +- .../commands/ListTabletsCommandTest.java | 10 +- .../org/apache/accumulo/test/LocatorIT.java | 2 +- .../functional/AmpleConditionalWriterIT.java | 13 +- .../test/functional/AssignLocationModeIT.java | 15 +- .../functional/CompactLocationModeIT.java | 15 +- .../test/functional/ManagerAssignmentIT.java | 18 +- .../test/functional/RegexGroupBalanceIT.java | 4 +- .../test/functional/SplitRecoveryIT.java | 4 +- .../TabletManagementIteratorIT.java | 4 +- .../accumulo/test/manager/MergeStateIT.java | 5 +- .../test/manager/SuspendedTabletsIT.java | 8 +- .../test/performance/NullTserver.java | 4 +- .../test/ChaoticLoadBalancerTest.java | 16 +- 71 files changed, 588 insertions(+), 501 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java index daa3dd35da2..07dca96f8a0 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java @@ -498,7 +498,7 @@ public String getRootTabletLocation() { return null; } - return loc.getHostPort(); + return loc.getServerInstance().getHostPort(); } /** diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java index fb8c4f7a8c4..014cdadcd50 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java @@ -43,9 +43,9 @@ import org.apache.accumulo.core.metadata.MetadataCachedTabletObtainer; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.singletons.SingletonManager; import org.apache.accumulo.core.singletons.SingletonService; -import org.apache.accumulo.core.util.Interner; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.hadoop.io.Text; @@ -270,34 +270,29 @@ public List getCachedTablets() { } public static class CachedTablet { - private static final Interner interner = new Interner<>(); private final KeyExtent tablet_extent; - private final String tserverLocation; - private final String tserverSession; + private final TServerInstance tsi; private final TabletHostingGoal goal; private final boolean hostingRequested; private final Long creationTime = System.nanoTime(); - public CachedTablet(KeyExtent tablet_extent, String tablet_location, String session, - TabletHostingGoal goal, boolean hostingRequested) { + public CachedTablet(KeyExtent tablet_extent, TServerInstance instance, TabletHostingGoal goal, + boolean hostingRequested) { checkArgument(tablet_extent != null, "tablet_extent is null"); - checkArgument(tablet_location != null, "tablet_location is null"); - checkArgument(session != null, "session is null"); + checkArgument(instance != null, "tablet_location is null"); this.tablet_extent = tablet_extent; - this.tserverLocation = interner.intern(tablet_location); - this.tserverSession = interner.intern(session); + this.tsi = instance; this.goal = Objects.requireNonNull(goal); this.hostingRequested = hostingRequested; } - public CachedTablet(KeyExtent tablet_extent, Optional tablet_location, - Optional session, TabletHostingGoal goal, boolean hostingRequested) { + public CachedTablet(KeyExtent tablet_extent, Optional tablet_location, + TabletHostingGoal goal, boolean hostingRequested) { checkArgument(tablet_extent != null, "tablet_extent is null"); this.tablet_extent = tablet_extent; - this.tserverLocation = tablet_location.map(interner::intern).orElse(null); - this.tserverSession = session.map(interner::intern).orElse(null); + this.tsi = tablet_location.orElse(null); this.goal = Objects.requireNonNull(goal); this.hostingRequested = hostingRequested; } @@ -305,8 +300,7 @@ public CachedTablet(KeyExtent tablet_extent, Optional tablet_location, public CachedTablet(KeyExtent tablet_extent, TabletHostingGoal goal, boolean hostingRequested) { checkArgument(tablet_extent != null, "tablet_extent is null"); this.tablet_extent = tablet_extent; - this.tserverLocation = null; - this.tserverSession = null; + this.tsi = null; this.goal = Objects.requireNonNull(goal); this.hostingRequested = hostingRequested; } @@ -325,13 +319,13 @@ && getTserverSession().equals(otl.getTserverSession()) && getGoal() == otl.getGo @Override public int hashCode() { - return Objects.hash(getExtent(), tserverLocation, tserverSession, goal, hostingRequested); + return Objects.hash(getExtent(), tsi, goal, hostingRequested); } @Override public String toString() { return "(" + getExtent() + "," + getTserverLocation() + "," + getTserverSession() + "," - + getGoal() + ")"; + + getResourceGroup() + "," + getGoal() + ")"; } public KeyExtent getExtent() { @@ -339,11 +333,19 @@ public KeyExtent getExtent() { } public Optional getTserverLocation() { - return Optional.ofNullable(tserverLocation); + return Optional.ofNullable(tsi == null ? null : tsi.getHostPort()); } public Optional getTserverSession() { - return Optional.ofNullable(tserverSession); + return Optional.ofNullable(tsi == null ? null : tsi.getSession()); + } + + public Optional getResourceGroup() { + return Optional.ofNullable(tsi == null ? null : tsi.getGroup()); + } + + public Optional getServer() { + return Optional.ofNullable(tsi == null ? null : tsi); } /** diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java index 5e3e8bb440e..5c37c0a5a1c 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java @@ -541,8 +541,8 @@ public CachedTablet findTablet(ClientContext context, Text row, boolean skipRow, OpTimer timer = null; if (log.isTraceEnabled()) { - log.trace("tid={} Locating tablet table={} row={} skipRow={}", - Thread.currentThread().getId(), tableId, TextUtil.truncate(row), skipRow); + log.trace("tid={} Locating tablet table={} row={} skipRow={}", Thread.currentThread().getId(), + tableId, TextUtil.truncate(row), skipRow); timer = new OpTimer().start(); } @@ -686,8 +686,7 @@ private void lookupTablet(ClientContext context, Text row, boolean retry, if ((lastEndRow != null) && (ke.prevEndRow() != null) && ke.prevEndRow().equals(lastEndRow)) { locToCache = new CachedTablet(new KeyExtent(ke.tableId(), ke.endRow(), lastEndRow), - cachedTablet.getTserverLocation(), cachedTablet.getTserverSession(), - cachedTablet.getGoal(), cachedTablet.wasHostingRequested()); + cachedTablet.getServer(), cachedTablet.getGoal(), cachedTablet.wasHostingRequested()); } else { locToCache = cachedTablet; } @@ -695,8 +694,11 @@ private void lookupTablet(ClientContext context, Text row, boolean retry, // save endRow for next iteration lastEndRow = locToCache.getExtent().endRow(); + log.trace("Caching tablet location: {}", locToCache); updateCache(locToCache, lcSession); } + } else { + log.warn("Metadata tablet not found for row: {}", metadataRow); } } diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java index 7e35fb259e0..a9ad981307f 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java @@ -123,18 +123,17 @@ protected CachedTablet getRootTabletLocation(ClientContext context) { } if (loc == null || loc.getType() != LocationType.CURRENT) { - return new CachedTablet(RootTable.EXTENT, Optional.empty(), Optional.empty(), - TabletHostingGoal.ALWAYS, false); + return new CachedTablet(RootTable.EXTENT, Optional.empty(), TabletHostingGoal.ALWAYS, false); } - String server = loc.getHostPort(); + String server = loc.getServerInstance().getHostPort(); + String session = loc.getServerInstance().getSession(); - if (lockChecker.isLockHeld(server, loc.getSession())) { - return new CachedTablet(RootTable.EXTENT, server, loc.getSession(), TabletHostingGoal.ALWAYS, + if (lockChecker.isLockHeld(server, session)) { + return new CachedTablet(RootTable.EXTENT, loc.getServerInstance(), TabletHostingGoal.ALWAYS, false); } else { - return new CachedTablet(RootTable.EXTENT, Optional.empty(), Optional.empty(), - TabletHostingGoal.ALWAYS, false); + return new CachedTablet(RootTable.EXTENT, Optional.empty(), TabletHostingGoal.ALWAYS, false); } } diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java index a063cff2413..e9925993e59 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java @@ -1308,7 +1308,7 @@ private void waitForTableStateTransition(TableId tableId, TableState expectedSta lastRow = tablet.getExtent().toMetaRow(); if (loc != null) { - serverCounts.increment(loc.getHostPortSession(), 1); + serverCounts.increment(loc.getServerInstance().getHostPortSessionGroup(), 1); } } diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 5f652b57ae9..3a627837997 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.file.rfile.RFile; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iteratorsImpl.system.DeletingIterator; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner; import org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher; @@ -724,6 +725,11 @@ public enum Property { PropertyType.TIMEDURATION, "The interval at which the TabletServer will check if on-demand tablets can be unloaded", "4.0.0"), + TSERV_GROUP_NAME("tserver.group", ServiceDescriptor.DEFAULT_GROUP_NAME, PropertyType.STRING, + "Optional group name that will be made available to the " + + "TablerServer plugins. Groups can be used to dedicate resources " + + " to specific tables (e.g. balancing tablets for table(s) within a group)", + "4.0.0"), // accumulo garbage collector properties GC_PREFIX("gc.", null, PropertyType.PREFIX, @@ -808,6 +814,11 @@ public enum Property { + " global setting to take effect. However, you must use the API or the shell" + " to change properties in zookeeper that are set on a table.", "1.3.5"), + TABLE_ASSIGNMENT_GROUP("table.assignment.group", ServiceDescriptor.DEFAULT_GROUP_NAME, + PropertyType.STRING, + "Tablets for this table will be assigned to TabletServers that have a corresponding" + + " tserver.group property value.", + "4.0.0"), TABLE_ARBITRARY_PROP_PREFIX("table.custom.", null, PropertyType.PREFIX, "Prefix to be used for user defined arbitrary properties.", "1.7.0"), TABLE_MINC_OUTPUT_DROP_CACHE("table.compaction.minor.output.drop.cache", "false", @@ -1095,7 +1106,8 @@ public enum Property { COMPACTOR_MAX_MESSAGE_SIZE("compactor.message.size.max", "10M", PropertyType.BYTES, "The maximum size of a message that can be sent to a tablet server.", "2.1.0"), @Experimental - COMPACTOR_QUEUE_NAME("compactor.queue", "", PropertyType.STRING, + // ELASTICITY_TODO: rename to compactor.group + COMPACTOR_QUEUE_NAME("compactor.queue", ServiceDescriptor.DEFAULT_GROUP_NAME, PropertyType.STRING, "The queue for which this Compactor will perform compactions", "3.0.0"), // CompactionCoordinator properties @Experimental diff --git a/core/src/main/java/org/apache/accumulo/core/lock/ServiceLockData.java b/core/src/main/java/org/apache/accumulo/core/lock/ServiceLockData.java index 746d5b5ce59..628db2c3580 100644 --- a/core/src/main/java/org/apache/accumulo/core/lock/ServiceLockData.java +++ b/core/src/main/java/org/apache/accumulo/core/lock/ServiceLockData.java @@ -70,10 +70,6 @@ public static class ServiceDescriptor { private final String address; private final String group; - public ServiceDescriptor(UUID uuid, ThriftService service, String address) { - this(uuid, service, address, DEFAULT_GROUP_NAME); - } - public ServiceDescriptor(UUID uuid, ThriftService service, String address, String group) { this.uuid = requireNonNull(uuid); this.service = requireNonNull(service); @@ -159,11 +155,6 @@ public ServiceLockData(UUID uuid, String address, ThriftService service, String Collections.singleton(new ServiceDescriptor(uuid, service, address, group))))); } - public ServiceLockData(UUID uuid, String address, ThriftService service) { - this(new ServiceDescriptors( - new HashSet<>(Collections.singleton(new ServiceDescriptor(uuid, service, address))))); - } - public String getAddressString(ThriftService service) { ServiceDescriptor sd = services.get(service); if (sd == null) { diff --git a/core/src/main/java/org/apache/accumulo/core/manager/balancer/TabletServerIdImpl.java b/core/src/main/java/org/apache/accumulo/core/manager/balancer/TabletServerIdImpl.java index 2fefb466d33..7da93ff3ea3 100644 --- a/core/src/main/java/org/apache/accumulo/core/manager/balancer/TabletServerIdImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/manager/balancer/TabletServerIdImpl.java @@ -35,9 +35,9 @@ public static TabletServerIdImpl fromThrift(TServerInstance tsi) { return (tsi == null) ? null : new TabletServerIdImpl(tsi); } - public TabletServerIdImpl(String host, int port, String session) { + public TabletServerIdImpl(String host, int port, String session, String group) { requireNonNull(host); - this.tServerInstance = new TServerInstance(HostAndPort.fromParts(host, port), session); + this.tServerInstance = new TServerInstance(HostAndPort.fromParts(host, port), session, group); } public TabletServerIdImpl(TServerInstance tServerInstance) { @@ -59,6 +59,11 @@ public String getSession() { return tServerInstance.getSession(); } + @Override + public String getGroup() { + return tServerInstance.getGroup(); + } + @Override public int compareTo(TabletServerId o) { return tServerInstance.compareTo(((TabletServerIdImpl) o).tServerInstance); @@ -96,7 +101,7 @@ public static TServerInstance toThrift(TabletServerId tabletServerId) { } else { return new TServerInstance( HostAndPort.fromParts(tabletServerId.getHost(), tabletServerId.getPort()), - tabletServerId.getSession()); + tabletServerId.getSession(), tabletServerId.getGroup()); } } } diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataCachedTabletObtainer.java b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataCachedTabletObtainer.java index 73ed8528f66..e65b455dfda 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataCachedTabletObtainer.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataCachedTabletObtainer.java @@ -220,8 +220,7 @@ public List lookupTablets(ClientContext context, String tserver, } public static CachedTablets getMetadataLocationEntries(SortedMap entries) { - Text location = null; - Text session = null; + TServerInstance tsi = null; TabletHostingGoal goal = null; boolean hostingRequested = false; @@ -238,8 +237,7 @@ public static CachedTablets getMetadataLocationEntries(SortedMap entr Value val = entry.getValue(); if (key.compareRow(lastRowFromKey) != 0) { - location = null; - session = null; + tsi = null; goal = null; hostingRequested = false; key.getRow(lastRowFromKey); @@ -251,11 +249,10 @@ public static CachedTablets getMetadataLocationEntries(SortedMap entr // interpret the row id as a key extent if (colf.equals(CurrentLocationColumnFamily.NAME) || colf.equals(FutureLocationColumnFamily.NAME)) { - if (location != null) { + if (tsi != null) { throw new IllegalStateException("Tablet has multiple locations : " + lastRowFromKey); } - location = new Text(val.toString()); - session = new Text(colq); + tsi = TServerInstance.fromString(val.toString()); } else if (HostingColumnFamily.GOAL_COLUMN.equals(colf, colq)) { goal = TabletHostingGoalUtil.fromValue(val); } else if (HostingColumnFamily.REQUESTED_COLUMN.equals(colf, colq)) { @@ -265,13 +262,12 @@ public static CachedTablets getMetadataLocationEntries(SortedMap entr if (ke.isMeta()) { goal = TabletHostingGoal.ALWAYS; } - if (location != null) { - results.add(new CachedTablet(ke, location.toString(), session.toString(), goal, - hostingRequested)); + if (tsi != null) { + results.add(new CachedTablet(ke, tsi, goal, hostingRequested)); } else { results.add(new CachedTablet(ke, goal, hostingRequested)); } - location = null; + tsi = null; } } diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java b/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java index 56aaa06a8ae..3f0dc2d6781 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java @@ -20,6 +20,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import java.util.Objects; + import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.util.AddressUtil; import org.apache.hadoop.io.Text; @@ -34,39 +36,49 @@ */ public class TServerInstance implements Comparable { + private static final String FORMAT = "%s#%s#%s"; + private final HostAndPort hostAndPort; private final String hostPort; private final String session; - private final String hostPortSession; + private final String hostPortSessionGroup; + private final String group; + + public static TServerInstance fromString(String val) { + Objects.requireNonNull(val, "String value for TServerInstance cannot be null"); + String[] parts = val.split("#"); + if (parts.length != 3) { + // could throw IllegalArgumentException, but IllegalStateException is something + // that is already expected in a large portion of the codebase. + throw new IllegalStateException( + "Supplied TServerInstance string: " + val + " does not follow format: " + FORMAT); + } + return new TServerInstance(parts[0], parts[1], parts[2]); + } - public TServerInstance(HostAndPort address, String session) { + public TServerInstance(HostAndPort address, String session, String group) { this.hostAndPort = address; this.session = session; this.hostPort = hostAndPort.toString(); - this.hostPortSession = hostPort + "[" + session + "]"; + this.group = group; + this.hostPortSessionGroup = String.format(FORMAT, hostPort, session, group); } - public TServerInstance(String formattedString) { - int pos = formattedString.indexOf("["); - if (pos < 0 || !formattedString.endsWith("]")) { - throw new IllegalArgumentException(formattedString); - } - this.hostAndPort = HostAndPort.fromString(formattedString.substring(0, pos)); - this.session = formattedString.substring(pos + 1, formattedString.length() - 1); - this.hostPort = hostAndPort.toString(); - this.hostPortSession = hostPort + "[" + session + "]"; + public TServerInstance(String address, String session, String group) { + this(AddressUtil.parseAddress(address, false), session, group); } - public TServerInstance(HostAndPort address, long session) { - this(address, Long.toHexString(session)); + public TServerInstance(HostAndPort address, long session, String group) { + this(address, Long.toHexString(session), group); } - public TServerInstance(String address, long session) { - this(AddressUtil.parseAddress(address, false), Long.toHexString(session)); + public TServerInstance(String address, long session, String group) { + this(AddressUtil.parseAddress(address, false), Long.toHexString(session), group); } - public TServerInstance(Value address, Text session) { - this(AddressUtil.parseAddress(new String(address.get(), UTF_8), false), session.toString()); + public TServerInstance(Value address, Text session, String group) { + this(AddressUtil.parseAddress(new String(address.get(), UTF_8), false), session.toString(), + group); } @Override @@ -74,16 +86,24 @@ public int compareTo(TServerInstance other) { if (this == other) { return 0; } - return this.getHostPortSession().compareTo(other.getHostPortSession()); + return this.getHostPortSessionGroup().compareTo(other.getHostPortSessionGroup()); } @Override public int hashCode() { - return getHostPortSession().hashCode(); + final int prime = 31; + int result = 1; + result = prime * result + ((group == null) ? 0 : group.hashCode()); + result = + prime * result + ((hostPortSessionGroup == null) ? 0 : hostPortSessionGroup.hashCode()); + return result; } @Override public boolean equals(Object obj) { + if (obj == null) { + return false; + } if (obj instanceof TServerInstance) { return compareTo((TServerInstance) obj) == 0; } @@ -92,11 +112,11 @@ public boolean equals(Object obj) { @Override public String toString() { - return hostPortSession; + return hostPortSessionGroup; } - public String getHostPortSession() { - return hostPortSession; + public String getHostPortSessionGroup() { + return hostPortSessionGroup; } public String getHost() { @@ -114,4 +134,8 @@ public HostAndPort getHostAndPort() { public String getSession() { return session; } + + public String getGroup() { + return group; + } } diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java index f4757919f39..0983e166217 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java @@ -54,6 +54,7 @@ import org.apache.accumulo.core.fate.zookeeper.ZooCache; import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.lock.ServiceLockData; +import org.apache.accumulo.core.lock.ServiceLockData.ThriftService; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.StoredTabletFile; @@ -150,10 +151,6 @@ public static class Location { private final TServerInstance tServerInstance; private final LocationType lt; - private Location(final String server, final String session, final LocationType lt) { - this(new TServerInstance(HostAndPort.fromString(server), session), lt); - } - private Location(final TServerInstance tServerInstance, final LocationType lt) { this.tServerInstance = Objects.requireNonNull(tServerInstance, "tServerInstance must not be null"); @@ -168,26 +165,6 @@ public TServerInstance getServerInstance() { return tServerInstance; } - public String getHostPortSession() { - return tServerInstance.getHostPortSession(); - } - - public String getHost() { - return tServerInstance.getHost(); - } - - public String getHostPort() { - return tServerInstance.getHostPort(); - } - - public HostAndPort getHostAndPort() { - return tServerInstance.getHostAndPort(); - } - - public String getSession() { - return tServerInstance.getSession(); - } - @Override public boolean equals(Object o) { if (this == o) { @@ -219,26 +196,14 @@ public static Location last(TServerInstance instance) { return new Location(instance, LocationType.LAST); } - public static Location last(final String server, final String session) { - return last(new TServerInstance(HostAndPort.fromString(server), session)); - } - public static Location current(TServerInstance instance) { return new Location(instance, LocationType.CURRENT); } - public static Location current(final String server, final String session) { - return current(new TServerInstance(HostAndPort.fromString(server), session)); - } - public static Location future(TServerInstance instance) { return new Location(instance, LocationType.FUTURE); } - public static Location future(final String server, final String session) { - return future(new TServerInstance(HostAndPort.fromString(server), session)); - } - } public TableId getTableId() { @@ -527,13 +492,13 @@ public static > TabletMetadata convertRow(Iterator BulkFileColumnFamily.getBulkLoadTid(val)); break; case CurrentLocationColumnFamily.STR_NAME: - te.setLocationOnce(val, qual, LocationType.CURRENT, suppressLocationError); + te.setLocationOnce(val, LocationType.CURRENT, suppressLocationError); break; case FutureLocationColumnFamily.STR_NAME: - te.setLocationOnce(val, qual, LocationType.FUTURE, suppressLocationError); + te.setLocationOnce(val, LocationType.FUTURE, suppressLocationError); break; case LastLocationColumnFamily.STR_NAME: - te.last = Location.last(val, qual); + te.last = Location.last(TServerInstance.fromString(val)); break; case SuspendLocationColumn.STR_NAME: te.suspend = SuspendingTServer.fromValue(kv.getValue()); @@ -586,15 +551,15 @@ public static > TabletMetadata convertRow(Iterator return te; } - private void setLocationOnce(String val, String qual, LocationType lt, boolean suppressError) { + private void setLocationOnce(String val, LocationType lt, boolean suppressError) { if (location != null) { if (!suppressError) { throw new IllegalStateException("Attempted to set second location for tableId: " + tableId - + " endrow: " + endRow + " -- " + location + " " + qual + " " + val); + + " endrow: " + endRow + " -- " + location + " " + val); } futureAndCurrentLocationSet = true; } - location = new Location(val, qual, lt); + location = new Location(TServerInstance.fromString(val), lt); } @VisibleForTesting @@ -638,9 +603,10 @@ private static Optional checkServer(ClientContext context, Stri if (sld.isPresent()) { log.trace("Checking server at ZK path = " + lockPath); - HostAndPort client = sld.orElseThrow().getAddress(ServiceLockData.ThriftService.TSERV); + HostAndPort client = sld.orElseThrow().getAddress(ThriftService.TSERV); + String group = sld.orElseThrow().getGroup(ThriftService.TSERV); if (client != null) { - server = Optional.of(new TServerInstance(client, stat.getEphemeralOwner())); + server = Optional.of(new TServerInstance(client, stat.getEphemeralOwner(), group)); } } return server; diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java index ce3a0c5ea4f..80fe3b760d8 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java @@ -50,18 +50,19 @@ public class MetricsUtil { private static List commonTags; public static void initializeMetrics(final AccumuloConfiguration conf, final String appName, - final HostAndPort address) throws ClassNotFoundException, InstantiationException, - IllegalAccessException, IllegalArgumentException, InvocationTargetException, - NoSuchMethodException, SecurityException { + final HostAndPort address, String groupName) throws ClassNotFoundException, + InstantiationException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException { initializeMetrics(conf.getBoolean(Property.GENERAL_MICROMETER_ENABLED), conf.getBoolean(Property.GENERAL_MICROMETER_JVM_METRICS_ENABLED), - conf.get(Property.GENERAL_MICROMETER_FACTORY), appName, address); + conf.get(Property.GENERAL_MICROMETER_FACTORY), appName, address, groupName); } private static void initializeMetrics(boolean enabled, boolean jvmMetricsEnabled, - String factoryClass, String appName, HostAndPort address) throws ClassNotFoundException, - InstantiationException, IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException, SecurityException { + String factoryClass, String appName, HostAndPort address, String groupName) + throws ClassNotFoundException, InstantiationException, IllegalAccessException, + IllegalArgumentException, InvocationTargetException, NoSuchMethodException, + SecurityException { LOG.info("initializing metrics, enabled:{}, class:{}", enabled, factoryClass); @@ -84,6 +85,9 @@ private static void initializeMetrics(boolean enabled, boolean jvmMetricsEnabled tags.add(Tag.of("port", Integer.toString(address.getPort()))); } } + if (groupName != null) { + tags.add(Tag.of("group", groupName)); + } commonTags = Collections.unmodifiableList(tags); diff --git a/core/src/main/java/org/apache/accumulo/core/spi/balancer/GroupBalancer.java b/core/src/main/java/org/apache/accumulo/core/spi/balancer/GroupBalancer.java index 3527ba6f4c1..0ef8c3816cd 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/balancer/GroupBalancer.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/balancer/GroupBalancer.java @@ -131,11 +131,12 @@ public void getAssignments(AssignmentParameters params) { // Maintain locality String fakeSessionID = " "; TabletServerId simple = - new TabletServerIdImpl(last.getHost(), last.getPort(), fakeSessionID); + new TabletServerIdImpl(last.getHost(), last.getPort(), fakeSessionID, last.getGroup()); Iterator find = params.currentStatus().tailMap(simple).keySet().iterator(); if (find.hasNext()) { TabletServerId tserver = find.next(); - if (tserver.getHost().equals(last.getHost())) { + if (tserver.getHost().equals(last.getHost()) + && tserver.getGroup().equals(last.getGroup())) { params.addAssignment(entry.getKey(), tserver); continue; } diff --git a/core/src/main/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancer.java b/core/src/main/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancer.java index 50fcd8417fb..c81861b94b8 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancer.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancer.java @@ -95,11 +95,13 @@ public TabletServerId getAssignment(SortedMap loca if (last != null) { // Maintain locality String fakeSessionID = " "; - TabletServerId simple = new TabletServerIdImpl(last.getHost(), last.getPort(), fakeSessionID); + TabletServerId simple = + new TabletServerIdImpl(last.getHost(), last.getPort(), fakeSessionID, last.getGroup()); Iterator find = locations.tailMap(simple).keySet().iterator(); if (find.hasNext()) { TabletServerId current = find.next(); - if (current.getHost().equals(last.getHost())) { + if (current.getHost().equals(last.getHost()) + && current.getGroup().equals(last.getGroup())) { return current; } } diff --git a/core/src/main/java/org/apache/accumulo/core/spi/balancer/data/TabletServerId.java b/core/src/main/java/org/apache/accumulo/core/spi/balancer/data/TabletServerId.java index 6adcc803f17..8a96eb34e11 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/balancer/data/TabletServerId.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/balancer/data/TabletServerId.java @@ -27,4 +27,6 @@ public interface TabletServerId extends Comparable { int getPort(); String getSession(); + + String getGroup(); } diff --git a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java index 2bbcdbbabda..72b37f8f8ee 100644 --- a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java +++ b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java @@ -193,12 +193,14 @@ private TSummaryRequest getRequest() { // tablets // w/o a // location - .map(tm -> tm.getLocation().getHostPort()) // convert to host:port strings + .map(tm -> tm.getLocation().getServerInstance().getHostPort()) // convert to host:port + // strings .min(String::compareTo) // find minimum host:port .orElse(entry.getValue().stream().filter(tm -> tm.getLast() != null) // if no locations, // then look at last // locations - .map(tm -> tm.getLast().getHostPort()) // convert to host:port strings + .map(tm -> tm.getLast().getServerInstance().getHostPort()) // convert to host:port + // strings .min(String::compareTo).orElse(null)); // find minimum last location or return null if (location == null) { diff --git a/core/src/test/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImplTest.java b/core/src/test/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImplTest.java index 70860271bc6..ae054374ac5 100644 --- a/core/src/test/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImplTest.java +++ b/core/src/test/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImplTest.java @@ -23,7 +23,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; @@ -55,9 +54,11 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.MetadataCachedTabletObtainer; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; @@ -110,12 +111,12 @@ public RangeLocation(String location, KeyExtent extent1, List range1, Key } static RangeLocation createRangeLocation(String location, KeyExtent extent, List ranges) { - return new RangeLocation(location, extent, ranges); + return new RangeLocation(fixServerAddress(location), extent, ranges); } static RangeLocation createRangeLocation(String location, KeyExtent extent1, List range1, KeyExtent extent2, List range2) { - return new RangeLocation(location, extent1, range1, extent2, range2); + return new RangeLocation(fixServerAddress(location), extent1, range1, extent2, range2); } static Map>> @@ -138,7 +139,7 @@ static TreeMap createMetaCacheKE(Object... data) { for (int i = 0; i < data.length; i += 2) { KeyExtent ke = (KeyExtent) data[i]; String loc = (String) data[i + 1]; - mcke.put(ke, new CachedTablet(ke, loc, "1", TabletHostingGoal.ONDEMAND, false)); + mcke.put(ke, new CachedTablet(ke, createTSI(loc, "1"), TabletHostingGoal.ONDEMAND, false)); } return mcke; @@ -343,7 +344,7 @@ public ServerExtent(String location, String row, KeyExtent extent) { } static ServerExtent createServerExtent(String row, String location, KeyExtent extent) { - return new ServerExtent(location, row, extent); + return new ServerExtent(fixServerAddress(location), row, extent); } static Map>> createServerExtentMap(ServerExtent... locations) { @@ -606,8 +607,9 @@ static class TestRootClientTabletCache extends RootClientTabletCache { @Override protected CachedTablet getRootTabletLocation(ClientContext context) { - return new CachedTablet(RootTable.EXTENT, context.getRootTabletLocation(), "1", - TabletHostingGoal.ALWAYS, false); + TServerInstance rootTabletLocation = createTSI(context.getRootTabletLocation(), "1"); + return new CachedTablet(RootTable.EXTENT, rootTabletLocation, TabletHostingGoal.ALWAYS, + false); } @Override @@ -617,7 +619,7 @@ public void invalidateCache(ClientContext context, String server) {} static void createEmptyTablet(TServers tservers, String server, KeyExtent tablet) { Map> tablets = - tservers.tservers.computeIfAbsent(server, k -> new HashMap<>()); + tservers.tservers.computeIfAbsent(fixServerAddress(server), k -> new HashMap<>()); SortedMap tabletData = tablets.computeIfAbsent(tablet, k -> new TreeMap<>()); if (!tabletData.isEmpty()) { throw new IllegalStateException("Asked for empty tablet, but non empty tablet exists"); @@ -642,10 +644,19 @@ static void clearLocation(TServers tservers, String server, KeyExtent tablet, Ke } + static String fixServerAddress(String server) { + return (server.contains(":") ? server : server + ":9997"); + } + + static TServerInstance createTSI(String server, String session) { + return new TServerInstance(fixServerAddress(server), session, + ServiceDescriptor.DEFAULT_GROUP_NAME); + } + static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke, String location, String instance) { Map> tablets = - tservers.tservers.computeIfAbsent(server, k -> new HashMap<>()); + tservers.tservers.computeIfAbsent(fixServerAddress(server), k -> new HashMap<>()); SortedMap tabletData = tablets.computeIfAbsent(tablet, k -> new TreeMap<>()); Text mr = ke.toMetaRow(); @@ -655,8 +666,8 @@ static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyE if (instance == null) { instance = ""; } - Key lk = new Key(mr, CurrentLocationColumnFamily.NAME, new Text(instance)); - tabletData.put(lk, new Value(location)); + Key lk = new Key(mr, CurrentLocationColumnFamily.NAME); + tabletData.put(lk, new Value(createTSI(location, instance).toString())); } Key hk = new Key(mr, HostingColumnFamily.GOAL_COLUMN.getColumnFamily(), @@ -671,11 +682,11 @@ static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyE static void deleteLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke, String instance) { Map> tablets = - tservers.tservers.computeIfAbsent(server, k -> new HashMap<>()); + tservers.tservers.computeIfAbsent(fixServerAddress(server), k -> new HashMap<>()); SortedMap tabletData = tablets.computeIfAbsent(tablet, k -> new TreeMap<>()); Text mr = ke.toMetaRow(); - Key lk = new Key(mr, CurrentLocationColumnFamily.NAME, new Text(instance)); + Key lk = new Key(mr, CurrentLocationColumnFamily.NAME); tabletData.remove(lk); } @@ -685,12 +696,13 @@ static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyE } static void deleteServer(TServers tservers, String server) { - tservers.tservers.remove(server); + tservers.tservers.remove(fixServerAddress(server)); } private void locateTabletTest(ClientTabletCacheImpl cache, String row, boolean skipRow, KeyExtent expected, String server, LocationNeed locationNeeded) throws Exception { + CachedTablet tl = cache.findTablet(context, new Text(row), skipRow, locationNeeded); if (expected == null) { @@ -704,7 +716,7 @@ private void locateTabletTest(ClientTabletCacheImpl cache, String row, boolean s assertTrue(tl.getTserverLocation().isEmpty()); assertTrue(tl.getTserverSession().isEmpty()); } else { - assertEquals(server, tl.getTserverLocation().orElseThrow()); + assertEquals(fixServerAddress(server), tl.getTserverLocation().orElseThrow()); } assertEquals(expected, tl.getExtent()); } @@ -774,7 +786,7 @@ public void test1() throws Exception { // simulate a server failure setLocation(tservers, "tserver2", METADATA_TABLE_EXTENT, tab1e21, "tserver9"); - tab1TabletCache.invalidateCache(context, "tserver8"); + tab1TabletCache.invalidateCache(context, "tserver8:9997"); locateTabletTest(tab1TabletCache, "r1", tab1e22, "tserver6"); locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver9"); locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver4"); @@ -782,9 +794,9 @@ public void test1() throws Exception { // simulate all servers failing deleteServer(tservers, "tserver1"); deleteServer(tservers, "tserver2"); - tab1TabletCache.invalidateCache(context, "tserver4"); - tab1TabletCache.invalidateCache(context, "tserver6"); - tab1TabletCache.invalidateCache(context, "tserver9"); + tab1TabletCache.invalidateCache(context, "tserver4:9997"); + tab1TabletCache.invalidateCache(context, "tserver6:9997"); + tab1TabletCache.invalidateCache(context, "tserver9:9997"); locateTabletTest(tab1TabletCache, "r1", null, null); locateTabletTest(tab1TabletCache, "h", null, null); @@ -838,7 +850,7 @@ public void test1() throws Exception { // simulate metadata and regular server down and the reassigned deleteServer(tservers, "tserver5"); - tab1TabletCache.invalidateCache(context, "tserver7"); + tab1TabletCache.invalidateCache(context, "tserver7:9997"); locateTabletTest(tab1TabletCache, "a", null, null); locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver8"); locateTabletTest(tab1TabletCache, "r", tab1e22, "tserver9"); @@ -850,7 +862,7 @@ public void test1() throws Exception { locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver7"); locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver8"); locateTabletTest(tab1TabletCache, "r", tab1e22, "tserver9"); - tab1TabletCache.invalidateCache(context, "tserver7"); + tab1TabletCache.invalidateCache(context, "tserver7:9997"); setLocation(tservers, "tserver10", mte1, tab1e1, "tserver2"); locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver2"); locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver8"); @@ -1230,13 +1242,14 @@ public void testBinRangesNonContiguousExtentsAndMultipleRanges() throws Exceptio @Test public void testIsContiguous() { - CachedTablet e1 = new CachedTablet(createNewKeyExtent("foo", "1", null), "l1", "1", + TServerInstance l1 = createTSI("l1", "1"); + CachedTablet e1 = new CachedTablet(createNewKeyExtent("foo", "1", null), l1, TabletHostingGoal.ONDEMAND, false); - CachedTablet e2 = new CachedTablet(createNewKeyExtent("foo", "2", "1"), "l1", "1", + CachedTablet e2 = new CachedTablet(createNewKeyExtent("foo", "2", "1"), l1, TabletHostingGoal.ONDEMAND, false); - CachedTablet e3 = new CachedTablet(createNewKeyExtent("foo", "3", "2"), "l1", "1", + CachedTablet e3 = new CachedTablet(createNewKeyExtent("foo", "3", "2"), l1, TabletHostingGoal.ONDEMAND, false); - CachedTablet e4 = new CachedTablet(createNewKeyExtent("foo", null, "3"), "l1", "1", + CachedTablet e4 = new CachedTablet(createNewKeyExtent("foo", null, "3"), l1, TabletHostingGoal.ONDEMAND, false); assertTrue(ClientTabletCacheImpl.isContiguous(List.of(e1, e2, e3, e4))); @@ -1250,7 +1263,7 @@ public void testIsContiguous() { assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e1, e2, e4))); assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e1, e3, e4))); - CachedTablet e5 = new CachedTablet(createNewKeyExtent("foo", null, null), "l1", "1", + CachedTablet e5 = new CachedTablet(createNewKeyExtent("foo", null, null), l1, TabletHostingGoal.ONDEMAND, false); assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e1, e2, e3, e4, e5))); assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e5, e1, e2, e3, e4))); @@ -1258,12 +1271,12 @@ public void testIsContiguous() { assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e5, e2, e3, e4))); assertTrue(ClientTabletCacheImpl.isContiguous(List.of(e5))); - CachedTablet e6 = new CachedTablet(createNewKeyExtent("foo", null, "1"), "l1", "1", + CachedTablet e6 = new CachedTablet(createNewKeyExtent("foo", null, "1"), l1, TabletHostingGoal.ONDEMAND, false); assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e1, e2, e3, e6))); - CachedTablet e7 = new CachedTablet(createNewKeyExtent("foo", "33", "11"), "l1", "1", + CachedTablet e7 = new CachedTablet(createNewKeyExtent("foo", "33", "11"), l1, TabletHostingGoal.ONDEMAND, false); assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e1, e2, e7, e4))); @@ -1564,25 +1577,6 @@ public void testBug3() throws Exception { } - @Test - public void testAccumulo1248() { - TServers tservers = new TServers(); - ClientTabletCacheImpl metaCache = createLocators(tservers, "tserver1", "tserver2", "foo"); - - KeyExtent ke1 = createNewKeyExtent("foo", null, null); - - // set two locations for a tablet, this is not supposed to happen. The metadata cache should - // throw an exception if it sees this rather than caching one of - // the locations. - setLocation(tservers, "tserver2", METADATA_TABLE_EXTENT, ke1, "L1", "I1"); - setLocation(tservers, "tserver2", METADATA_TABLE_EXTENT, ke1, "L2", "I2"); - - var e = assertThrows(IllegalStateException.class, - () -> metaCache.findTablet(context, new Text("a"), false, LocationNeed.REQUIRED)); - assertTrue(e.getMessage().startsWith("Tablet has multiple locations : ")); - - } - @Test public void testLostLock() throws Exception { @@ -1603,7 +1597,7 @@ public void invalidateCache(String server) {} KeyExtent ke1 = createNewKeyExtent("foo", null, null); setLocation(tservers, "tserver2", METADATA_TABLE_EXTENT, ke1, "L1", "5"); - activeLocks.add("L1:5"); + activeLocks.add("L1:9997:5"); locateTabletTest(metaCache, "a", ke1, "L1"); locateTabletTest(metaCache, "a", ke1, "L1"); @@ -1617,7 +1611,7 @@ public void invalidateCache(String server) {} clearLocation(tservers, "tserver2", METADATA_TABLE_EXTENT, ke1, "5"); setLocation(tservers, "tserver2", METADATA_TABLE_EXTENT, ke1, "L2", "6"); - activeLocks.add("L2:6"); + activeLocks.add("L2:9997:6"); locateTabletTest(metaCache, "a", ke1, "L2"); locateTabletTest(metaCache, "a", ke1, "L2"); @@ -1635,7 +1629,7 @@ public void invalidateCache(String server) {} locateTabletTest(metaCache, "a", null, null); locateTabletTest(metaCache, "a", null, null); - activeLocks.add("L3:7"); + activeLocks.add("L3:9997:7"); locateTabletTest(metaCache, "a", ke1, "L3"); locateTabletTest(metaCache, "a", ke1, "L3"); @@ -1665,12 +1659,12 @@ public void invalidateCache(String server) {} runTest(metaCache, ml, emb, "a", "w"); - activeLocks.add("L1:8"); + activeLocks.add("L1:9997:8"); emb = createServerExtentMap(createServerExtent("a", "L1", ke11)); runTest(metaCache, ml, emb, "w"); - activeLocks.add("L2:9"); + activeLocks.add("L2:9997:9"); emb = createServerExtentMap(createServerExtent("a", "L1", ke11), createServerExtent("w", "L2", ke12)); @@ -1687,7 +1681,7 @@ public void invalidateCache(String server) {} runTest(ranges, metaCache, expected); - activeLocks.remove("L2:9"); + activeLocks.remove("L2:9997:9"); expected = createExpectedBinnings(createRangeLocation("L1", ke11, createNewRangeList(new Range("a")))); @@ -1708,14 +1702,14 @@ public void invalidateCache(String server) {} runTest(ranges, metaCache, expected, createNewRangeList(new Range("a"), createNewRange("b", "o"), createNewRange("r", "z"))); - activeLocks.add("L3:10"); + activeLocks.add("L3:9997:10"); expected = createExpectedBinnings(createRangeLocation("L3", ke11, createNewRangeList(new Range("a")))); runTest(ranges, metaCache, expected, createNewRangeList(createNewRange("b", "o"), createNewRange("r", "z"))); - activeLocks.add("L4:11"); + activeLocks.add("L4:9997:11"); expected = createExpectedBinnings( createRangeLocation("L3", ke11, @@ -1759,7 +1753,7 @@ public void testCachingUnhosted() throws Exception { assertEquals(List.of(), failures); var tl1 = new CachedTablet(ke1, TabletHostingGoal.ONDEMAND, false); var tl2 = new CachedTablet(ke2, TabletHostingGoal.ONDEMAND, false); - var tl3 = new CachedTablet(ke3, "L2", "I2", TabletHostingGoal.ONDEMAND, false); + var tl3 = new CachedTablet(ke3, createTSI("L2", "I2"), TabletHostingGoal.ONDEMAND, false); var expected = Set.of(new Pair<>(tl1, r1), new Pair<>(tl1, r2), new Pair<>(tl2, r2), new Pair<>(tl3, r2)); assertEquals(expected, actual); @@ -1786,7 +1780,7 @@ public void testCachingUnhosted() throws Exception { failures = metaCache.findTablets(context, ranges, (tl, r) -> actual.add(new Pair<>(tl, r)), LocationNeed.NOT_REQUIRED); assertEquals(List.of(), failures); - tl1 = new CachedTablet(ke1, "L3", "I3", TabletHostingGoal.ONDEMAND, false); + tl1 = new CachedTablet(ke1, createTSI("L3", "I3"), TabletHostingGoal.ONDEMAND, false); expected = Set.of(new Pair<>(tl1, r1), new Pair<>(tl1, r2), new Pair<>(tl2, r2), new Pair<>(tl3, r2)); assertEquals(expected, actual); @@ -1795,7 +1789,7 @@ public void testCachingUnhosted() throws Exception { failures = metaCache.findTablets(context, ranges, (tl, r) -> actual.add(new Pair<>(tl, r)), LocationNeed.REQUIRED); assertEquals(List.of(), failures); - tl2 = new CachedTablet(ke2, "L4", "I4", TabletHostingGoal.ONDEMAND, false); + tl2 = new CachedTablet(ke2, createTSI("L4", "I4"), TabletHostingGoal.ONDEMAND, false); expected = Set.of(new Pair<>(tl1, r1), new Pair<>(tl1, r2), new Pair<>(tl2, r2), new Pair<>(tl3, r2)); assertEquals(expected, actual); diff --git a/core/src/test/java/org/apache/accumulo/core/lock/ServiceLockDataTest.java b/core/src/test/java/org/apache/accumulo/core/lock/ServiceLockDataTest.java index 5f89af57b17..74adb89c262 100644 --- a/core/src/test/java/org/apache/accumulo/core/lock/ServiceLockDataTest.java +++ b/core/src/test/java/org/apache/accumulo/core/lock/ServiceLockDataTest.java @@ -40,7 +40,8 @@ public class ServiceLockDataTest { @Test public void testSingleServiceConstructor() throws Exception { - ServiceLockData ss = new ServiceLockData(serverUUID, "127.0.0.1", ThriftService.TSERV); + ServiceLockData ss = new ServiceLockData(serverUUID, "127.0.0.1", ThriftService.TSERV, + ServiceDescriptor.DEFAULT_GROUP_NAME); assertEquals(serverUUID, ss.getServerUUID(ThriftService.TSERV)); assertEquals("127.0.0.1", ss.getAddressString(ThriftService.TSERV)); assertThrows(IllegalArgumentException.class, () -> ss.getAddress(ThriftService.TSERV)); @@ -54,8 +55,10 @@ public void testSingleServiceConstructor() throws Exception { @Test public void testMultipleServiceConstructor() throws Exception { ServiceDescriptors sds = new ServiceDescriptors(); - sds.addService(new ServiceDescriptor(serverUUID, ThriftService.TSERV, "127.0.0.1:9997")); - sds.addService(new ServiceDescriptor(serverUUID, ThriftService.TABLET_SCAN, "127.0.0.1:9998")); + sds.addService(new ServiceDescriptor(serverUUID, ThriftService.TSERV, "127.0.0.1:9997", + ServiceDescriptor.DEFAULT_GROUP_NAME)); + sds.addService(new ServiceDescriptor(serverUUID, ThriftService.TABLET_SCAN, "127.0.0.1:9998", + ServiceDescriptor.DEFAULT_GROUP_NAME)); ServiceLockData ss = new ServiceLockData(sds); assertEquals(serverUUID, ss.getServerUUID(ThriftService.TSERV)); assertEquals("127.0.0.1:9997", ss.getAddressString(ThriftService.TSERV)); diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java index 43e6a0e5e0d..0958025523d 100644 --- a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java +++ b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java @@ -47,6 +47,7 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.fate.FateTxId; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.SuspendingTServer; import org.apache.accumulo.core.metadata.TServerInstance; @@ -97,9 +98,11 @@ public void testAllColumns() { mutation.at().family(DataFileColumnFamily.NAME).qualifier(tf2.getMetaUpdateDelete()) .put(dfv2.encode()); - mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("s001").put("server1:8555"); + mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server1:8555", "s001", "default").toString()); - mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555"); + mutation.at().family(LastLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server2:8555", "s000", "default").toString()); LogEntry le1 = new LogEntry(extent, 55, "lf1"); mutation.at().family(le1.getColumnFamily()).qualifier(le1.getColumnQualifier()) @@ -129,12 +132,14 @@ public void testAllColumns() { assertEquals(rowMap, tm.getKeyValues()); assertEquals(Map.of(new StoredTabletFile(bf1), 56L, new StoredTabletFile(bf2), 59L), tm.getLoaded()); - assertEquals(HostAndPort.fromParts("server1", 8555), tm.getLocation().getHostAndPort()); - assertEquals("s001", tm.getLocation().getSession()); + assertEquals(HostAndPort.fromParts("server1", 8555), + tm.getLocation().getServerInstance().getHostAndPort()); + assertEquals("s001", tm.getLocation().getServerInstance().getSession()); assertEquals(LocationType.CURRENT, tm.getLocation().getType()); assertTrue(tm.hasCurrent()); - assertEquals(HostAndPort.fromParts("server2", 8555), tm.getLast().getHostAndPort()); - assertEquals("s000", tm.getLast().getSession()); + assertEquals(HostAndPort.fromParts("server2", 8555), + tm.getLast().getServerInstance().getHostAndPort()); + assertEquals("s000", tm.getLast().getServerInstance().getSession()); assertEquals(LocationType.LAST, tm.getLast().getType()); assertEquals(Set.of(le1.getValue() + " " + le1.timestamp, le2.getValue() + " " + le2.timestamp), tm.getLogs().stream().map(le -> le.getValue() + " " + le.timestamp).collect(toSet())); @@ -150,7 +155,8 @@ public void testFuture() { KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new Text("da")); Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent); - mutation.at().family(FutureLocationColumnFamily.NAME).qualifier("s001").put("server1:8555"); + mutation.at().family(FutureLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server1:8555", "s001", "default").toString()); SortedMap rowMap = toRowMap(mutation); @@ -158,8 +164,9 @@ public void testFuture() { EnumSet.allOf(ColumnType.class), false, false); assertEquals(extent, tm.getExtent()); - assertEquals(HostAndPort.fromParts("server1", 8555), tm.getLocation().getHostAndPort()); - assertEquals("s001", tm.getLocation().getSession()); + assertEquals(HostAndPort.fromParts("server1", 8555), + tm.getLocation().getServerInstance().getHostAndPort()); + assertEquals("s001", tm.getLocation().getServerInstance().getSession()); assertEquals(LocationType.FUTURE, tm.getLocation().getType()); assertFalse(tm.hasCurrent()); } @@ -169,8 +176,10 @@ public void testFutureAndCurrent() { KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new Text("da")); Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent); - mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("s001").put("server1:8555"); - mutation.at().family(FutureLocationColumnFamily.NAME).qualifier("s001").put("server1:8555"); + mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server1:8555", "s001", "default").toString()); + mutation.at().family(FutureLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server1:8555", "s001", "default").toString()); SortedMap rowMap = toRowMap(mutation); @@ -185,9 +194,12 @@ public void testFutureAndCurrent() { @Test public void testLocationStates() { KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new Text("da")); - TServerInstance ser1 = new TServerInstance(HostAndPort.fromParts("server1", 8555), "s001"); - TServerInstance ser2 = new TServerInstance(HostAndPort.fromParts("server2", 8111), "s002"); - TServerInstance deadSer = new TServerInstance(HostAndPort.fromParts("server3", 8000), "s003"); + TServerInstance ser1 = new TServerInstance(HostAndPort.fromParts("server1", 8555), "s001", + ServiceDescriptor.DEFAULT_GROUP_NAME); + TServerInstance ser2 = new TServerInstance(HostAndPort.fromParts("server2", 8111), "s002", + ServiceDescriptor.DEFAULT_GROUP_NAME); + TServerInstance deadSer = new TServerInstance(HostAndPort.fromParts("server3", 8000), "s003", + ServiceDescriptor.DEFAULT_GROUP_NAME); Set tservers = new LinkedHashSet<>(); tservers.add(ser1); tservers.add(ser2); @@ -196,8 +208,7 @@ public void testLocationStates() { // test assigned Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent); - mutation.at().family(FutureLocationColumnFamily.NAME).qualifier(ser1.getSession()) - .put(ser1.getHostPort()); + mutation.at().family(FutureLocationColumnFamily.NAME).qualifier("").put(ser1.toString()); SortedMap rowMap = toRowMap(mutation); TabletMetadata tm = @@ -206,35 +217,33 @@ public void testLocationStates() { assertEquals(TabletState.ASSIGNED, state); assertEquals(ser1, tm.getLocation().getServerInstance()); - assertEquals(ser1.getSession(), tm.getLocation().getSession()); + assertEquals(ser1.getSession(), tm.getLocation().getServerInstance().getSession()); assertEquals(LocationType.FUTURE, tm.getLocation().getType()); assertFalse(tm.hasCurrent()); // test hosted mutation = TabletColumnFamily.createPrevRowMutation(extent); - mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier(ser2.getSession()) - .put(ser2.getHostPort()); + mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("").put(ser2.toString()); rowMap = toRowMap(mutation); tm = TabletMetadata.convertRow(rowMap.entrySet().iterator(), colsToFetch, false, false); assertEquals(TabletState.HOSTED, tm.getTabletState(tservers)); assertEquals(ser2, tm.getLocation().getServerInstance()); - assertEquals(ser2.getSession(), tm.getLocation().getSession()); + assertEquals(ser2.getSession(), tm.getLocation().getServerInstance().getSession()); assertEquals(LocationType.CURRENT, tm.getLocation().getType()); assertTrue(tm.hasCurrent()); // test ASSIGNED_TO_DEAD_SERVER mutation = TabletColumnFamily.createPrevRowMutation(extent); - mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier(deadSer.getSession()) - .put(deadSer.getHostPort()); + mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("").put(deadSer.toString()); rowMap = toRowMap(mutation); tm = TabletMetadata.convertRow(rowMap.entrySet().iterator(), colsToFetch, false, false); assertEquals(TabletState.ASSIGNED_TO_DEAD_SERVER, tm.getTabletState(tservers)); assertEquals(deadSer, tm.getLocation().getServerInstance()); - assertEquals(deadSer.getSession(), tm.getLocation().getSession()); + assertEquals(deadSer.getSession(), tm.getLocation().getServerInstance().getSession()); assertEquals(LocationType.CURRENT, tm.getLocation().getType()); assertTrue(tm.hasCurrent()); diff --git a/core/src/test/java/org/apache/accumulo/core/spi/balancer/BaseHostRegexTableLoadBalancerTest.java b/core/src/test/java/org/apache/accumulo/core/spi/balancer/BaseHostRegexTableLoadBalancerTest.java index e00350069ab..ad2e9cb342c 100644 --- a/core/src/test/java/org/apache/accumulo/core/spi/balancer/BaseHostRegexTableLoadBalancerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/spi/balancer/BaseHostRegexTableLoadBalancerTest.java @@ -31,6 +31,7 @@ import org.apache.accumulo.core.data.TabletId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.TabletIdImpl; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.balancer.TServerStatusImpl; import org.apache.accumulo.core.manager.balancer.TableStatisticsImpl; import org.apache.accumulo.core.manager.balancer.TabletServerIdImpl; @@ -78,6 +79,11 @@ public TableId getId() { TestSimpleBalancer.class.getName()); } + protected static TabletServerIdImpl createTSID(String host) { + return new TabletServerIdImpl(host, 9997, Integer.toHexString(1), + ServiceDescriptor.DEFAULT_GROUP_NAME); + } + protected static final TestTable FOO = new TestTable("foo", TableId.of("1")); protected static final TestTable BAR = new TestTable("bar", TableId.of("2")); protected static final TestTable BAZ = new TestTable("baz", TableId.of("3")); @@ -126,43 +132,40 @@ public List getOnlineTabletsForTable(TabletServerId tserver, servers.put("192.168.0.14", "r03s04"); servers.put("192.168.0.15", "r03s05"); - allTabletServers.put(new TabletServerIdImpl("192.168.0.1", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.1"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.2", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.2"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.3", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.3"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.4", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.4"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.5", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.5"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.6", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.6"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.7", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.7"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.8", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.8"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.9", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.9"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.10", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.10"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.11", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.11"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.12", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.12"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.13", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.13"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.14", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.14"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.15", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.15"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - initialTableLocation.put(FOO.getTableName(), - new TabletServerIdImpl("192.168.0.1", 9997, Integer.toHexString(1))); - initialTableLocation.put(BAR.getTableName(), - new TabletServerIdImpl("192.168.0.6", 9997, Integer.toHexString(1))); - initialTableLocation.put(BAZ.getTableName(), - new TabletServerIdImpl("192.168.0.11", 9997, Integer.toHexString(1))); + initialTableLocation.put(FOO.getTableName(), createTSID("192.168.0.1")); + initialTableLocation.put(BAR.getTableName(), createTSID("192.168.0.6")); + initialTableLocation.put(BAZ.getTableName(), createTSID("192.168.0.11")); tableTablets.put(FOO.getTableName(), new ArrayList<>()); tableTablets.get(FOO.getTableName()) @@ -255,7 +258,7 @@ protected SortedMap createCurrent(int numTservers) tableMap.put(BAR.getId().canonical(), new TableStatisticsImpl(new TableInfo())); tableMap.put(BAZ.getId().canonical(), new TableStatisticsImpl(new TableInfo())); status.setTableMap(tableMap); - current.put(new TabletServerIdImpl(base + i, 9997, Integer.toHexString(1)), status); + current.put(createTSID(base + i), status); } // now put all of the tablets on one server for (Map.Entry entry : initialTableLocation.entrySet()) { diff --git a/core/src/test/java/org/apache/accumulo/core/spi/balancer/GroupBalancerTest.java b/core/src/test/java/org/apache/accumulo/core/spi/balancer/GroupBalancerTest.java index 4320d58968c..3fdc5fb57ea 100644 --- a/core/src/test/java/org/apache/accumulo/core/spi/balancer/GroupBalancerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/spi/balancer/GroupBalancerTest.java @@ -37,6 +37,7 @@ import org.apache.accumulo.core.data.TabletId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.TabletIdImpl; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.balancer.BalanceParamsImpl; import org.apache.accumulo.core.manager.balancer.TServerStatusImpl; import org.apache.accumulo.core.manager.balancer.TabletServerIdImpl; @@ -67,14 +68,17 @@ public void addTservers(String... locs) { } public void addTserver(String host, int port) { - tservers.add(new TabletServerIdImpl(host, port, Long.toHexString(6))); + tservers.add(new TabletServerIdImpl(host, port, Long.toHexString(6), + ServiceDescriptor.DEFAULT_GROUP_NAME)); } public void addTablet(String er, String host, int port) { - TabletServerId tsi = new TabletServerIdImpl(host, port, Long.toHexString(6)); + TabletServerId tsi = new TabletServerIdImpl(host, port, Long.toHexString(6), + ServiceDescriptor.DEFAULT_GROUP_NAME); tabletLocs.put( new TabletIdImpl(new KeyExtent(TableId.of("b"), er == null ? null : new Text(er), null)), - new TabletServerIdImpl(host, port, Long.toHexString(6))); + new TabletServerIdImpl(host, port, Long.toHexString(6), + ServiceDescriptor.DEFAULT_GROUP_NAME)); tservers.add(tsi); } diff --git a/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java b/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java index c70841e9821..0f4887a6696 100644 --- a/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java @@ -46,7 +46,6 @@ import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent; import org.apache.accumulo.core.manager.balancer.AssignmentParamsImpl; import org.apache.accumulo.core.manager.balancer.BalanceParamsImpl; -import org.apache.accumulo.core.manager.balancer.TabletServerIdImpl; import org.apache.accumulo.core.manager.balancer.TabletStatisticsImpl; import org.apache.accumulo.core.spi.balancer.data.TServerStatus; import org.apache.accumulo.core.spi.balancer.data.TabletMigration; @@ -163,42 +162,27 @@ public void testSplitCurrentByRegexUsingHostname() { assertTrue(groups.containsKey(FOO.getTableName())); SortedMap fooHosts = groups.get(FOO.getTableName()); assertEquals(5, fooHosts.size()); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.1", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.2", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.3", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.4", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.5", 9997, Integer.toHexString(1)))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.1"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.2"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.3"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.4"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.5"))); assertTrue(groups.containsKey(BAR.getTableName())); SortedMap barHosts = groups.get(BAR.getTableName()); assertEquals(5, barHosts.size()); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.6", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.7", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.8", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.9", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.10", 9997, Integer.toHexString(1)))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.6"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.7"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.8"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.9"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.10"))); assertTrue(groups.containsKey(DEFAULT_POOL)); SortedMap defHosts = groups.get(DEFAULT_POOL); assertEquals(5, defHosts.size()); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.11", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.12", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.13", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.14", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.15", 9997, Integer.toHexString(1)))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.11"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.12"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.13"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.14"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.15"))); } @Test @@ -217,61 +201,36 @@ public void testSplitCurrentByRegexUsingOverlappingPools() { for (String pool : new String[] {FOO.getTableName(), DEFAULT_POOL}) { SortedMap fooHosts = groups.get(pool); assertEquals(15, fooHosts.size()); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.1", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.2", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.3", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.4", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.5", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.6", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.7", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.8", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.9", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.10", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.11", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.12", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.13", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.14", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.15", 9997, Integer.toHexString(1)))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.1"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.2"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.3"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.4"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.5"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.6"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.7"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.8"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.9"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.10"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.11"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.12"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.13"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.14"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.15"))); } assertTrue(groups.containsKey(BAR.getTableName())); SortedMap barHosts = groups.get(BAR.getTableName()); assertEquals(10, barHosts.size()); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.1", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.2", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.3", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.4", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.5", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.6", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.7", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.8", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.9", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.10", 9997, Integer.toHexString(1)))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.1"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.2"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.3"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.4"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.5"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.6"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.7"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.8"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.9"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.10"))); } @Test @@ -292,42 +251,27 @@ public void testSplitCurrentByRegexUsingIP() { assertTrue(groups.containsKey(FOO.getTableName())); SortedMap fooHosts = groups.get(FOO.getTableName()); assertEquals(5, fooHosts.size()); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.1", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.2", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.3", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.4", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.5", 9997, Integer.toHexString(1)))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.1"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.2"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.3"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.4"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.5"))); assertTrue(groups.containsKey(BAR.getTableName())); SortedMap barHosts = groups.get(BAR.getTableName()); assertEquals(5, barHosts.size()); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.6", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.7", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.8", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.9", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.10", 9997, Integer.toHexString(1)))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.6"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.7"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.8"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.9"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.10"))); assertTrue(groups.containsKey(DEFAULT_POOL)); SortedMap defHosts = groups.get(DEFAULT_POOL); assertEquals(5, defHosts.size()); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.11", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.12", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.13", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.14", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.15", 9997, Integer.toHexString(1)))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.11"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.12"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.13"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.14"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.15"))); } @Test diff --git a/core/src/test/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancerTest.java b/core/src/test/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancerTest.java index 9b7c2b7a56f..c58e3ceb38b 100644 --- a/core/src/test/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancerTest.java @@ -37,6 +37,7 @@ import org.apache.accumulo.core.data.TabletId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.TabletIdImpl; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.balancer.BalanceParamsImpl; import org.apache.accumulo.core.manager.balancer.TServerStatusImpl; import org.apache.accumulo.core.manager.balancer.TabletServerIdImpl; @@ -103,9 +104,15 @@ public void setUp() { @Test public void testAssignMigrations() { - servers.put(new TabletServerIdImpl("127.0.0.1", 1234, "a"), new FakeTServer()); - servers.put(new TabletServerIdImpl("127.0.0.2", 1234, "b"), new FakeTServer()); - servers.put(new TabletServerIdImpl("127.0.0.3", 1234, "c"), new FakeTServer()); + servers.put( + new TabletServerIdImpl("127.0.0.1", 1234, "a", ServiceDescriptor.DEFAULT_GROUP_NAME), + new FakeTServer()); + servers.put( + new TabletServerIdImpl("127.0.0.2", 1234, "b", ServiceDescriptor.DEFAULT_GROUP_NAME), + new FakeTServer()); + servers.put( + new TabletServerIdImpl("127.0.0.3", 1234, "c", ServiceDescriptor.DEFAULT_GROUP_NAME), + new FakeTServer()); List metadataTable = new ArrayList<>(); String table = "t1"; metadataTable.add(makeTablet(table, null, null)); @@ -180,7 +187,8 @@ SortedMap getAssignments(Map check(Environment env, Mutation mutation) { } catch (IllegalArgumentException e) { violations = addViolation(violations, 9); } + } else if (columnFamily.equals(CurrentLocationColumnFamily.NAME) + || columnFamily.equals(LastLocationColumnFamily.NAME) + || columnFamily.equals(FutureLocationColumnFamily.NAME)) { + if (!columnQualifier.toString().isEmpty()) { + violations = addViolation(violations, 11); + } else { + try { + TServerInstance.fromString(new String(columnUpdate.getValue(), UTF_8)); + } catch (IllegalStateException e) { + violations = addViolation(violations, 12); + } + } } else if (columnFamily.equals(BulkFileColumnFamily.NAME)) { if (!columnUpdate.isDeleted() && !checkedBulk) { // splits, which also write the time reference, are allowed to write this reference even @@ -359,6 +372,10 @@ public String getViolationDescription(short violationCode) { return "Malformed operation id"; case 10: return "Malformed hosting goal"; + case 11: + return "location information in column qualifier"; + case 12: + return "location information in wrong format"; } return null; } diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java index b2cc9336de0..1e8849ac96f 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java @@ -192,7 +192,7 @@ public Map> getAllMarkers() throws WalMarkerException try { String path = root(); for (String child : zoo.getChildren(path)) { - TServerInstance inst = new TServerInstance(child); + TServerInstance inst = TServerInstance.fromString(child); List logs = result.computeIfAbsent(inst, k -> new ArrayList<>()); // This function is called by the Accumulo GC which deletes WAL markers. Therefore we do not diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java b/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java index e4af38f93f0..d2f6a07eb6c 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java @@ -39,6 +39,7 @@ import org.apache.accumulo.core.fate.zookeeper.ZooCache.ZcStat; import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.lock.ServiceLockData; +import org.apache.accumulo.core.lock.ServiceLockData.ThriftService; import org.apache.accumulo.core.manager.thrift.TabletServerStatus; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.rpc.ThriftUtil; @@ -325,8 +326,10 @@ private synchronized void checkServer(final Set updates, } } else { locklessServers.remove(zPath); - HostAndPort client = sld.orElseThrow().getAddress(ServiceLockData.ThriftService.TSERV); - TServerInstance instance = new TServerInstance(client, stat.getEphemeralOwner()); + HostAndPort client = sld.orElseThrow().getAddress(ThriftService.TSERV); + String resourceGroup = sld.orElseThrow().getGroup(ThriftService.TSERV); + TServerInstance instance = + new TServerInstance(client, stat.getEphemeralOwner(), resourceGroup); if (info == null) { updates.add(instance); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java index 0189b9294ae..ef1e3bb07e7 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java @@ -93,7 +93,7 @@ public void suspend(Collection tablets, var location = tm.getLocation(); HostAndPort server = null; if (location != null) { - server = location.getHostAndPort(); + server = location.getServerInstance().getHostAndPort(); } TabletLogger.suspended(tm.getExtent(), server, suspensionTimestamp, TimeUnit.MILLISECONDS, logsForDeadServers.size()); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java index 814c4ee2241..431a86ccdff 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java @@ -61,7 +61,6 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; import org.apache.accumulo.core.metadata.schema.TabletMetadata; -import org.apache.accumulo.core.util.AddressUtil; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.slf4j.Logger; @@ -92,7 +91,7 @@ private static void setCurrentServers(final IteratorSetting cfg, if (goodServers != null) { List servers = new ArrayList<>(); for (TServerInstance server : goodServers) { - servers.add(server.getHostPortSession()); + servers.add(server.getHostPortSessionGroup()); } cfg.addOption(SERVERS_OPTION, Joiner.on(",").join(servers)); } @@ -177,16 +176,9 @@ private static Set parseTableIDs(final String tableIDs) { private static Set parseServers(final String servers) { Set result = new HashSet<>(); if (servers != null) { - // parse "host:port[INSTANCE]" if (!servers.isEmpty()) { - for (String part : servers.split(",")) { - String[] parts = part.split("\\[", 2); - String hostport = parts[0]; - String instance = parts[1]; - if (instance != null && instance.endsWith("]")) { - instance = instance.substring(0, instance.length() - 1); - } - result.add(new TServerInstance(AddressUtil.parseAddress(hostport, false), instance)); + for (String server : servers.split(",")) { + result.add(TServerInstance.fromString(server)); } } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java index 45e6664f465..360cb6f5002 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java @@ -89,8 +89,8 @@ public Ample.ConditionalTabletMutator requireLocation(Location location) { Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate."); Preconditions.checkArgument(location.getType() == TabletMetadata.LocationType.FUTURE || location.getType() == TabletMetadata.LocationType.CURRENT); - Condition c = new Condition(getLocationFamily(location.getType()), location.getSession()) - .setValue(location.getHostPort()); + Condition c = new Condition(getLocationFamily(location.getType()), "") + .setValue(location.getServerInstance().toString()); mutation.addCondition(c); return this; } diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java index 0e3a48102ed..01f865ca517 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java @@ -170,15 +170,15 @@ protected String getLocationFamily(LocationType type) { @Override public T putLocation(Location location) { Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate."); - mutation.put(getLocationFamily(location.getType()), location.getSession(), - location.getHostPort()); + mutation.put(getLocationFamily(location.getType()), "", + location.getServerInstance().toString()); return getThis(); } @Override public T deleteLocation(Location location) { Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate."); - mutation.putDelete(getLocationFamily(location.getType()), location.getSession()); + mutation.putDelete(getLocationFamily(location.getType()), ""); return getThis(); } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java index 4320f61e2ef..af495d7f883 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java @@ -183,10 +183,11 @@ private static KeyExtent fixSplit(ServerContext context, TableId tableId, Text m } } - private static TServerInstance getTServerInstance(String address, ServiceLock zooLock) { + private static TServerInstance getTServerInstance(String address, ServiceLock zooLock, + String resourceGroupName) { while (true) { try { - return new TServerInstance(address, zooLock.getSessionId()); + return new TServerInstance(address, zooLock.getSessionId(), resourceGroupName); } catch (KeeperException | InterruptedException e) { log.error("{}", e.getMessage(), e); } @@ -197,7 +198,8 @@ private static TServerInstance getTServerInstance(String address, ServiceLock zo public static void replaceDatafiles(ServerContext context, KeyExtent extent, Set datafilesToDelete, Set scanFiles, Optional path, Long compactionId, DataFileValue size, String address, - Location lastLocation, ServiceLock zooLock, Optional ecid) { + Location lastLocation, ServiceLock zooLock, Optional ecid, + String resourceGroupName) { context.getAmple().putGcCandidates(extent.tableId(), datafilesToDelete); @@ -214,7 +216,7 @@ public static void replaceDatafiles(ServerContext context, KeyExtent extent, tablet.putCompactionId(compactionId); } - updateLastForCompactionMode(context, tablet, lastLocation, address, zooLock); + updateLastForCompactionMode(context, tablet, lastLocation, address, zooLock, resourceGroupName); if (ecid.isPresent()) { tablet.deleteExternalCompaction(ecid.orElseThrow()); @@ -231,7 +233,7 @@ public static void replaceDatafiles(ServerContext context, KeyExtent extent, public static Optional updateTabletDataFile(ServerContext context, KeyExtent extent, ReferencedTabletFile newDatafile, DataFileValue dfv, MetadataTime time, String address, ServiceLock zooLock, Set unusedWalLogs, Location lastLocation, - long flushId) { + long flushId, String resourceGroupName) { TabletMutator tablet = context.getAmple().mutateTablet(extent); // if there are no entries, the path doesn't get stored in metadata table, only the flush ID @@ -243,7 +245,8 @@ public static Optional updateTabletDataFile(ServerContext cont tablet.putTime(time); newFile = Optional.of(newDatafile.insert()); - updateLastForCompactionMode(context, tablet, lastLocation, address, zooLock); + updateLastForCompactionMode(context, tablet, lastLocation, address, zooLock, + resourceGroupName); } tablet.putFlushId(flushId); @@ -286,12 +289,13 @@ public static void updateLastForAssignmentMode(ClientContext context, * @param address The server address * @param zooLock The zookeeper lock */ - public static void updateLastForCompactionMode(ClientContext context, TabletMutator tabletMutator, - Location lastLocation, String address, ServiceLock zooLock) { + private static void updateLastForCompactionMode(ClientContext context, + TabletMutator tabletMutator, Location lastLocation, String address, ServiceLock zooLock, + String resourceGroupName) { // if the location mode is 'compaction', then preserve the current compaction location in the // last location value if ("compaction".equals(context.getConfiguration().get(Property.TSERV_LAST_LOCATION_MODE))) { - Location newLocation = Location.last(getTServerInstance(address, zooLock)); + Location newLocation = Location.last(getTServerInstance(address, zooLock, resourceGroupName)); updateLocation(tabletMutator, lastLocation, newLocation); } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java index a77fe3000ab..7086c6cb20e 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java @@ -27,6 +27,7 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily; @@ -207,14 +208,16 @@ public void testBulkFileCheck() { // active txid, mutation that looks like a load m = new Mutation(new Text("0;foo")); m.put(BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("5")); - m.put(CurrentLocationColumnFamily.NAME, new Text("789"), new Value("127.0.0.1:9997")); + m.put(CurrentLocationColumnFamily.NAME, new Text(""), + new Value(new TServerInstance("127.0.0.1:9997", "789", "default").toString())); violations = mc.check(createEnv(), m); assertNull(violations); // inactive txid, mutation that looks like a load m = new Mutation(new Text("0;foo")); m.put(BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("12345")); - m.put(CurrentLocationColumnFamily.NAME, new Text("789"), new Value("127.0.0.1:9997")); + m.put(CurrentLocationColumnFamily.NAME, new Text(""), + new Value(new TServerInstance("127.0.0.1:9997", "789", "default").toString())); violations = mc.check(createEnv(), m); assertNull(violations); diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/LiveTServerSetTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/LiveTServerSetTest.java index f67b7cb80da..b3206400a94 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/manager/LiveTServerSetTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/LiveTServerSetTest.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.Map; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.manager.LiveTServerSet.Listener; @@ -41,8 +42,9 @@ public void testSessionIds() { Map servers = new HashMap<>(); TServerConnection mockConn = EasyMock.createMock(TServerConnection.class); - TServerInfo server1 = new TServerInfo( - new TServerInstance(HostAndPort.fromParts("localhost", 1234), "5555"), mockConn); + TServerInfo server1 = + new TServerInfo(new TServerInstance(HostAndPort.fromParts("localhost", 1234), "5555", + ServiceDescriptor.DEFAULT_GROUP_NAME), mockConn); servers.put("server1", server1); LiveTServerSet tservers = new LiveTServerSet(EasyMock.createMock(ServerContext.class), diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java index 90272b3bfd8..e779b4660a8 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java @@ -44,6 +44,7 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.StoredTabletFile; @@ -266,8 +267,8 @@ public void testRootTabletStateStore() throws DistributedStoreException { ZooTabletStateStore tstore = new ZooTabletStateStore(context); KeyExtent root = RootTable.EXTENT; String sessionId = "this is my unique session data"; - TServerInstance server = - new TServerInstance(HostAndPort.fromParts("127.0.0.1", 10000), sessionId); + TServerInstance server = new TServerInstance(HostAndPort.fromParts("127.0.0.1", 10000), + sessionId, ServiceDescriptor.DEFAULT_GROUP_NAME); List assignments = Collections.singletonList(new Assignment(root, server, null)); tstore.setFutureLocations(assignments); int count = 0; diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java index e09cca2b34e..cd673d7a8e6 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java @@ -40,6 +40,7 @@ import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction; import org.apache.accumulo.core.metadata.StoredTabletFile; +import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily; @@ -90,9 +91,11 @@ private SortedMap createMetadataEntryKV(KeyExtent extent) { mutation.at().family(DataFileColumnFamily.NAME).qualifier(tf2.getMetaUpdateDelete()) .put(dfv2.encode()); - mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("s001").put("server1:8555"); + mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server1:8555", "s001", "default").toString()); - mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555"); + mutation.at().family(LastLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server2:8555", "s000", "default").toString()); LogEntry le1 = new LogEntry(extent, 55, "lf1"); mutation.at().family(le1.getColumnFamily()).qualifier(le1.getColumnQualifier()) diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/ManagerMetadataUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/ManagerMetadataUtilTest.java index 748dffaa657..49b505a7c26 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/ManagerMetadataUtilTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/ManagerMetadataUtilTest.java @@ -23,6 +23,7 @@ import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; @@ -35,9 +36,11 @@ public class ManagerMetadataUtilTest { private AccumuloConfiguration conf; private ClientContext context; private Ample.TabletMutator tabletMutator; - private final TServerInstance server1 = new TServerInstance("127.0.0.1:10000", 0); + private final TServerInstance server1 = + new TServerInstance("127.0.0.1:10000", 0, ServiceDescriptor.DEFAULT_GROUP_NAME); private final Location last1 = Location.last(server1); - private final TServerInstance server2 = new TServerInstance("127.0.0.2:10000", 1); + private final TServerInstance server2 = + new TServerInstance("127.0.0.2:10000", 1, ServiceDescriptor.DEFAULT_GROUP_NAME); private final Location last2 = Location.last(server2); @BeforeEach diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index 159de1504d1..3936f7b9981 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -59,6 +59,7 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.NamespaceId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; @@ -130,7 +131,6 @@ public class Compactor extends AbstractServer implements MetricsProducer, Compac private final UUID compactorId = UUID.randomUUID(); private final AccumuloConfiguration aconf; - private final String queueName; protected final AtomicReference currentCompactionId = new AtomicReference<>(); private final CompactionWatcher watcher; @@ -152,7 +152,6 @@ protected Compactor(ConfigOpts opts, String[] args) { protected Compactor(ConfigOpts opts, String[] args, AccumuloConfiguration conf) { super("compactor", opts, args); aconf = conf == null ? super.getConfiguration() : conf; - queueName = aconf.get(Property.COMPACTOR_QUEUE_NAME); setupSecurity(); watcher = new CompactionWatcher(aconf); var schedExecutor = @@ -166,6 +165,11 @@ public AccumuloConfiguration getConfiguration() { return aconf; } + @Override + protected String getResourceGroupNamePropertyValue(SiteConfiguration conf) { + return conf.get(Property.COMPACTOR_QUEUE_NAME); + } + @Override public void registerMetrics(MeterRegistry registry) { super.registerMetrics(registry); @@ -245,7 +249,7 @@ protected void announceExistence(HostAndPort clientAddress) ZooReaderWriter zoo = getContext().getZooReaderWriter(); String compactorQueuePath = - getContext().getZooKeeperRoot() + Constants.ZCOMPACTORS + "/" + this.queueName; + getContext().getZooKeeperRoot() + Constants.ZCOMPACTORS + "/" + getResourceGroupName(); String zPath = compactorQueuePath + "/" + hostPort; try { @@ -280,8 +284,8 @@ public void unableToMonitorLockNode(final Exception e) { for (int i = 0; i < 25; i++) { zoo.putPersistentData(zPath, new byte[0], NodeExistsPolicy.SKIP); - if (compactorLock.tryLock(lw, - new ServiceLockData(compactorId, hostPort, ThriftService.COMPACTOR, this.queueName))) { + if (compactorLock.tryLock(lw, new ServiceLockData(compactorId, hostPort, + ThriftService.COMPACTOR, getResourceGroupName()))) { LOG.debug("Obtained Compactor lock {}", compactorLock.getLockPath()); return; } @@ -435,7 +439,7 @@ protected TExternalCompactionJob getNextJob(Supplier uuid) throws RetriesE LOG.trace("Attempting to get next job, eci = {}", eci); currentCompactionId.set(eci); return coordinatorClient.getCompactionJob(TraceUtil.traceInfo(), - getContext().rpcCreds(), queueName, + getContext().rpcCreds(), getResourceGroupName(), ExternalCompactionUtil.getHostPortString(compactorAddress.getAddress()), eci.toString()); } catch (Exception e) { @@ -518,7 +522,7 @@ protected Runnable createCompactionJob(final TExternalCompactionJob job, job.getIteratorSettings().getIterators() .forEach(tis -> iters.add(SystemIteratorUtil.toIteratorSetting(tis))); - ExtCEnv cenv = new ExtCEnv(JOB_HOLDER, queueName); + ExtCEnv cenv = new ExtCEnv(JOB_HOLDER, getResourceGroupName()); FileCompactor compactor = new FileCompactor(getContext(), extent, files, outputFile, job.isPropagateDeletes(), cenv, iters, aConfig, tConfig.getCryptoService(), pausedMetrics); @@ -566,7 +570,8 @@ protected Supplier getNextId() { protected long getWaitTimeBetweenCompactionChecks() { // get the total number of compactors assigned to this queue - int numCompactors = ExternalCompactionUtil.countCompactors(queueName, getContext()); + int numCompactors = + ExternalCompactionUtil.countCompactors(getResourceGroupName(), getContext()); // Aim for around 3 compactors checking in every second long sleepTime = numCompactors * 1000L / 3; // Ensure a compactor sleeps at least around a second @@ -598,7 +603,7 @@ public void run() { try { MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, - clientAddress); + clientAddress, getResourceGroupName()); pausedMetrics = new PausedCompactionMetrics(); MetricsUtil.initializeProducers(this, pausedMetrics); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException @@ -621,7 +626,7 @@ public void run() { try { job = getNextJob(getNextId()); if (!job.isSetExternalCompactionId()) { - LOG.trace("No external compactions in queue {}", this.queueName); + LOG.trace("No external compactions in queue {}", getResourceGroupName()); UtilWaitThread.sleep(getWaitTimeBetweenCompactionChecks()); continue; } diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 1946ffee3b0..958711e1462 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@ -157,7 +157,8 @@ public void run() { } try { - MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, address); + MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, address, + getResourceGroupName()); MetricsUtil.initializeProducers(this, new GcMetrics(this)); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException @@ -355,8 +356,8 @@ public void unableToMonitorLockNode(final Exception e) { while (true) { ServiceLock lock = new ServiceLock(getContext().getZooReaderWriter().getZooKeeper(), path, zooLockUUID); - if (lock.tryLock(lockWatcher, - new ServiceLockData(zooLockUUID, addr.toString(), ThriftService.GC))) { + if (lock.tryLock(lockWatcher, new ServiceLockData(zooLockUUID, addr.toString(), + ThriftService.GC, getResourceGroupName()))) { log.debug("Got GC ZooKeeper lock"); return; } diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java index dc5802eef41..67d53e0ac44 100644 --- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java +++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java @@ -46,8 +46,10 @@ public class GarbageCollectWriteAheadLogsTest { - private final TServerInstance server1 = new TServerInstance("localhost:1234[SESSION]"); - private final TServerInstance server2 = new TServerInstance("localhost:1234[OTHERSESS]"); + private final TServerInstance server1 = + TServerInstance.fromString("localhost:1234#SESSION#default"); + private final TServerInstance server2 = + TServerInstance.fromString("localhost:1234#OTHERSESS#default"); private final UUID id = UUID.randomUUID(); private final Map> markers = Collections.singletonMap(server1, Collections.singletonList(id)); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 4796a1aebc9..ad46832df2a 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -1189,7 +1189,7 @@ public void run() { try { MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, - sa.getAddress()); + sa.getAddress(), getResourceGroupName()); ManagerMetrics mm = new ManagerMetrics(getConfiguration(), this); MetricsUtil.initializeProducers(this, mm); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException @@ -1337,7 +1337,7 @@ boolean canSuspendTablets() { ServiceDescriptors descriptors = new ServiceDescriptors(); for (ThriftService svc : new ThriftService[] {ThriftService.MANAGER, ThriftService.COORDINATOR}) { - descriptors.addService(new ServiceDescriptor(uuid, svc, address)); + descriptors.addService(new ServiceDescriptor(uuid, svc, address, getResourceGroupName())); } sld = new ServiceLockData(descriptors); @@ -1571,7 +1571,8 @@ private ServiceLockData getManagerLock(final ServiceLockPath zManagerLoc) ServiceDescriptors descriptors = new ServiceDescriptors(); for (ThriftService svc : new ThriftService[] {ThriftService.MANAGER, ThriftService.COORDINATOR}) { - descriptors.addService(new ServiceDescriptor(zooLockUUID, svc, managerClientAddress)); + descriptors.addService( + new ServiceDescriptor(zooLockUUID, svc, managerClientAddress, getResourceGroupName())); } ServiceLockData sld = new ServiceLockData(descriptors); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java index 44fb1ba38f5..2fc9a006960 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java @@ -469,8 +469,9 @@ private void hostSuspendedTablet(TabletLists tLists, TabletMetadata tm, Location < tableConf.getTimeInMillis(Property.TABLE_SUSPEND_DURATION)) { // Tablet is suspended. See if its tablet server is back. TServerInstance returnInstance = null; - Iterator find = tLists.destinations - .tailMap(new TServerInstance(tm.getSuspend().server, " ")).keySet().iterator(); + Iterator find = + tLists.destinations.tailMap(new TServerInstance(tm.getSuspend().server, " ", + tableConf.get(Property.TABLE_ASSIGNMENT_GROUP))).keySet().iterator(); if (find.hasNext()) { TServerInstance found = find.next(); if (found.getHostAndPort().equals(tm.getSuspend().server)) { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionFinalizer.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionFinalizer.java index 7279bc08a9a..29632153321 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionFinalizer.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionFinalizer.java @@ -122,7 +122,8 @@ private void notifyTserver(Location loc, ExternalCompactionFinalState ecfs) { Client client = null; try { - client = ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER, loc.getHostAndPort(), context); + client = ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER, + loc.getServerInstance().getHostAndPort(), context); if (ecfs.getFinalState() == FinalState.FINISHED) { LOG.debug("Notifying tserver {} that compaction {} has finished.", loc, ecfs); client.compactionJobFinished(TraceUtil.traceInfo(), context.rpcCreds(), @@ -136,7 +137,7 @@ private void notifyTserver(Location loc, ExternalCompactionFinalState ecfs) { throw new IllegalArgumentException(ecfs.getFinalState().name()); } } catch (TException e) { - LOG.warn("Failed to notify tserver {}", loc.getHostAndPort(), e); + LOG.warn("Failed to notify tserver {}", loc.getServerInstance().getHostAndPort(), e); } finally { ThriftUtil.returnClient(client, context); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/RefreshTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/RefreshTablets.java index edfa3664f81..007fd243942 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/RefreshTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/RefreshTablets.java @@ -176,8 +176,8 @@ private List sendSyncRefreshRequest(Manager manager, long tid, Locati log.trace("{} sending refresh request to {} for {} extents", FateTxId.formatTid(tid), location, extents.size()); var timeInMillis = manager.getConfiguration().getTimeInMillis(Property.MANAGER_BULK_TIMEOUT); - client = ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER, location.getHostAndPort(), - manager.getContext(), timeInMillis); + client = ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER, + location.getServerInstance().getHostAndPort(), manager.getContext(), timeInMillis); var unrefreshed = client.refreshTablets(TraceUtil.traceInfo(), manager.getContext().rpcCreds(), diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java index f1bec9dfc11..7803e2ec7bc 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java @@ -42,17 +42,19 @@ public class ShutdownTServer extends ManagerRepo { private static final Logger log = LoggerFactory.getLogger(ShutdownTServer.class); private final HostAndPort hostAndPort; private final String serverSession; + private final String resourceGroup; private final boolean force; public ShutdownTServer(TServerInstance server, boolean force) { this.hostAndPort = server.getHostAndPort(); this.serverSession = server.getSession(); + this.resourceGroup = server.getGroup(); this.force = force; } @Override public long isReady(long tid, Manager manager) { - TServerInstance server = new TServerInstance(hostAndPort, serverSession); + TServerInstance server = new TServerInstance(hostAndPort, serverSession, resourceGroup); // suppress assignment of tablets to the server if (force) { return 0; diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java index a02021b4dd6..d69b9459dbd 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java @@ -240,7 +240,7 @@ public void testCoordinatorColdStart() throws Exception { assertNotNull(t); assertEquals(1, t.size()); TServerInstance queuedTsi = t.iterator().next(); - assertEquals(tsi.getHostPortSession(), queuedTsi.getHostPortSession()); + assertEquals(tsi.getHostPortSessionGroup(), queuedTsi.getHostPortSessionGroup()); assertEquals(1, coordinator.getIndex().size()); assertTrue(coordinator.getIndex().containsKey(queuedTsi)); Set i = coordinator.getIndex().get(queuedTsi); @@ -298,7 +298,7 @@ public void testCoordinatorRestartNoRunningCompactions() throws Exception { assertNotNull(t); assertEquals(1, t.size()); TServerInstance queuedTsi = t.iterator().next(); - assertEquals(instance.getHostPortSession(), queuedTsi.getHostPortSession()); + assertEquals(instance.getHostPortSessionGroup(), queuedTsi.getHostPortSessionGroup()); assertEquals(1, coordinator.getIndex().size()); assertTrue(coordinator.getIndex().containsKey(queuedTsi)); Set i = coordinator.getIndex().get(queuedTsi); @@ -364,7 +364,7 @@ public void testCoordinatorRestartOneRunningCompaction() throws Exception { assertNotNull(t); assertEquals(1, t.size()); TServerInstance queuedTsi = t.iterator().next(); - assertEquals(instance.getHostPortSession(), queuedTsi.getHostPortSession()); + assertEquals(instance.getHostPortSessionGroup(), queuedTsi.getHostPortSessionGroup()); assertEquals(1, coordinator.getIndex().size()); assertTrue(coordinator.getIndex().containsKey(queuedTsi)); Set i = coordinator.getIndex().get(queuedTsi); @@ -435,7 +435,7 @@ public void testGetCompactionJob() throws Exception { assertNotNull(t); assertEquals(1, t.size()); TServerInstance queuedTsi = t.iterator().next(); - assertEquals(tsi.getHostPortSession(), queuedTsi.getHostPortSession()); + assertEquals(tsi.getHostPortSessionGroup(), queuedTsi.getHostPortSessionGroup()); assertEquals(1, coordinator.getIndex().size()); assertTrue(coordinator.getIndex().containsKey(queuedTsi)); Set i = coordinator.getIndex().get(queuedTsi); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/QueueSummariesTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/QueueSummariesTest.java index 7a213149cd4..3278e3840a4 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/QueueSummariesTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/QueueSummariesTest.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Set; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.tabletserver.thrift.TCompactionQueueSummary; import org.apache.accumulo.manager.compaction.QueueSummaries.PrioTserver; @@ -33,7 +34,7 @@ public class QueueSummariesTest { private TServerInstance ntsi(String tserver) { - return new TServerInstance(tserver + ":9997", 0); + return new TServerInstance(tserver + ":9997", 0, ServiceDescriptor.DEFAULT_GROUP_NAME); } private PrioTserver npt(String tserver, short prio) { diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/ShutdownTServerTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/ShutdownTServerTest.java index 9aff9a89faa..62250a7f47c 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/ShutdownTServerTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/ShutdownTServerTest.java @@ -26,6 +26,7 @@ import java.util.HashMap; import org.apache.accumulo.core.fate.Repo; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.thrift.TableInfo; import org.apache.accumulo.core.manager.thrift.TabletServerStatus; import org.apache.accumulo.core.metadata.TServerInstance; @@ -42,7 +43,8 @@ public class ShutdownTServerTest { @Test public void testSingleShutdown() throws Exception { HostAndPort hap = HostAndPort.fromParts("localhost", 1234); - final TServerInstance tserver = new TServerInstance(hap, "fake"); + final TServerInstance tserver = + new TServerInstance(hap, "fake", ServiceDescriptor.DEFAULT_GROUP_NAME); final boolean force = false; final ShutdownTServer op = new ShutdownTServer(tserver, force); diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java index 7e1242def63..30f3480a5a8 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java @@ -825,8 +825,8 @@ private void getMonitorLock() throws KeeperException, InterruptedException { while (true) { MoniterLockWatcher monitorLockWatcher = new MoniterLockWatcher(); monitorLock = new ServiceLock(zoo.getZooKeeper(), monitorLockPath, zooLockUUID); - monitorLock.lock(monitorLockWatcher, - new ServiceLockData(zooLockUUID, getHostname(), ThriftService.NONE)); + monitorLock.lock(monitorLockWatcher, new ServiceLockData(zooLockUUID, getHostname(), + ThriftService.NONE, getResourceGroupName())); monitorLockWatcher.waitForChange(); diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java index 39a6b51cf7b..ff8d22a8d60 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java @@ -156,7 +156,7 @@ public TabletServers getParticipatingTabletServers(@PathParam("tableId") @NotNul final TabletMetadata tm = scanner.next().getTabletMetadata(); if (tm.hasCurrent()) { try { - locs.add(tm.getLocation().getHostPort()); + locs.add(tm.getLocation().getServerInstance().getHostPort()); } catch (Exception ex) { scanner.close(); return tabletServers; diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java index 102267aaf70..258a2919caf 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java @@ -53,6 +53,7 @@ import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.thrift.InitialMultiScan; import org.apache.accumulo.core.dataImpl.thrift.InitialScan; @@ -190,8 +191,6 @@ private TabletMetadataLoader(Ample ample) { private ZooCache managerLockCache; - private final String groupName; - public ScanServer(ConfigOpts opts, String[] args) { super("sserver", opts, args); @@ -236,8 +235,6 @@ public ScanServer(ConfigOpts opts, String[] args) { delegate = newThriftScanClientHandler(new WriteTracker()); - this.groupName = getConfiguration().get(Property.SSERV_GROUP_NAME); - ThreadPools.watchCriticalScheduledTask(getContext().getScheduledExecutor() .scheduleWithFixedDelay(() -> cleanUpReservedFiles(scanServerReservationExpiration), scanServerReservationExpiration, scanServerReservationExpiration, @@ -245,6 +242,11 @@ public ScanServer(ConfigOpts opts, String[] args) { } + @Override + protected String getResourceGroupNamePropertyValue(SiteConfiguration conf) { + return conf.get(Property.SSERV_GROUP_NAME); + } + @VisibleForTesting protected ThriftScanClientHandler newThriftScanClientHandler(WriteTracker writeTracker) { return new ThriftScanClientHandler(this, writeTracker); @@ -327,7 +329,7 @@ public void unableToMonitorLockNode(final Exception e) { zoo.putPersistentData(zLockPath.toString(), new byte[0], NodeExistsPolicy.SKIP); if (scanServerLock.tryLock(lw, new ServiceLockData(serverLockUUID, getClientAddressString(), - ThriftService.TABLET_SCAN, this.groupName))) { + ThriftService.TABLET_SCAN, getResourceGroupName()))) { LOG.debug("Obtained scan server lock {}", scanServerLock.getLockPath()); return scanServerLock; } @@ -357,7 +359,7 @@ public void run() { try { MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, - clientAddress); + clientAddress, getResourceGroupName()); scanMetrics = new TabletServerScanMetrics(); MetricsUtil.initializeProducers(this, scanMetrics); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index 805d32a01b9..077f122b3aa 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@ -73,6 +73,7 @@ import org.apache.accumulo.core.clientImpl.DurabilityImpl; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; @@ -368,9 +369,15 @@ private void logBusyTablets(List> busyTablets, } else { authKeyWatcher = null; } + config(); } + @Override + protected String getResourceGroupNamePropertyValue(SiteConfiguration conf) { + return conf.get(Property.TSERV_GROUP_NAME); + } + public InstanceId getInstanceID() { return getContext().getInstanceID(); } @@ -672,8 +679,8 @@ public void unableToMonitorLockNode(final Exception e) { for (ThriftService svc : new ThriftService[] {ThriftService.CLIENT, ThriftService.TABLET_INGEST, ThriftService.TABLET_MANAGEMENT, ThriftService.TABLET_SCAN, ThriftService.TSERV}) { - descriptors - .addService(new ServiceDescriptor(tabletServerUUID, svc, getClientAddressString())); + descriptors.addService(new ServiceDescriptor(tabletServerUUID, svc, + getClientAddressString(), getResourceGroupName())); } if (tabletServerLock.tryLock(lw, new ServiceLockData(descriptors))) { @@ -719,8 +726,8 @@ public void run() { } try { - MetricsUtil.initializeMetrics(context.getConfiguration(), this.applicationName, - clientAddress); + MetricsUtil.initializeMetrics(context.getConfiguration(), this.applicationName, clientAddress, + getResourceGroupName()); metrics = new TabletServerMetrics(this); updateMetrics = new TabletServerUpdateMetrics(); @@ -945,7 +952,7 @@ public TServerInstance getTabletSession() { } try { - return new TServerInstance(address, tabletServerLock.getSessionId()); + return new TServerInstance(address, tabletServerLock.getSessionId(), getResourceGroupName()); } catch (Exception ex) { log.warn("Unable to read session from tablet server lock" + ex); return null; diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java index e2843890f7f..d544db1302f 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java @@ -112,8 +112,8 @@ public void run() { } try { - TServerInstance instance = - new TServerInstance(server.clientAddress, server.getLock().getSessionId()); + TServerInstance instance = new TServerInstance(server.clientAddress, + server.getLock().getSessionId(), server.getResourceGroupName()); // ELASTICITY_TODO: Modify Tablet to keep a reference to TableMetadata so that we // can get rid of TabletMetadataImposter TabletMetadata tm = new TabletMetadataImposter(extent, null, Location.current(instance), null, diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java index 8d1171b4221..1b34968ea78 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java @@ -473,7 +473,8 @@ Optional bringMajorCompactionOnline(Set oldD ManagerMetadataUtil.replaceDatafiles(tablet.getContext(), extent, oldDatafiles, filesInUseByScans, newFile, compactionIdToWrite, dfv, tablet.getTabletServer().getClientAddressString(), lastLocation, - tablet.getTabletServer().getLock(), ecid); + tablet.getTabletServer().getLock(), ecid, + tablet.getTabletServer().getResourceGroupName()); tablet.setLastCompactionID(compactionIdToWrite); removeFilesAfterScan(filesInUseByScans); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java index 538c2387c76..3260683e4d5 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java @@ -2045,7 +2045,7 @@ public Optional updateTabletDataFile(long maxCommittedTime, return ManagerMetadataUtil.updateTabletDataFile(getTabletServer().getContext(), extent, newDatafile, dfv, tabletTime.getMetadataTime(persistedTime), tabletServer.getClientAddressString(), tabletServer.getLock(), unusedWalLogs, - lastLocation, flushId); + lastLocation, flushId, tabletServer.getResourceGroupName()); } } diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java index 346854c003c..5c01acc088b 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.FutureLocationColumnFamily; @@ -96,19 +97,26 @@ public void testBadTabletMetadata() throws Exception { TabletColumnFamily.encodePrevEndRow(null).get()); put(tabletMeta, "1<", ServerColumnFamily.DIRECTORY_COLUMN, "t1".getBytes()); put(tabletMeta, "1<", ServerColumnFamily.TIME_COLUMN, "M0".getBytes()); - put(tabletMeta, "1<", FutureLocationColumnFamily.NAME, "4", "127.0.0.1:9997"); + put(tabletMeta, "1<", FutureLocationColumnFamily.NAME, "", + new TServerInstance("127.0.0.1:9997", 4, "default").toString()); - TServerInstance tsi = new TServerInstance("127.0.0.1:9997", 4); + TServerInstance tsi = + new TServerInstance("127.0.0.1:9997", 4, ServiceDescriptor.DEFAULT_GROUP_NAME); TabletMetadata tm = TabletMetadata.convertRow(tabletMeta.entrySet().iterator(), EnumSet.allOf(ColumnType.class), true, false); assertTrue(checkTabletMetadata(ke, tsi, tm)); - assertFail(tabletMeta, ke, new TServerInstance("127.0.0.1:9998", 4)); - assertFail(tabletMeta, ke, new TServerInstance("127.0.0.1:9998", 5)); - assertFail(tabletMeta, ke, new TServerInstance("127.0.0.1:9997", 5)); - assertFail(tabletMeta, ke, new TServerInstance("127.0.0.2:9997", 4)); - assertFail(tabletMeta, ke, new TServerInstance("127.0.0.2:9997", 5)); + assertFail(tabletMeta, ke, + new TServerInstance("127.0.0.1:9998", 4, ServiceDescriptor.DEFAULT_GROUP_NAME)); + assertFail(tabletMeta, ke, + new TServerInstance("127.0.0.1:9998", 5, ServiceDescriptor.DEFAULT_GROUP_NAME)); + assertFail(tabletMeta, ke, + new TServerInstance("127.0.0.1:9997", 5, ServiceDescriptor.DEFAULT_GROUP_NAME)); + assertFail(tabletMeta, ke, + new TServerInstance("127.0.0.2:9997", 4, ServiceDescriptor.DEFAULT_GROUP_NAME)); + assertFail(tabletMeta, ke, + new TServerInstance("127.0.0.2:9997", 5, ServiceDescriptor.DEFAULT_GROUP_NAME)); assertFail(tabletMeta, new KeyExtent(TableId.of("1"), null, new Text("m")), tsi); @@ -120,12 +128,12 @@ public void testBadTabletMetadata() throws Exception { assertFail(tabletMeta, ke, tsi, newKey("1<", ServerColumnFamily.TIME_COLUMN)); - assertFail(tabletMeta, ke, tsi, newKey("1<", FutureLocationColumnFamily.NAME, "4")); + assertFail(tabletMeta, ke, tsi, newKey("1<", FutureLocationColumnFamily.NAME, "")); TreeMap copy = new TreeMap<>(tabletMeta); put(copy, "1<", CurrentLocationColumnFamily.NAME, "4", "127.0.0.1:9997"); assertFail(copy, ke, tsi); - assertFail(copy, ke, tsi, newKey("1<", FutureLocationColumnFamily.NAME, "4")); + assertFail(copy, ke, tsi, newKey("1<", FutureLocationColumnFamily.NAME, "")); copy = new TreeMap<>(tabletMeta); put(copy, "1<", CurrentLocationColumnFamily.NAME, "5", "127.0.0.1:9998"); diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java index 71b2a18b75d..9b0877f3952 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java +++ b/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java @@ -427,7 +427,7 @@ public Factory location(Location location) { if (location == null) { this.location = "None"; } else { - String server = location.getHostPort(); + String server = location.getServerInstance().getHostPort(); this.location = location.getType() + ":" + server; } return this; diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java index 583819b96b9..7672ee710c9 100644 --- a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java +++ b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java @@ -34,6 +34,8 @@ import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; +import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletState; import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; import org.apache.accumulo.shell.Shell; @@ -91,7 +93,8 @@ protected List getMetadataInfo(Shell shellState, KeyExtent ke1 = new KeyExtent(tableId, new Text("a"), null); KeyExtent ke2 = new KeyExtent(tableId, new Text("m"), new Text("a")); KeyExtent ke3 = new KeyExtent(tableId, null, new Text("m")); - Location loc = Location.current("localhost", ""); + Location loc = Location + .current(new TServerInstance("localhost:9997", "", ServiceDescriptor.DEFAULT_GROUP_NAME)); ListTabletsCommand.TabletRowInfo.Factory factory = new ListTabletsCommand.TabletRowInfo.Factory(tableName, ke1).dir("t-dir1").numFiles(1) .numWalLogs(1).numEntries(1).size(100).status(TabletState.HOSTED.toString()) @@ -168,7 +171,8 @@ public void builderTest() { Text startRow = new Text("a"); Text endRow = new Text("z"); KeyExtent ke = new KeyExtent(id, endRow, startRow); - Location loc = Location.current("localhost", ""); + Location loc = Location + .current(new TServerInstance("localhost:9997", "", ServiceDescriptor.DEFAULT_GROUP_NAME)); ListTabletsCommand.TabletRowInfo.Factory factory = new ListTabletsCommand.TabletRowInfo.Factory("aName", ke).numFiles(1).numWalLogs(2) .numEntries(3).size(4).status(TabletState.HOSTED.toString()).location(loc) @@ -184,7 +188,7 @@ public void builderTest() { assertEquals("4", info.getSize(false)); assertEquals(4, info.size); assertEquals("HOSTED", info.status); - assertEquals("CURRENT:localhost", info.location); + assertEquals("CURRENT:localhost:9997", info.location); assertEquals(TableId.of("123"), info.tableId); assertEquals(startRow + " " + endRow, info.getTablet()); assertTrue(info.tableExists); diff --git a/test/src/main/java/org/apache/accumulo/test/LocatorIT.java b/test/src/main/java/org/apache/accumulo/test/LocatorIT.java index bf090bcb33c..8e537f18418 100644 --- a/test/src/main/java/org/apache/accumulo/test/LocatorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/LocatorIT.java @@ -96,7 +96,7 @@ public void testBasic() throws Exception { final Predicate alwaysHostedAndCurrentNotNull = t -> t.getHostingGoal() == TabletHostingGoal.ALWAYS && t.hasCurrent() - && t.getLocation().getHostAndPort() != null; + && t.getLocation().getServerInstance().getHostAndPort() != null; try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { String tableName = getUniqueNames(1)[0]; diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java index 5c25fd9fd70..37edf32c098 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java @@ -33,6 +33,7 @@ import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.StoredTabletFile; @@ -69,8 +70,8 @@ public void testLocations() throws Exception { var e1 = new KeyExtent(tid, new Text("c"), null); - var ts1 = new TServerInstance("localhost:9997", 5000L); - var ts2 = new TServerInstance("localhost:9997", 6000L); + var ts1 = new TServerInstance("localhost:9997", 5000L, ServiceDescriptor.DEFAULT_GROUP_NAME); + var ts2 = new TServerInstance("localhost:9997", 6000L, ServiceDescriptor.DEFAULT_GROUP_NAME); var context = cluster.getServerContext(); @@ -146,8 +147,8 @@ public void testFiles() throws Exception { var e1 = new KeyExtent(tid, new Text("c"), null); - var ts1 = new TServerInstance("localhost:9997", 5000L); - var ts2 = new TServerInstance("localhost:9997", 6000L); + var ts1 = new TServerInstance("localhost:9997", 5000L, ServiceDescriptor.DEFAULT_GROUP_NAME); + var ts2 = new TServerInstance("localhost:9997", 6000L, ServiceDescriptor.DEFAULT_GROUP_NAME); var context = cluster.getServerContext(); @@ -279,8 +280,8 @@ public void testMultipleExtents() throws Exception { var e3 = new KeyExtent(tid, new Text("j"), new Text("f")); var e4 = new KeyExtent(tid, null, new Text("j")); - var ts1 = new TServerInstance("localhost:9997", 5000L); - var ts2 = new TServerInstance("localhost:9997", 6000L); + var ts1 = new TServerInstance("localhost:9997", 5000L, ServiceDescriptor.DEFAULT_GROUP_NAME); + var ts2 = new TServerInstance("localhost:9997", 6000L, ServiceDescriptor.DEFAULT_GROUP_NAME); var context = cluster.getServerContext(); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java index 6e326de445a..af5a9113e0c 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java @@ -67,7 +67,8 @@ public void test() throws Exception { newTablet = ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); } while (!newTablet.hasCurrent()); // this would be null if the mode was not "assign" - assertEquals(newTablet.getLocation().getHostPort(), newTablet.getLast().getHostPort()); + assertEquals(newTablet.getLocation().getServerInstance().getHostPort(), + newTablet.getLast().getServerInstance().getHostPort()); // put something in it try (BatchWriter bw = c.createBatchWriter(tableName)) { @@ -82,8 +83,10 @@ public void test() throws Exception { // last location should not be set yet TabletMetadata unflushed = ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); - assertEquals(newTablet.getLocation().getHostPort(), unflushed.getLocation().getHostPort()); - assertEquals(newTablet.getLocation().getHostPort(), unflushed.getLast().getHostPort()); + assertEquals(newTablet.getLocation().getServerInstance().getHostPort(), + unflushed.getLocation().getServerInstance().getHostPort()); + assertEquals(newTablet.getLocation().getServerInstance().getHostPort(), + unflushed.getLast().getServerInstance().getHostPort()); assertTrue(newTablet.hasCurrent()); // take the tablet offline @@ -92,7 +95,8 @@ public void test() throws Exception { ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertNull(offline.getLocation()); assertFalse(offline.hasCurrent()); - assertEquals(newTablet.getLocation().getHostPort(), offline.getLast().getHostPort()); + assertEquals(newTablet.getLocation().getServerInstance().getHostPort(), + offline.getLast().getServerInstance().getHostPort()); // put it back online, should have the same last location c.tableOperations().online(tableName, true); @@ -100,7 +104,8 @@ public void test() throws Exception { ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(online.hasCurrent()); assertNotNull(online.getLocation()); - assertEquals(newTablet.getLast().getHostPort(), online.getLast().getHostPort()); + assertEquals(newTablet.getLast().getServerInstance().getHostPort(), + online.getLast().getServerInstance().getHostPort()); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java b/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java index 471aedf566d..8e12e97eaf8 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java @@ -81,7 +81,8 @@ public void test() throws Exception { // no last location should be set yet TabletMetadata unflushed = ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); - assertEquals(newTablet.getLocation().getHostPort(), unflushed.getLocation().getHostPort()); + assertEquals(newTablet.getLocation().getServerInstance().getHostPort(), + unflushed.getLocation().getServerInstance().getHostPort()); assertNull(unflushed.getLast()); assertTrue(newTablet.hasCurrent()); @@ -90,8 +91,10 @@ public void test() throws Exception { TabletMetadata flushed = ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); - assertEquals(newTablet.getLocation().getHostPort(), flushed.getLocation().getHostPort()); - assertEquals(flushed.getLocation().getHostPort(), flushed.getLast().getHostPort()); + assertEquals(newTablet.getLocation().getServerInstance().getHostPort(), + flushed.getLocation().getServerInstance().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + flushed.getLast().getServerInstance().getHostPort()); assertTrue(newTablet.hasCurrent()); // take the tablet offline @@ -99,7 +102,8 @@ public void test() throws Exception { TabletMetadata offline = ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertNull(offline.getLocation()); - assertEquals(flushed.getLocation().getHostPort(), offline.getLast().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + offline.getLast().getServerInstance().getHostPort()); // put it back online, should have the same last location c.tableOperations().online(tableName, true); @@ -107,7 +111,8 @@ public void test() throws Exception { ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(online.hasCurrent()); assertNotNull(online.getLocation()); - assertEquals(offline.getLast().getHostPort(), online.getLast().getHostPort()); + assertEquals(offline.getLast().getServerInstance().getHostPort(), + online.getLast().getServerInstance().getHostPort()); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java index 93626de9580..699b23c7e08 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java @@ -140,7 +140,8 @@ public void test() throws Exception { TabletMetadata flushed = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(flushed.hasCurrent()); assertNotNull(flushed.getLocation()); - assertEquals(flushed.getLocation().getHostPort(), flushed.getLast().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + flushed.getLast().getServerInstance().getHostPort()); assertFalse(flushed.getLocation().getType().equals(LocationType.FUTURE)); assertEquals(TabletHostingGoal.ONDEMAND, flushed.getHostingGoal()); @@ -149,7 +150,8 @@ public void test() throws Exception { TabletMetadata offline = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertFalse(offline.hasCurrent()); assertNull(offline.getLocation()); - assertEquals(flushed.getLocation().getHostPort(), offline.getLast().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + offline.getLast().getServerInstance().getHostPort()); assertEquals(TabletHostingGoal.ONDEMAND, offline.getHostingGoal()); // put it back online @@ -157,7 +159,8 @@ public void test() throws Exception { TabletMetadata online = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(online.hasCurrent()); assertNotNull(online.getLocation()); - assertEquals(online.getLocation().getHostPort(), online.getLast().getHostPort()); + assertEquals(online.getLocation().getServerInstance().getHostPort(), + online.getLast().getServerInstance().getHostPort()); assertEquals(TabletHostingGoal.ONDEMAND, online.getHostingGoal()); // set the hosting goal to always @@ -172,7 +175,8 @@ public void test() throws Exception { final TabletMetadata always = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(alwaysHostedOrCurrentNotNull.test(always)); assertTrue(always.hasCurrent()); - assertEquals(flushed.getLocation().getHostPort(), always.getLast().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + always.getLast().getServerInstance().getHostPort()); assertEquals(TabletHostingGoal.ALWAYS, always.getHostingGoal()); // set the hosting goal to never @@ -185,7 +189,8 @@ public void test() throws Exception { final TabletMetadata never = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(neverHostedOrCurrentNull.test(never)); assertNull(never.getLocation()); - assertEquals(flushed.getLocation().getHostPort(), never.getLast().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + never.getLast().getServerInstance().getHostPort()); assertEquals(TabletHostingGoal.NEVER, never.getHostingGoal()); // set the hosting goal to ondemand @@ -198,7 +203,8 @@ public void test() throws Exception { final TabletMetadata ondemand = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(ondemandHosted.test(ondemand)); assertNull(ondemand.getLocation()); - assertEquals(flushed.getLocation().getHostPort(), ondemand.getLast().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + ondemand.getLast().getServerInstance().getHostPort()); assertEquals(TabletHostingGoal.ONDEMAND, ondemand.getHostingGoal()); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/RegexGroupBalanceIT.java b/test/src/main/java/org/apache/accumulo/test/functional/RegexGroupBalanceIT.java index b7194b86bd2..6327f73a42a 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/RegexGroupBalanceIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/RegexGroupBalanceIT.java @@ -194,8 +194,8 @@ private Table getCounts(AccumuloClient client, String } else { group = group.substring(tableId.canonical().length() + 1).substring(0, 2); } - String loc = new TServerInstance(entry.getValue(), entry.getKey().getColumnQualifier()) - .getHostPortSession(); + String loc = + TServerInstance.fromString(entry.getValue().toString()).getHostPortSessionGroup(); MutableInt count = groupLocationCounts.get(group, loc); if (count == null) { diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java index 379c259494d..fc3e563e83e 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java @@ -51,6 +51,7 @@ import org.apache.accumulo.core.lock.ServiceLock.LockLossReason; import org.apache.accumulo.core.lock.ServiceLock.LockWatcher; import org.apache.accumulo.core.lock.ServiceLockData; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.lock.ServiceLockData.ThriftService; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.ReferencedTabletFile; @@ -211,7 +212,8 @@ private void splitPartiallyAndRecover(ServerContext context, KeyExtent extent, K lowDatafileSizes, highDatafileSizes, highDatafilesToRemove); MetadataTableUtil.splitTablet(high, extent.prevEndRow(), splitRatio, context, zl, Set.of()); - TServerInstance instance = new TServerInstance(location, zl.getSessionId()); + TServerInstance instance = + new TServerInstance(location, zl.getSessionId(), ServiceDescriptor.DEFAULT_GROUP_NAME); Assignment assignment = new Assignment(high, instance, null); TabletMutator tabletMutator = context.getAmple().mutateTablet(extent); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java index 1b3a14df5e5..1b2ecbc10c3 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java @@ -57,6 +57,7 @@ import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.lock.ServiceLock; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.manager.thrift.ManagerState; @@ -375,7 +376,8 @@ public Set onlineTabletServers() { var zPath = ServiceLock.path(ZooUtil.getRoot(context.instanceOperations().getInstanceId()) + Constants.ZTSERVERS + "/" + tserver); long sessionId = ServiceLock.getSessionId(context.getZooCache(), zPath); - tservers.add(new TServerInstance(tserver, sessionId)); + tservers + .add(new TServerInstance(tserver, sessionId, ServiceDescriptor.DEFAULT_GROUP_NAME)); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/test/src/main/java/org/apache/accumulo/test/manager/MergeStateIT.java b/test/src/main/java/org/apache/accumulo/test/manager/MergeStateIT.java index d49c932f2d2..2a12aa3394b 100644 --- a/test/src/main/java/org/apache/accumulo/test/manager/MergeStateIT.java +++ b/test/src/main/java/org/apache/accumulo/test/manager/MergeStateIT.java @@ -39,6 +39,7 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.manager.thrift.ManagerState; import org.apache.accumulo.core.metadata.MetadataTable; @@ -69,8 +70,8 @@ public class MergeStateIT extends ConfigurableMacBase { private static class MockCurrentState implements CurrentState { - TServerInstance someTServer = - new TServerInstance(HostAndPort.fromParts("127.0.0.1", 1234), 0x123456); + TServerInstance someTServer = new TServerInstance(HostAndPort.fromParts("127.0.0.1", 1234), + 0x123456, ServiceDescriptor.DEFAULT_GROUP_NAME); MergeInfo mergeInfo; MockCurrentState(MergeInfo info) { diff --git a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java index 125af5b23c8..fb4d6fcb517 100644 --- a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java @@ -53,6 +53,7 @@ import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.TServerInstance; @@ -191,7 +192,8 @@ public void shutdownAndResumeTserver() throws Exception { ClientTabletCache.LocationNeed.REQUIRED); // add it to the set of servers with metadata metadataServerSet.add(new TServerInstance(tab.getTserverLocation().orElseThrow(), - Long.valueOf(tab.getTserverSession().orElseThrow(), 16))); + Long.valueOf(tab.getTserverSession().orElseThrow(), 16), + ServiceDescriptor.DEFAULT_GROUP_NAME)); } } @@ -206,7 +208,7 @@ public void shutdownAndResumeTserver() throws Exception { Collections.shuffle(tserversList, random); for (int i1 = 0; i1 < count; ++i1) { - final String tserverName = tserversList.get(i1).getHostPortSession(); + final String tserverName = tserversList.get(i1).getHostPortSessionGroup(); ThriftClientTypes.MANAGER.executeVoid(ctx, client -> { log.info("Sending shutdown command to {} via ManagerClientService", tserverName); client.shutdownTabletServer(null, ctx.rpcCreds(), tserverName, false); @@ -443,7 +445,7 @@ private void scan(ClientContext ctx, String tableName, String metaName) { suspended.put(tm.getSuspend().server, ke); ++suspendedCount; } else if (tm.hasCurrent()) { - hosted.put(tm.getLocation().getHostAndPort(), ke); + hosted.put(tm.getLocation().getServerInstance().getHostAndPort(), ke); ++hostedCount; } else if (tm.getLocation() != null && tm.getLocation().getType().equals(LocationType.FUTURE)) { diff --git a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java index 5d2be5df56b..e47001de011 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java @@ -53,6 +53,7 @@ import org.apache.accumulo.core.dataImpl.thrift.TSummaries; import org.apache.accumulo.core.dataImpl.thrift.TSummaryRequest; import org.apache.accumulo.core.dataImpl.thrift.UpdateErrors; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.thrift.TabletServerStatus; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.TServerInstance; @@ -358,7 +359,8 @@ public static void main(String[] args) throws Exception { List assignments = new ArrayList<>(); try (var s = new TabletManagementScanner(context, tableRange, MetadataTable.NAME)) { long randomSessionID = opts.port; - TServerInstance instance = new TServerInstance(addr, randomSessionID); + TServerInstance instance = + new TServerInstance(addr, randomSessionID, ServiceDescriptor.DEFAULT_GROUP_NAME); while (s.hasNext()) { TabletMetadata next = s.next().getTabletMetadata(); diff --git a/test/src/test/java/org/apache/accumulo/test/ChaoticLoadBalancerTest.java b/test/src/test/java/org/apache/accumulo/test/ChaoticLoadBalancerTest.java index 8b803e70895..e740146a33b 100644 --- a/test/src/test/java/org/apache/accumulo/test/ChaoticLoadBalancerTest.java +++ b/test/src/test/java/org/apache/accumulo/test/ChaoticLoadBalancerTest.java @@ -34,6 +34,7 @@ import org.apache.accumulo.core.data.TabletId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.TabletIdImpl; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.balancer.AssignmentParamsImpl; import org.apache.accumulo.core.manager.balancer.BalanceParamsImpl; import org.apache.accumulo.core.manager.balancer.TServerStatusImpl; @@ -95,9 +96,15 @@ public List getOnlineTabletsForTable(TabletServerId tserver, T @Test public void testAssignMigrations() { servers.clear(); - servers.put(new TabletServerIdImpl("127.0.0.1", 1234, "a"), new FakeTServer()); - servers.put(new TabletServerIdImpl("127.0.0.1", 1235, "b"), new FakeTServer()); - servers.put(new TabletServerIdImpl("127.0.0.1", 1236, "c"), new FakeTServer()); + servers.put( + new TabletServerIdImpl("127.0.0.1", 1234, "a", ServiceDescriptor.DEFAULT_GROUP_NAME), + new FakeTServer()); + servers.put( + new TabletServerIdImpl("127.0.0.1", 1235, "b", ServiceDescriptor.DEFAULT_GROUP_NAME), + new FakeTServer()); + servers.put( + new TabletServerIdImpl("127.0.0.1", 1236, "c", ServiceDescriptor.DEFAULT_GROUP_NAME), + new FakeTServer()); Map metadataTable = new TreeMap<>(); String table = "t1"; metadataTable.put(makeTablet(table, null, null), null); @@ -134,7 +141,8 @@ public void testUnevenAssignment() { servers.clear(); for (char c : "abcdefghijklmnopqrstuvwxyz".toCharArray()) { String cString = Character.toString(c); - TabletServerId tsi = new TabletServerIdImpl("127.0.0.1", c, cString); + TabletServerId tsi = + new TabletServerIdImpl("127.0.0.1", c, cString, ServiceDescriptor.DEFAULT_GROUP_NAME); FakeTServer fakeTServer = new FakeTServer(); servers.put(tsi, fakeTServer); fakeTServer.tablets.add(makeTablet(cString, null, null)); From 8db080a4fa6ae43e116fa6189842c5c25651f37b Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Mon, 12 Jun 2023 14:56:22 +0000 Subject: [PATCH 2/3] Removed Interner, no longer referenced --- .../apache/accumulo/core/util/Interner.java | 46 ------- .../accumulo/core/util/InternerTest.java | 112 ------------------ 2 files changed, 158 deletions(-) delete mode 100644 core/src/main/java/org/apache/accumulo/core/util/Interner.java delete mode 100644 core/src/test/java/org/apache/accumulo/core/util/InternerTest.java diff --git a/core/src/main/java/org/apache/accumulo/core/util/Interner.java b/core/src/main/java/org/apache/accumulo/core/util/Interner.java deleted file mode 100644 index 0f82d5185ae..00000000000 --- a/core/src/main/java/org/apache/accumulo/core/util/Interner.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.core.util; - -import java.lang.ref.WeakReference; -import java.util.WeakHashMap; - -/** - * A utility that mimics String.intern() for any immutable object type (including String). - */ -public class Interner { - private final WeakHashMap> internTable = new WeakHashMap<>(); - - public synchronized T intern(T item) { - WeakReference ref = internTable.get(item); - if (ref != null) { - T oldItem = ref.get(); - if (oldItem != null) { - return oldItem; - } - } - internTable.put(item, new WeakReference<>(item)); - return item; - } - - // for testing - synchronized int size() { - return internTable.size(); - } -} diff --git a/core/src/test/java/org/apache/accumulo/core/util/InternerTest.java b/core/src/test/java/org/apache/accumulo/core/util/InternerTest.java deleted file mode 100644 index fb7642eed5e..00000000000 --- a/core/src/test/java/org/apache/accumulo/core/util/InternerTest.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.core.util; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotSame; -import static org.junit.jupiter.api.Assertions.assertSame; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -public class InternerTest { - - private class TestObj { - private final int id; - - TestObj(int id) { - this.id = id; - } - - @Override - public int hashCode() { - return id; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof TestObj) { - return ((TestObj) obj).id == this.id; - } - return false; - } - } - - @Test - public void testInternDedupes() { - var interner = new Interner(); - - var obj1 = new TestObj(1); - var obj1_dupe = new TestObj(1); - assertSame(obj1, obj1); - assertNotSame(obj1, obj1_dupe); - assertEquals(obj1, obj1_dupe); - assertEquals(obj1.hashCode(), obj1_dupe.hashCode()); - - // verify object gets added to the intern pool - assertSame(obj1, interner.intern(obj1)); - assertEquals(1, interner.size()); - - // verify equivalent, but not the same object, gets deduplicated - assertSame(obj1, interner.intern(obj1_dupe)); - assertEquals(1, interner.size()); - - // verify second object grows the intern pool size - var obj2 = new TestObj(2); - assertNotSame(obj1, obj2); - assertNotEquals(obj1, obj2); - var intern2 = interner.intern(obj2); - assertEquals(2, interner.size()); - - // sanity check to ensure we got the same object back for obj2, and it's not mangled with obj1 - assertSame(obj2, intern2); - assertNotEquals(obj1, intern2); - } - - @Test - @Timeout(20) - public void testInternsGetGarbageCollected() { - var interner = new Interner(); - assertEquals(0, interner.size()); // ensure empty - - // add one and keep a strong reference - var obj1 = interner.intern(new TestObj(1)); - assertEquals(1, interner.size()); - - // try to add a second, weakly referenced object until it sticks (may be GC'd between checks) - do { - interner.intern(new TestObj(2)); - } while (interner.size() != 2); - - // best effort to GC until the weakly reachable object goes away or until test times out - do { - System.gc(); - } while (interner.size() != 1); - - // ensure obj1 is still interned (because we kept a strong reference) - assertSame(obj1, interner.intern(new TestObj(1))); - - // ensure second test object is entirely new (previous ones should have been GC'd) - var obj2 = new TestObj(2); - assertSame(obj2, interner.intern(obj2)); - assertEquals(2, interner.size()); - } - -} From eeb38b5f242a6e8d1efa6179430cf8660b15a7c9 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Mon, 12 Jun 2023 21:13:41 +0000 Subject: [PATCH 3/3] Changes to TServerInstance based on PR feedback --- .../apache/accumulo/core/metadata/TServerInstance.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java b/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java index 3f0dc2d6781..65fb3762430 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java @@ -91,19 +91,11 @@ public int compareTo(TServerInstance other) { @Override public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((group == null) ? 0 : group.hashCode()); - result = - prime * result + ((hostPortSessionGroup == null) ? 0 : hostPortSessionGroup.hashCode()); - return result; + return this.hostPortSessionGroup.hashCode(); } @Override public boolean equals(Object obj) { - if (obj == null) { - return false; - } if (obj instanceof TServerInstance) { return compareTo((TServerInstance) obj) == 0; }