diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java index daa3dd35da2..07dca96f8a0 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java @@ -498,7 +498,7 @@ public String getRootTabletLocation() { return null; } - return loc.getHostPort(); + return loc.getServerInstance().getHostPort(); } /** diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java index fb8c4f7a8c4..014cdadcd50 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java @@ -43,9 +43,9 @@ import org.apache.accumulo.core.metadata.MetadataCachedTabletObtainer; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.singletons.SingletonManager; import org.apache.accumulo.core.singletons.SingletonService; -import org.apache.accumulo.core.util.Interner; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.hadoop.io.Text; @@ -270,34 +270,29 @@ public List getCachedTablets() { } public static class CachedTablet { - private static final Interner interner = new Interner<>(); private final KeyExtent tablet_extent; - private final String tserverLocation; - private final String tserverSession; + private final TServerInstance tsi; private final TabletHostingGoal goal; private final boolean hostingRequested; private final Long creationTime = System.nanoTime(); - public CachedTablet(KeyExtent tablet_extent, String tablet_location, String session, - TabletHostingGoal goal, boolean hostingRequested) { + public CachedTablet(KeyExtent tablet_extent, TServerInstance instance, TabletHostingGoal goal, + boolean hostingRequested) { checkArgument(tablet_extent != null, "tablet_extent is null"); - checkArgument(tablet_location != null, "tablet_location is null"); - checkArgument(session != null, "session is null"); + checkArgument(instance != null, "tablet_location is null"); this.tablet_extent = tablet_extent; - this.tserverLocation = interner.intern(tablet_location); - this.tserverSession = interner.intern(session); + this.tsi = instance; this.goal = Objects.requireNonNull(goal); this.hostingRequested = hostingRequested; } - public CachedTablet(KeyExtent tablet_extent, Optional tablet_location, - Optional session, TabletHostingGoal goal, boolean hostingRequested) { + public CachedTablet(KeyExtent tablet_extent, Optional tablet_location, + TabletHostingGoal goal, boolean hostingRequested) { checkArgument(tablet_extent != null, "tablet_extent is null"); this.tablet_extent = tablet_extent; - this.tserverLocation = tablet_location.map(interner::intern).orElse(null); - this.tserverSession = session.map(interner::intern).orElse(null); + this.tsi = tablet_location.orElse(null); this.goal = Objects.requireNonNull(goal); this.hostingRequested = hostingRequested; } @@ -305,8 +300,7 @@ public CachedTablet(KeyExtent tablet_extent, Optional tablet_location, public CachedTablet(KeyExtent tablet_extent, TabletHostingGoal goal, boolean hostingRequested) { checkArgument(tablet_extent != null, "tablet_extent is null"); this.tablet_extent = tablet_extent; - this.tserverLocation = null; - this.tserverSession = null; + this.tsi = null; this.goal = Objects.requireNonNull(goal); this.hostingRequested = hostingRequested; } @@ -325,13 +319,13 @@ && getTserverSession().equals(otl.getTserverSession()) && getGoal() == otl.getGo @Override public int hashCode() { - return Objects.hash(getExtent(), tserverLocation, tserverSession, goal, hostingRequested); + return Objects.hash(getExtent(), tsi, goal, hostingRequested); } @Override public String toString() { return "(" + getExtent() + "," + getTserverLocation() + "," + getTserverSession() + "," - + getGoal() + ")"; + + getResourceGroup() + "," + getGoal() + ")"; } public KeyExtent getExtent() { @@ -339,11 +333,19 @@ public KeyExtent getExtent() { } public Optional getTserverLocation() { - return Optional.ofNullable(tserverLocation); + return Optional.ofNullable(tsi == null ? null : tsi.getHostPort()); } public Optional getTserverSession() { - return Optional.ofNullable(tserverSession); + return Optional.ofNullable(tsi == null ? null : tsi.getSession()); + } + + public Optional getResourceGroup() { + return Optional.ofNullable(tsi == null ? null : tsi.getGroup()); + } + + public Optional getServer() { + return Optional.ofNullable(tsi == null ? null : tsi); } /** diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java index 5e3e8bb440e..5c37c0a5a1c 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java @@ -541,8 +541,8 @@ public CachedTablet findTablet(ClientContext context, Text row, boolean skipRow, OpTimer timer = null; if (log.isTraceEnabled()) { - log.trace("tid={} Locating tablet table={} row={} skipRow={}", - Thread.currentThread().getId(), tableId, TextUtil.truncate(row), skipRow); + log.trace("tid={} Locating tablet table={} row={} skipRow={}", Thread.currentThread().getId(), + tableId, TextUtil.truncate(row), skipRow); timer = new OpTimer().start(); } @@ -686,8 +686,7 @@ private void lookupTablet(ClientContext context, Text row, boolean retry, if ((lastEndRow != null) && (ke.prevEndRow() != null) && ke.prevEndRow().equals(lastEndRow)) { locToCache = new CachedTablet(new KeyExtent(ke.tableId(), ke.endRow(), lastEndRow), - cachedTablet.getTserverLocation(), cachedTablet.getTserverSession(), - cachedTablet.getGoal(), cachedTablet.wasHostingRequested()); + cachedTablet.getServer(), cachedTablet.getGoal(), cachedTablet.wasHostingRequested()); } else { locToCache = cachedTablet; } @@ -695,8 +694,11 @@ private void lookupTablet(ClientContext context, Text row, boolean retry, // save endRow for next iteration lastEndRow = locToCache.getExtent().endRow(); + log.trace("Caching tablet location: {}", locToCache); updateCache(locToCache, lcSession); } + } else { + log.warn("Metadata tablet not found for row: {}", metadataRow); } } diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java index 7e35fb259e0..a9ad981307f 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java @@ -123,18 +123,17 @@ protected CachedTablet getRootTabletLocation(ClientContext context) { } if (loc == null || loc.getType() != LocationType.CURRENT) { - return new CachedTablet(RootTable.EXTENT, Optional.empty(), Optional.empty(), - TabletHostingGoal.ALWAYS, false); + return new CachedTablet(RootTable.EXTENT, Optional.empty(), TabletHostingGoal.ALWAYS, false); } - String server = loc.getHostPort(); + String server = loc.getServerInstance().getHostPort(); + String session = loc.getServerInstance().getSession(); - if (lockChecker.isLockHeld(server, loc.getSession())) { - return new CachedTablet(RootTable.EXTENT, server, loc.getSession(), TabletHostingGoal.ALWAYS, + if (lockChecker.isLockHeld(server, session)) { + return new CachedTablet(RootTable.EXTENT, loc.getServerInstance(), TabletHostingGoal.ALWAYS, false); } else { - return new CachedTablet(RootTable.EXTENT, Optional.empty(), Optional.empty(), - TabletHostingGoal.ALWAYS, false); + return new CachedTablet(RootTable.EXTENT, Optional.empty(), TabletHostingGoal.ALWAYS, false); } } diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java index a063cff2413..e9925993e59 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java @@ -1308,7 +1308,7 @@ private void waitForTableStateTransition(TableId tableId, TableState expectedSta lastRow = tablet.getExtent().toMetaRow(); if (loc != null) { - serverCounts.increment(loc.getHostPortSession(), 1); + serverCounts.increment(loc.getServerInstance().getHostPortSessionGroup(), 1); } } diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 5f652b57ae9..3a627837997 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.file.rfile.RFile; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iteratorsImpl.system.DeletingIterator; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner; import org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher; @@ -724,6 +725,11 @@ public enum Property { PropertyType.TIMEDURATION, "The interval at which the TabletServer will check if on-demand tablets can be unloaded", "4.0.0"), + TSERV_GROUP_NAME("tserver.group", ServiceDescriptor.DEFAULT_GROUP_NAME, PropertyType.STRING, + "Optional group name that will be made available to the " + + "TablerServer plugins. Groups can be used to dedicate resources " + + " to specific tables (e.g. balancing tablets for table(s) within a group)", + "4.0.0"), // accumulo garbage collector properties GC_PREFIX("gc.", null, PropertyType.PREFIX, @@ -808,6 +814,11 @@ public enum Property { + " global setting to take effect. However, you must use the API or the shell" + " to change properties in zookeeper that are set on a table.", "1.3.5"), + TABLE_ASSIGNMENT_GROUP("table.assignment.group", ServiceDescriptor.DEFAULT_GROUP_NAME, + PropertyType.STRING, + "Tablets for this table will be assigned to TabletServers that have a corresponding" + + " tserver.group property value.", + "4.0.0"), TABLE_ARBITRARY_PROP_PREFIX("table.custom.", null, PropertyType.PREFIX, "Prefix to be used for user defined arbitrary properties.", "1.7.0"), TABLE_MINC_OUTPUT_DROP_CACHE("table.compaction.minor.output.drop.cache", "false", @@ -1095,7 +1106,8 @@ public enum Property { COMPACTOR_MAX_MESSAGE_SIZE("compactor.message.size.max", "10M", PropertyType.BYTES, "The maximum size of a message that can be sent to a tablet server.", "2.1.0"), @Experimental - COMPACTOR_QUEUE_NAME("compactor.queue", "", PropertyType.STRING, + // ELASTICITY_TODO: rename to compactor.group + COMPACTOR_QUEUE_NAME("compactor.queue", ServiceDescriptor.DEFAULT_GROUP_NAME, PropertyType.STRING, "The queue for which this Compactor will perform compactions", "3.0.0"), // CompactionCoordinator properties @Experimental diff --git a/core/src/main/java/org/apache/accumulo/core/lock/ServiceLockData.java b/core/src/main/java/org/apache/accumulo/core/lock/ServiceLockData.java index 746d5b5ce59..628db2c3580 100644 --- a/core/src/main/java/org/apache/accumulo/core/lock/ServiceLockData.java +++ b/core/src/main/java/org/apache/accumulo/core/lock/ServiceLockData.java @@ -70,10 +70,6 @@ public static class ServiceDescriptor { private final String address; private final String group; - public ServiceDescriptor(UUID uuid, ThriftService service, String address) { - this(uuid, service, address, DEFAULT_GROUP_NAME); - } - public ServiceDescriptor(UUID uuid, ThriftService service, String address, String group) { this.uuid = requireNonNull(uuid); this.service = requireNonNull(service); @@ -159,11 +155,6 @@ public ServiceLockData(UUID uuid, String address, ThriftService service, String Collections.singleton(new ServiceDescriptor(uuid, service, address, group))))); } - public ServiceLockData(UUID uuid, String address, ThriftService service) { - this(new ServiceDescriptors( - new HashSet<>(Collections.singleton(new ServiceDescriptor(uuid, service, address))))); - } - public String getAddressString(ThriftService service) { ServiceDescriptor sd = services.get(service); if (sd == null) { diff --git a/core/src/main/java/org/apache/accumulo/core/manager/balancer/TabletServerIdImpl.java b/core/src/main/java/org/apache/accumulo/core/manager/balancer/TabletServerIdImpl.java index 2fefb466d33..7da93ff3ea3 100644 --- a/core/src/main/java/org/apache/accumulo/core/manager/balancer/TabletServerIdImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/manager/balancer/TabletServerIdImpl.java @@ -35,9 +35,9 @@ public static TabletServerIdImpl fromThrift(TServerInstance tsi) { return (tsi == null) ? null : new TabletServerIdImpl(tsi); } - public TabletServerIdImpl(String host, int port, String session) { + public TabletServerIdImpl(String host, int port, String session, String group) { requireNonNull(host); - this.tServerInstance = new TServerInstance(HostAndPort.fromParts(host, port), session); + this.tServerInstance = new TServerInstance(HostAndPort.fromParts(host, port), session, group); } public TabletServerIdImpl(TServerInstance tServerInstance) { @@ -59,6 +59,11 @@ public String getSession() { return tServerInstance.getSession(); } + @Override + public String getGroup() { + return tServerInstance.getGroup(); + } + @Override public int compareTo(TabletServerId o) { return tServerInstance.compareTo(((TabletServerIdImpl) o).tServerInstance); @@ -96,7 +101,7 @@ public static TServerInstance toThrift(TabletServerId tabletServerId) { } else { return new TServerInstance( HostAndPort.fromParts(tabletServerId.getHost(), tabletServerId.getPort()), - tabletServerId.getSession()); + tabletServerId.getSession(), tabletServerId.getGroup()); } } } diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataCachedTabletObtainer.java b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataCachedTabletObtainer.java index 73ed8528f66..e65b455dfda 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataCachedTabletObtainer.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataCachedTabletObtainer.java @@ -220,8 +220,7 @@ public List lookupTablets(ClientContext context, String tserver, } public static CachedTablets getMetadataLocationEntries(SortedMap entries) { - Text location = null; - Text session = null; + TServerInstance tsi = null; TabletHostingGoal goal = null; boolean hostingRequested = false; @@ -238,8 +237,7 @@ public static CachedTablets getMetadataLocationEntries(SortedMap entr Value val = entry.getValue(); if (key.compareRow(lastRowFromKey) != 0) { - location = null; - session = null; + tsi = null; goal = null; hostingRequested = false; key.getRow(lastRowFromKey); @@ -251,11 +249,10 @@ public static CachedTablets getMetadataLocationEntries(SortedMap entr // interpret the row id as a key extent if (colf.equals(CurrentLocationColumnFamily.NAME) || colf.equals(FutureLocationColumnFamily.NAME)) { - if (location != null) { + if (tsi != null) { throw new IllegalStateException("Tablet has multiple locations : " + lastRowFromKey); } - location = new Text(val.toString()); - session = new Text(colq); + tsi = TServerInstance.fromString(val.toString()); } else if (HostingColumnFamily.GOAL_COLUMN.equals(colf, colq)) { goal = TabletHostingGoalUtil.fromValue(val); } else if (HostingColumnFamily.REQUESTED_COLUMN.equals(colf, colq)) { @@ -265,13 +262,12 @@ public static CachedTablets getMetadataLocationEntries(SortedMap entr if (ke.isMeta()) { goal = TabletHostingGoal.ALWAYS; } - if (location != null) { - results.add(new CachedTablet(ke, location.toString(), session.toString(), goal, - hostingRequested)); + if (tsi != null) { + results.add(new CachedTablet(ke, tsi, goal, hostingRequested)); } else { results.add(new CachedTablet(ke, goal, hostingRequested)); } - location = null; + tsi = null; } } diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java b/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java index 56aaa06a8ae..65fb3762430 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/TServerInstance.java @@ -20,6 +20,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import java.util.Objects; + import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.util.AddressUtil; import org.apache.hadoop.io.Text; @@ -34,39 +36,49 @@ */ public class TServerInstance implements Comparable { + private static final String FORMAT = "%s#%s#%s"; + private final HostAndPort hostAndPort; private final String hostPort; private final String session; - private final String hostPortSession; + private final String hostPortSessionGroup; + private final String group; + + public static TServerInstance fromString(String val) { + Objects.requireNonNull(val, "String value for TServerInstance cannot be null"); + String[] parts = val.split("#"); + if (parts.length != 3) { + // could throw IllegalArgumentException, but IllegalStateException is something + // that is already expected in a large portion of the codebase. + throw new IllegalStateException( + "Supplied TServerInstance string: " + val + " does not follow format: " + FORMAT); + } + return new TServerInstance(parts[0], parts[1], parts[2]); + } - public TServerInstance(HostAndPort address, String session) { + public TServerInstance(HostAndPort address, String session, String group) { this.hostAndPort = address; this.session = session; this.hostPort = hostAndPort.toString(); - this.hostPortSession = hostPort + "[" + session + "]"; + this.group = group; + this.hostPortSessionGroup = String.format(FORMAT, hostPort, session, group); } - public TServerInstance(String formattedString) { - int pos = formattedString.indexOf("["); - if (pos < 0 || !formattedString.endsWith("]")) { - throw new IllegalArgumentException(formattedString); - } - this.hostAndPort = HostAndPort.fromString(formattedString.substring(0, pos)); - this.session = formattedString.substring(pos + 1, formattedString.length() - 1); - this.hostPort = hostAndPort.toString(); - this.hostPortSession = hostPort + "[" + session + "]"; + public TServerInstance(String address, String session, String group) { + this(AddressUtil.parseAddress(address, false), session, group); } - public TServerInstance(HostAndPort address, long session) { - this(address, Long.toHexString(session)); + public TServerInstance(HostAndPort address, long session, String group) { + this(address, Long.toHexString(session), group); } - public TServerInstance(String address, long session) { - this(AddressUtil.parseAddress(address, false), Long.toHexString(session)); + public TServerInstance(String address, long session, String group) { + this(AddressUtil.parseAddress(address, false), Long.toHexString(session), group); } - public TServerInstance(Value address, Text session) { - this(AddressUtil.parseAddress(new String(address.get(), UTF_8), false), session.toString()); + public TServerInstance(Value address, Text session, String group) { + this(AddressUtil.parseAddress(new String(address.get(), UTF_8), false), session.toString(), + group); } @Override @@ -74,12 +86,12 @@ public int compareTo(TServerInstance other) { if (this == other) { return 0; } - return this.getHostPortSession().compareTo(other.getHostPortSession()); + return this.getHostPortSessionGroup().compareTo(other.getHostPortSessionGroup()); } @Override public int hashCode() { - return getHostPortSession().hashCode(); + return this.hostPortSessionGroup.hashCode(); } @Override @@ -92,11 +104,11 @@ public boolean equals(Object obj) { @Override public String toString() { - return hostPortSession; + return hostPortSessionGroup; } - public String getHostPortSession() { - return hostPortSession; + public String getHostPortSessionGroup() { + return hostPortSessionGroup; } public String getHost() { @@ -114,4 +126,8 @@ public HostAndPort getHostAndPort() { public String getSession() { return session; } + + public String getGroup() { + return group; + } } diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java index f4757919f39..0983e166217 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java @@ -54,6 +54,7 @@ import org.apache.accumulo.core.fate.zookeeper.ZooCache; import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.lock.ServiceLockData; +import org.apache.accumulo.core.lock.ServiceLockData.ThriftService; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.StoredTabletFile; @@ -150,10 +151,6 @@ public static class Location { private final TServerInstance tServerInstance; private final LocationType lt; - private Location(final String server, final String session, final LocationType lt) { - this(new TServerInstance(HostAndPort.fromString(server), session), lt); - } - private Location(final TServerInstance tServerInstance, final LocationType lt) { this.tServerInstance = Objects.requireNonNull(tServerInstance, "tServerInstance must not be null"); @@ -168,26 +165,6 @@ public TServerInstance getServerInstance() { return tServerInstance; } - public String getHostPortSession() { - return tServerInstance.getHostPortSession(); - } - - public String getHost() { - return tServerInstance.getHost(); - } - - public String getHostPort() { - return tServerInstance.getHostPort(); - } - - public HostAndPort getHostAndPort() { - return tServerInstance.getHostAndPort(); - } - - public String getSession() { - return tServerInstance.getSession(); - } - @Override public boolean equals(Object o) { if (this == o) { @@ -219,26 +196,14 @@ public static Location last(TServerInstance instance) { return new Location(instance, LocationType.LAST); } - public static Location last(final String server, final String session) { - return last(new TServerInstance(HostAndPort.fromString(server), session)); - } - public static Location current(TServerInstance instance) { return new Location(instance, LocationType.CURRENT); } - public static Location current(final String server, final String session) { - return current(new TServerInstance(HostAndPort.fromString(server), session)); - } - public static Location future(TServerInstance instance) { return new Location(instance, LocationType.FUTURE); } - public static Location future(final String server, final String session) { - return future(new TServerInstance(HostAndPort.fromString(server), session)); - } - } public TableId getTableId() { @@ -527,13 +492,13 @@ public static > TabletMetadata convertRow(Iterator BulkFileColumnFamily.getBulkLoadTid(val)); break; case CurrentLocationColumnFamily.STR_NAME: - te.setLocationOnce(val, qual, LocationType.CURRENT, suppressLocationError); + te.setLocationOnce(val, LocationType.CURRENT, suppressLocationError); break; case FutureLocationColumnFamily.STR_NAME: - te.setLocationOnce(val, qual, LocationType.FUTURE, suppressLocationError); + te.setLocationOnce(val, LocationType.FUTURE, suppressLocationError); break; case LastLocationColumnFamily.STR_NAME: - te.last = Location.last(val, qual); + te.last = Location.last(TServerInstance.fromString(val)); break; case SuspendLocationColumn.STR_NAME: te.suspend = SuspendingTServer.fromValue(kv.getValue()); @@ -586,15 +551,15 @@ public static > TabletMetadata convertRow(Iterator return te; } - private void setLocationOnce(String val, String qual, LocationType lt, boolean suppressError) { + private void setLocationOnce(String val, LocationType lt, boolean suppressError) { if (location != null) { if (!suppressError) { throw new IllegalStateException("Attempted to set second location for tableId: " + tableId - + " endrow: " + endRow + " -- " + location + " " + qual + " " + val); + + " endrow: " + endRow + " -- " + location + " " + val); } futureAndCurrentLocationSet = true; } - location = new Location(val, qual, lt); + location = new Location(TServerInstance.fromString(val), lt); } @VisibleForTesting @@ -638,9 +603,10 @@ private static Optional checkServer(ClientContext context, Stri if (sld.isPresent()) { log.trace("Checking server at ZK path = " + lockPath); - HostAndPort client = sld.orElseThrow().getAddress(ServiceLockData.ThriftService.TSERV); + HostAndPort client = sld.orElseThrow().getAddress(ThriftService.TSERV); + String group = sld.orElseThrow().getGroup(ThriftService.TSERV); if (client != null) { - server = Optional.of(new TServerInstance(client, stat.getEphemeralOwner())); + server = Optional.of(new TServerInstance(client, stat.getEphemeralOwner(), group)); } } return server; diff --git a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java index ce3a0c5ea4f..80fe3b760d8 100644 --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java @@ -50,18 +50,19 @@ public class MetricsUtil { private static List commonTags; public static void initializeMetrics(final AccumuloConfiguration conf, final String appName, - final HostAndPort address) throws ClassNotFoundException, InstantiationException, - IllegalAccessException, IllegalArgumentException, InvocationTargetException, - NoSuchMethodException, SecurityException { + final HostAndPort address, String groupName) throws ClassNotFoundException, + InstantiationException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException { initializeMetrics(conf.getBoolean(Property.GENERAL_MICROMETER_ENABLED), conf.getBoolean(Property.GENERAL_MICROMETER_JVM_METRICS_ENABLED), - conf.get(Property.GENERAL_MICROMETER_FACTORY), appName, address); + conf.get(Property.GENERAL_MICROMETER_FACTORY), appName, address, groupName); } private static void initializeMetrics(boolean enabled, boolean jvmMetricsEnabled, - String factoryClass, String appName, HostAndPort address) throws ClassNotFoundException, - InstantiationException, IllegalAccessException, IllegalArgumentException, - InvocationTargetException, NoSuchMethodException, SecurityException { + String factoryClass, String appName, HostAndPort address, String groupName) + throws ClassNotFoundException, InstantiationException, IllegalAccessException, + IllegalArgumentException, InvocationTargetException, NoSuchMethodException, + SecurityException { LOG.info("initializing metrics, enabled:{}, class:{}", enabled, factoryClass); @@ -84,6 +85,9 @@ private static void initializeMetrics(boolean enabled, boolean jvmMetricsEnabled tags.add(Tag.of("port", Integer.toString(address.getPort()))); } } + if (groupName != null) { + tags.add(Tag.of("group", groupName)); + } commonTags = Collections.unmodifiableList(tags); diff --git a/core/src/main/java/org/apache/accumulo/core/spi/balancer/GroupBalancer.java b/core/src/main/java/org/apache/accumulo/core/spi/balancer/GroupBalancer.java index 3527ba6f4c1..0ef8c3816cd 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/balancer/GroupBalancer.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/balancer/GroupBalancer.java @@ -131,11 +131,12 @@ public void getAssignments(AssignmentParameters params) { // Maintain locality String fakeSessionID = " "; TabletServerId simple = - new TabletServerIdImpl(last.getHost(), last.getPort(), fakeSessionID); + new TabletServerIdImpl(last.getHost(), last.getPort(), fakeSessionID, last.getGroup()); Iterator find = params.currentStatus().tailMap(simple).keySet().iterator(); if (find.hasNext()) { TabletServerId tserver = find.next(); - if (tserver.getHost().equals(last.getHost())) { + if (tserver.getHost().equals(last.getHost()) + && tserver.getGroup().equals(last.getGroup())) { params.addAssignment(entry.getKey(), tserver); continue; } diff --git a/core/src/main/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancer.java b/core/src/main/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancer.java index 50fcd8417fb..c81861b94b8 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancer.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancer.java @@ -95,11 +95,13 @@ public TabletServerId getAssignment(SortedMap loca if (last != null) { // Maintain locality String fakeSessionID = " "; - TabletServerId simple = new TabletServerIdImpl(last.getHost(), last.getPort(), fakeSessionID); + TabletServerId simple = + new TabletServerIdImpl(last.getHost(), last.getPort(), fakeSessionID, last.getGroup()); Iterator find = locations.tailMap(simple).keySet().iterator(); if (find.hasNext()) { TabletServerId current = find.next(); - if (current.getHost().equals(last.getHost())) { + if (current.getHost().equals(last.getHost()) + && current.getGroup().equals(last.getGroup())) { return current; } } diff --git a/core/src/main/java/org/apache/accumulo/core/spi/balancer/data/TabletServerId.java b/core/src/main/java/org/apache/accumulo/core/spi/balancer/data/TabletServerId.java index 6adcc803f17..8a96eb34e11 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/balancer/data/TabletServerId.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/balancer/data/TabletServerId.java @@ -27,4 +27,6 @@ public interface TabletServerId extends Comparable { int getPort(); String getSession(); + + String getGroup(); } diff --git a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java index 2bbcdbbabda..72b37f8f8ee 100644 --- a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java +++ b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java @@ -193,12 +193,14 @@ private TSummaryRequest getRequest() { // tablets // w/o a // location - .map(tm -> tm.getLocation().getHostPort()) // convert to host:port strings + .map(tm -> tm.getLocation().getServerInstance().getHostPort()) // convert to host:port + // strings .min(String::compareTo) // find minimum host:port .orElse(entry.getValue().stream().filter(tm -> tm.getLast() != null) // if no locations, // then look at last // locations - .map(tm -> tm.getLast().getHostPort()) // convert to host:port strings + .map(tm -> tm.getLast().getServerInstance().getHostPort()) // convert to host:port + // strings .min(String::compareTo).orElse(null)); // find minimum last location or return null if (location == null) { diff --git a/core/src/main/java/org/apache/accumulo/core/util/Interner.java b/core/src/main/java/org/apache/accumulo/core/util/Interner.java deleted file mode 100644 index 0f82d5185ae..00000000000 --- a/core/src/main/java/org/apache/accumulo/core/util/Interner.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.core.util; - -import java.lang.ref.WeakReference; -import java.util.WeakHashMap; - -/** - * A utility that mimics String.intern() for any immutable object type (including String). - */ -public class Interner { - private final WeakHashMap> internTable = new WeakHashMap<>(); - - public synchronized T intern(T item) { - WeakReference ref = internTable.get(item); - if (ref != null) { - T oldItem = ref.get(); - if (oldItem != null) { - return oldItem; - } - } - internTable.put(item, new WeakReference<>(item)); - return item; - } - - // for testing - synchronized int size() { - return internTable.size(); - } -} diff --git a/core/src/test/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImplTest.java b/core/src/test/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImplTest.java index 70860271bc6..ae054374ac5 100644 --- a/core/src/test/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImplTest.java +++ b/core/src/test/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImplTest.java @@ -23,7 +23,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; @@ -55,9 +54,11 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.MetadataCachedTabletObtainer; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; @@ -110,12 +111,12 @@ public RangeLocation(String location, KeyExtent extent1, List range1, Key } static RangeLocation createRangeLocation(String location, KeyExtent extent, List ranges) { - return new RangeLocation(location, extent, ranges); + return new RangeLocation(fixServerAddress(location), extent, ranges); } static RangeLocation createRangeLocation(String location, KeyExtent extent1, List range1, KeyExtent extent2, List range2) { - return new RangeLocation(location, extent1, range1, extent2, range2); + return new RangeLocation(fixServerAddress(location), extent1, range1, extent2, range2); } static Map>> @@ -138,7 +139,7 @@ static TreeMap createMetaCacheKE(Object... data) { for (int i = 0; i < data.length; i += 2) { KeyExtent ke = (KeyExtent) data[i]; String loc = (String) data[i + 1]; - mcke.put(ke, new CachedTablet(ke, loc, "1", TabletHostingGoal.ONDEMAND, false)); + mcke.put(ke, new CachedTablet(ke, createTSI(loc, "1"), TabletHostingGoal.ONDEMAND, false)); } return mcke; @@ -343,7 +344,7 @@ public ServerExtent(String location, String row, KeyExtent extent) { } static ServerExtent createServerExtent(String row, String location, KeyExtent extent) { - return new ServerExtent(location, row, extent); + return new ServerExtent(fixServerAddress(location), row, extent); } static Map>> createServerExtentMap(ServerExtent... locations) { @@ -606,8 +607,9 @@ static class TestRootClientTabletCache extends RootClientTabletCache { @Override protected CachedTablet getRootTabletLocation(ClientContext context) { - return new CachedTablet(RootTable.EXTENT, context.getRootTabletLocation(), "1", - TabletHostingGoal.ALWAYS, false); + TServerInstance rootTabletLocation = createTSI(context.getRootTabletLocation(), "1"); + return new CachedTablet(RootTable.EXTENT, rootTabletLocation, TabletHostingGoal.ALWAYS, + false); } @Override @@ -617,7 +619,7 @@ public void invalidateCache(ClientContext context, String server) {} static void createEmptyTablet(TServers tservers, String server, KeyExtent tablet) { Map> tablets = - tservers.tservers.computeIfAbsent(server, k -> new HashMap<>()); + tservers.tservers.computeIfAbsent(fixServerAddress(server), k -> new HashMap<>()); SortedMap tabletData = tablets.computeIfAbsent(tablet, k -> new TreeMap<>()); if (!tabletData.isEmpty()) { throw new IllegalStateException("Asked for empty tablet, but non empty tablet exists"); @@ -642,10 +644,19 @@ static void clearLocation(TServers tservers, String server, KeyExtent tablet, Ke } + static String fixServerAddress(String server) { + return (server.contains(":") ? server : server + ":9997"); + } + + static TServerInstance createTSI(String server, String session) { + return new TServerInstance(fixServerAddress(server), session, + ServiceDescriptor.DEFAULT_GROUP_NAME); + } + static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke, String location, String instance) { Map> tablets = - tservers.tservers.computeIfAbsent(server, k -> new HashMap<>()); + tservers.tservers.computeIfAbsent(fixServerAddress(server), k -> new HashMap<>()); SortedMap tabletData = tablets.computeIfAbsent(tablet, k -> new TreeMap<>()); Text mr = ke.toMetaRow(); @@ -655,8 +666,8 @@ static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyE if (instance == null) { instance = ""; } - Key lk = new Key(mr, CurrentLocationColumnFamily.NAME, new Text(instance)); - tabletData.put(lk, new Value(location)); + Key lk = new Key(mr, CurrentLocationColumnFamily.NAME); + tabletData.put(lk, new Value(createTSI(location, instance).toString())); } Key hk = new Key(mr, HostingColumnFamily.GOAL_COLUMN.getColumnFamily(), @@ -671,11 +682,11 @@ static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyE static void deleteLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke, String instance) { Map> tablets = - tservers.tservers.computeIfAbsent(server, k -> new HashMap<>()); + tservers.tservers.computeIfAbsent(fixServerAddress(server), k -> new HashMap<>()); SortedMap tabletData = tablets.computeIfAbsent(tablet, k -> new TreeMap<>()); Text mr = ke.toMetaRow(); - Key lk = new Key(mr, CurrentLocationColumnFamily.NAME, new Text(instance)); + Key lk = new Key(mr, CurrentLocationColumnFamily.NAME); tabletData.remove(lk); } @@ -685,12 +696,13 @@ static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyE } static void deleteServer(TServers tservers, String server) { - tservers.tservers.remove(server); + tservers.tservers.remove(fixServerAddress(server)); } private void locateTabletTest(ClientTabletCacheImpl cache, String row, boolean skipRow, KeyExtent expected, String server, LocationNeed locationNeeded) throws Exception { + CachedTablet tl = cache.findTablet(context, new Text(row), skipRow, locationNeeded); if (expected == null) { @@ -704,7 +716,7 @@ private void locateTabletTest(ClientTabletCacheImpl cache, String row, boolean s assertTrue(tl.getTserverLocation().isEmpty()); assertTrue(tl.getTserverSession().isEmpty()); } else { - assertEquals(server, tl.getTserverLocation().orElseThrow()); + assertEquals(fixServerAddress(server), tl.getTserverLocation().orElseThrow()); } assertEquals(expected, tl.getExtent()); } @@ -774,7 +786,7 @@ public void test1() throws Exception { // simulate a server failure setLocation(tservers, "tserver2", METADATA_TABLE_EXTENT, tab1e21, "tserver9"); - tab1TabletCache.invalidateCache(context, "tserver8"); + tab1TabletCache.invalidateCache(context, "tserver8:9997"); locateTabletTest(tab1TabletCache, "r1", tab1e22, "tserver6"); locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver9"); locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver4"); @@ -782,9 +794,9 @@ public void test1() throws Exception { // simulate all servers failing deleteServer(tservers, "tserver1"); deleteServer(tservers, "tserver2"); - tab1TabletCache.invalidateCache(context, "tserver4"); - tab1TabletCache.invalidateCache(context, "tserver6"); - tab1TabletCache.invalidateCache(context, "tserver9"); + tab1TabletCache.invalidateCache(context, "tserver4:9997"); + tab1TabletCache.invalidateCache(context, "tserver6:9997"); + tab1TabletCache.invalidateCache(context, "tserver9:9997"); locateTabletTest(tab1TabletCache, "r1", null, null); locateTabletTest(tab1TabletCache, "h", null, null); @@ -838,7 +850,7 @@ public void test1() throws Exception { // simulate metadata and regular server down and the reassigned deleteServer(tservers, "tserver5"); - tab1TabletCache.invalidateCache(context, "tserver7"); + tab1TabletCache.invalidateCache(context, "tserver7:9997"); locateTabletTest(tab1TabletCache, "a", null, null); locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver8"); locateTabletTest(tab1TabletCache, "r", tab1e22, "tserver9"); @@ -850,7 +862,7 @@ public void test1() throws Exception { locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver7"); locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver8"); locateTabletTest(tab1TabletCache, "r", tab1e22, "tserver9"); - tab1TabletCache.invalidateCache(context, "tserver7"); + tab1TabletCache.invalidateCache(context, "tserver7:9997"); setLocation(tservers, "tserver10", mte1, tab1e1, "tserver2"); locateTabletTest(tab1TabletCache, "a", tab1e1, "tserver2"); locateTabletTest(tab1TabletCache, "h", tab1e21, "tserver8"); @@ -1230,13 +1242,14 @@ public void testBinRangesNonContiguousExtentsAndMultipleRanges() throws Exceptio @Test public void testIsContiguous() { - CachedTablet e1 = new CachedTablet(createNewKeyExtent("foo", "1", null), "l1", "1", + TServerInstance l1 = createTSI("l1", "1"); + CachedTablet e1 = new CachedTablet(createNewKeyExtent("foo", "1", null), l1, TabletHostingGoal.ONDEMAND, false); - CachedTablet e2 = new CachedTablet(createNewKeyExtent("foo", "2", "1"), "l1", "1", + CachedTablet e2 = new CachedTablet(createNewKeyExtent("foo", "2", "1"), l1, TabletHostingGoal.ONDEMAND, false); - CachedTablet e3 = new CachedTablet(createNewKeyExtent("foo", "3", "2"), "l1", "1", + CachedTablet e3 = new CachedTablet(createNewKeyExtent("foo", "3", "2"), l1, TabletHostingGoal.ONDEMAND, false); - CachedTablet e4 = new CachedTablet(createNewKeyExtent("foo", null, "3"), "l1", "1", + CachedTablet e4 = new CachedTablet(createNewKeyExtent("foo", null, "3"), l1, TabletHostingGoal.ONDEMAND, false); assertTrue(ClientTabletCacheImpl.isContiguous(List.of(e1, e2, e3, e4))); @@ -1250,7 +1263,7 @@ public void testIsContiguous() { assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e1, e2, e4))); assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e1, e3, e4))); - CachedTablet e5 = new CachedTablet(createNewKeyExtent("foo", null, null), "l1", "1", + CachedTablet e5 = new CachedTablet(createNewKeyExtent("foo", null, null), l1, TabletHostingGoal.ONDEMAND, false); assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e1, e2, e3, e4, e5))); assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e5, e1, e2, e3, e4))); @@ -1258,12 +1271,12 @@ public void testIsContiguous() { assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e5, e2, e3, e4))); assertTrue(ClientTabletCacheImpl.isContiguous(List.of(e5))); - CachedTablet e6 = new CachedTablet(createNewKeyExtent("foo", null, "1"), "l1", "1", + CachedTablet e6 = new CachedTablet(createNewKeyExtent("foo", null, "1"), l1, TabletHostingGoal.ONDEMAND, false); assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e1, e2, e3, e6))); - CachedTablet e7 = new CachedTablet(createNewKeyExtent("foo", "33", "11"), "l1", "1", + CachedTablet e7 = new CachedTablet(createNewKeyExtent("foo", "33", "11"), l1, TabletHostingGoal.ONDEMAND, false); assertFalse(ClientTabletCacheImpl.isContiguous(List.of(e1, e2, e7, e4))); @@ -1564,25 +1577,6 @@ public void testBug3() throws Exception { } - @Test - public void testAccumulo1248() { - TServers tservers = new TServers(); - ClientTabletCacheImpl metaCache = createLocators(tservers, "tserver1", "tserver2", "foo"); - - KeyExtent ke1 = createNewKeyExtent("foo", null, null); - - // set two locations for a tablet, this is not supposed to happen. The metadata cache should - // throw an exception if it sees this rather than caching one of - // the locations. - setLocation(tservers, "tserver2", METADATA_TABLE_EXTENT, ke1, "L1", "I1"); - setLocation(tservers, "tserver2", METADATA_TABLE_EXTENT, ke1, "L2", "I2"); - - var e = assertThrows(IllegalStateException.class, - () -> metaCache.findTablet(context, new Text("a"), false, LocationNeed.REQUIRED)); - assertTrue(e.getMessage().startsWith("Tablet has multiple locations : ")); - - } - @Test public void testLostLock() throws Exception { @@ -1603,7 +1597,7 @@ public void invalidateCache(String server) {} KeyExtent ke1 = createNewKeyExtent("foo", null, null); setLocation(tservers, "tserver2", METADATA_TABLE_EXTENT, ke1, "L1", "5"); - activeLocks.add("L1:5"); + activeLocks.add("L1:9997:5"); locateTabletTest(metaCache, "a", ke1, "L1"); locateTabletTest(metaCache, "a", ke1, "L1"); @@ -1617,7 +1611,7 @@ public void invalidateCache(String server) {} clearLocation(tservers, "tserver2", METADATA_TABLE_EXTENT, ke1, "5"); setLocation(tservers, "tserver2", METADATA_TABLE_EXTENT, ke1, "L2", "6"); - activeLocks.add("L2:6"); + activeLocks.add("L2:9997:6"); locateTabletTest(metaCache, "a", ke1, "L2"); locateTabletTest(metaCache, "a", ke1, "L2"); @@ -1635,7 +1629,7 @@ public void invalidateCache(String server) {} locateTabletTest(metaCache, "a", null, null); locateTabletTest(metaCache, "a", null, null); - activeLocks.add("L3:7"); + activeLocks.add("L3:9997:7"); locateTabletTest(metaCache, "a", ke1, "L3"); locateTabletTest(metaCache, "a", ke1, "L3"); @@ -1665,12 +1659,12 @@ public void invalidateCache(String server) {} runTest(metaCache, ml, emb, "a", "w"); - activeLocks.add("L1:8"); + activeLocks.add("L1:9997:8"); emb = createServerExtentMap(createServerExtent("a", "L1", ke11)); runTest(metaCache, ml, emb, "w"); - activeLocks.add("L2:9"); + activeLocks.add("L2:9997:9"); emb = createServerExtentMap(createServerExtent("a", "L1", ke11), createServerExtent("w", "L2", ke12)); @@ -1687,7 +1681,7 @@ public void invalidateCache(String server) {} runTest(ranges, metaCache, expected); - activeLocks.remove("L2:9"); + activeLocks.remove("L2:9997:9"); expected = createExpectedBinnings(createRangeLocation("L1", ke11, createNewRangeList(new Range("a")))); @@ -1708,14 +1702,14 @@ public void invalidateCache(String server) {} runTest(ranges, metaCache, expected, createNewRangeList(new Range("a"), createNewRange("b", "o"), createNewRange("r", "z"))); - activeLocks.add("L3:10"); + activeLocks.add("L3:9997:10"); expected = createExpectedBinnings(createRangeLocation("L3", ke11, createNewRangeList(new Range("a")))); runTest(ranges, metaCache, expected, createNewRangeList(createNewRange("b", "o"), createNewRange("r", "z"))); - activeLocks.add("L4:11"); + activeLocks.add("L4:9997:11"); expected = createExpectedBinnings( createRangeLocation("L3", ke11, @@ -1759,7 +1753,7 @@ public void testCachingUnhosted() throws Exception { assertEquals(List.of(), failures); var tl1 = new CachedTablet(ke1, TabletHostingGoal.ONDEMAND, false); var tl2 = new CachedTablet(ke2, TabletHostingGoal.ONDEMAND, false); - var tl3 = new CachedTablet(ke3, "L2", "I2", TabletHostingGoal.ONDEMAND, false); + var tl3 = new CachedTablet(ke3, createTSI("L2", "I2"), TabletHostingGoal.ONDEMAND, false); var expected = Set.of(new Pair<>(tl1, r1), new Pair<>(tl1, r2), new Pair<>(tl2, r2), new Pair<>(tl3, r2)); assertEquals(expected, actual); @@ -1786,7 +1780,7 @@ public void testCachingUnhosted() throws Exception { failures = metaCache.findTablets(context, ranges, (tl, r) -> actual.add(new Pair<>(tl, r)), LocationNeed.NOT_REQUIRED); assertEquals(List.of(), failures); - tl1 = new CachedTablet(ke1, "L3", "I3", TabletHostingGoal.ONDEMAND, false); + tl1 = new CachedTablet(ke1, createTSI("L3", "I3"), TabletHostingGoal.ONDEMAND, false); expected = Set.of(new Pair<>(tl1, r1), new Pair<>(tl1, r2), new Pair<>(tl2, r2), new Pair<>(tl3, r2)); assertEquals(expected, actual); @@ -1795,7 +1789,7 @@ public void testCachingUnhosted() throws Exception { failures = metaCache.findTablets(context, ranges, (tl, r) -> actual.add(new Pair<>(tl, r)), LocationNeed.REQUIRED); assertEquals(List.of(), failures); - tl2 = new CachedTablet(ke2, "L4", "I4", TabletHostingGoal.ONDEMAND, false); + tl2 = new CachedTablet(ke2, createTSI("L4", "I4"), TabletHostingGoal.ONDEMAND, false); expected = Set.of(new Pair<>(tl1, r1), new Pair<>(tl1, r2), new Pair<>(tl2, r2), new Pair<>(tl3, r2)); assertEquals(expected, actual); diff --git a/core/src/test/java/org/apache/accumulo/core/lock/ServiceLockDataTest.java b/core/src/test/java/org/apache/accumulo/core/lock/ServiceLockDataTest.java index 5f89af57b17..74adb89c262 100644 --- a/core/src/test/java/org/apache/accumulo/core/lock/ServiceLockDataTest.java +++ b/core/src/test/java/org/apache/accumulo/core/lock/ServiceLockDataTest.java @@ -40,7 +40,8 @@ public class ServiceLockDataTest { @Test public void testSingleServiceConstructor() throws Exception { - ServiceLockData ss = new ServiceLockData(serverUUID, "127.0.0.1", ThriftService.TSERV); + ServiceLockData ss = new ServiceLockData(serverUUID, "127.0.0.1", ThriftService.TSERV, + ServiceDescriptor.DEFAULT_GROUP_NAME); assertEquals(serverUUID, ss.getServerUUID(ThriftService.TSERV)); assertEquals("127.0.0.1", ss.getAddressString(ThriftService.TSERV)); assertThrows(IllegalArgumentException.class, () -> ss.getAddress(ThriftService.TSERV)); @@ -54,8 +55,10 @@ public void testSingleServiceConstructor() throws Exception { @Test public void testMultipleServiceConstructor() throws Exception { ServiceDescriptors sds = new ServiceDescriptors(); - sds.addService(new ServiceDescriptor(serverUUID, ThriftService.TSERV, "127.0.0.1:9997")); - sds.addService(new ServiceDescriptor(serverUUID, ThriftService.TABLET_SCAN, "127.0.0.1:9998")); + sds.addService(new ServiceDescriptor(serverUUID, ThriftService.TSERV, "127.0.0.1:9997", + ServiceDescriptor.DEFAULT_GROUP_NAME)); + sds.addService(new ServiceDescriptor(serverUUID, ThriftService.TABLET_SCAN, "127.0.0.1:9998", + ServiceDescriptor.DEFAULT_GROUP_NAME)); ServiceLockData ss = new ServiceLockData(sds); assertEquals(serverUUID, ss.getServerUUID(ThriftService.TSERV)); assertEquals("127.0.0.1:9997", ss.getAddressString(ThriftService.TSERV)); diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java index 43e6a0e5e0d..0958025523d 100644 --- a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java +++ b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java @@ -47,6 +47,7 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.fate.FateTxId; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.SuspendingTServer; import org.apache.accumulo.core.metadata.TServerInstance; @@ -97,9 +98,11 @@ public void testAllColumns() { mutation.at().family(DataFileColumnFamily.NAME).qualifier(tf2.getMetaUpdateDelete()) .put(dfv2.encode()); - mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("s001").put("server1:8555"); + mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server1:8555", "s001", "default").toString()); - mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555"); + mutation.at().family(LastLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server2:8555", "s000", "default").toString()); LogEntry le1 = new LogEntry(extent, 55, "lf1"); mutation.at().family(le1.getColumnFamily()).qualifier(le1.getColumnQualifier()) @@ -129,12 +132,14 @@ public void testAllColumns() { assertEquals(rowMap, tm.getKeyValues()); assertEquals(Map.of(new StoredTabletFile(bf1), 56L, new StoredTabletFile(bf2), 59L), tm.getLoaded()); - assertEquals(HostAndPort.fromParts("server1", 8555), tm.getLocation().getHostAndPort()); - assertEquals("s001", tm.getLocation().getSession()); + assertEquals(HostAndPort.fromParts("server1", 8555), + tm.getLocation().getServerInstance().getHostAndPort()); + assertEquals("s001", tm.getLocation().getServerInstance().getSession()); assertEquals(LocationType.CURRENT, tm.getLocation().getType()); assertTrue(tm.hasCurrent()); - assertEquals(HostAndPort.fromParts("server2", 8555), tm.getLast().getHostAndPort()); - assertEquals("s000", tm.getLast().getSession()); + assertEquals(HostAndPort.fromParts("server2", 8555), + tm.getLast().getServerInstance().getHostAndPort()); + assertEquals("s000", tm.getLast().getServerInstance().getSession()); assertEquals(LocationType.LAST, tm.getLast().getType()); assertEquals(Set.of(le1.getValue() + " " + le1.timestamp, le2.getValue() + " " + le2.timestamp), tm.getLogs().stream().map(le -> le.getValue() + " " + le.timestamp).collect(toSet())); @@ -150,7 +155,8 @@ public void testFuture() { KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new Text("da")); Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent); - mutation.at().family(FutureLocationColumnFamily.NAME).qualifier("s001").put("server1:8555"); + mutation.at().family(FutureLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server1:8555", "s001", "default").toString()); SortedMap rowMap = toRowMap(mutation); @@ -158,8 +164,9 @@ public void testFuture() { EnumSet.allOf(ColumnType.class), false, false); assertEquals(extent, tm.getExtent()); - assertEquals(HostAndPort.fromParts("server1", 8555), tm.getLocation().getHostAndPort()); - assertEquals("s001", tm.getLocation().getSession()); + assertEquals(HostAndPort.fromParts("server1", 8555), + tm.getLocation().getServerInstance().getHostAndPort()); + assertEquals("s001", tm.getLocation().getServerInstance().getSession()); assertEquals(LocationType.FUTURE, tm.getLocation().getType()); assertFalse(tm.hasCurrent()); } @@ -169,8 +176,10 @@ public void testFutureAndCurrent() { KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new Text("da")); Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent); - mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("s001").put("server1:8555"); - mutation.at().family(FutureLocationColumnFamily.NAME).qualifier("s001").put("server1:8555"); + mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server1:8555", "s001", "default").toString()); + mutation.at().family(FutureLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server1:8555", "s001", "default").toString()); SortedMap rowMap = toRowMap(mutation); @@ -185,9 +194,12 @@ public void testFutureAndCurrent() { @Test public void testLocationStates() { KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new Text("da")); - TServerInstance ser1 = new TServerInstance(HostAndPort.fromParts("server1", 8555), "s001"); - TServerInstance ser2 = new TServerInstance(HostAndPort.fromParts("server2", 8111), "s002"); - TServerInstance deadSer = new TServerInstance(HostAndPort.fromParts("server3", 8000), "s003"); + TServerInstance ser1 = new TServerInstance(HostAndPort.fromParts("server1", 8555), "s001", + ServiceDescriptor.DEFAULT_GROUP_NAME); + TServerInstance ser2 = new TServerInstance(HostAndPort.fromParts("server2", 8111), "s002", + ServiceDescriptor.DEFAULT_GROUP_NAME); + TServerInstance deadSer = new TServerInstance(HostAndPort.fromParts("server3", 8000), "s003", + ServiceDescriptor.DEFAULT_GROUP_NAME); Set tservers = new LinkedHashSet<>(); tservers.add(ser1); tservers.add(ser2); @@ -196,8 +208,7 @@ public void testLocationStates() { // test assigned Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent); - mutation.at().family(FutureLocationColumnFamily.NAME).qualifier(ser1.getSession()) - .put(ser1.getHostPort()); + mutation.at().family(FutureLocationColumnFamily.NAME).qualifier("").put(ser1.toString()); SortedMap rowMap = toRowMap(mutation); TabletMetadata tm = @@ -206,35 +217,33 @@ public void testLocationStates() { assertEquals(TabletState.ASSIGNED, state); assertEquals(ser1, tm.getLocation().getServerInstance()); - assertEquals(ser1.getSession(), tm.getLocation().getSession()); + assertEquals(ser1.getSession(), tm.getLocation().getServerInstance().getSession()); assertEquals(LocationType.FUTURE, tm.getLocation().getType()); assertFalse(tm.hasCurrent()); // test hosted mutation = TabletColumnFamily.createPrevRowMutation(extent); - mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier(ser2.getSession()) - .put(ser2.getHostPort()); + mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("").put(ser2.toString()); rowMap = toRowMap(mutation); tm = TabletMetadata.convertRow(rowMap.entrySet().iterator(), colsToFetch, false, false); assertEquals(TabletState.HOSTED, tm.getTabletState(tservers)); assertEquals(ser2, tm.getLocation().getServerInstance()); - assertEquals(ser2.getSession(), tm.getLocation().getSession()); + assertEquals(ser2.getSession(), tm.getLocation().getServerInstance().getSession()); assertEquals(LocationType.CURRENT, tm.getLocation().getType()); assertTrue(tm.hasCurrent()); // test ASSIGNED_TO_DEAD_SERVER mutation = TabletColumnFamily.createPrevRowMutation(extent); - mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier(deadSer.getSession()) - .put(deadSer.getHostPort()); + mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("").put(deadSer.toString()); rowMap = toRowMap(mutation); tm = TabletMetadata.convertRow(rowMap.entrySet().iterator(), colsToFetch, false, false); assertEquals(TabletState.ASSIGNED_TO_DEAD_SERVER, tm.getTabletState(tservers)); assertEquals(deadSer, tm.getLocation().getServerInstance()); - assertEquals(deadSer.getSession(), tm.getLocation().getSession()); + assertEquals(deadSer.getSession(), tm.getLocation().getServerInstance().getSession()); assertEquals(LocationType.CURRENT, tm.getLocation().getType()); assertTrue(tm.hasCurrent()); diff --git a/core/src/test/java/org/apache/accumulo/core/spi/balancer/BaseHostRegexTableLoadBalancerTest.java b/core/src/test/java/org/apache/accumulo/core/spi/balancer/BaseHostRegexTableLoadBalancerTest.java index e00350069ab..ad2e9cb342c 100644 --- a/core/src/test/java/org/apache/accumulo/core/spi/balancer/BaseHostRegexTableLoadBalancerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/spi/balancer/BaseHostRegexTableLoadBalancerTest.java @@ -31,6 +31,7 @@ import org.apache.accumulo.core.data.TabletId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.TabletIdImpl; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.balancer.TServerStatusImpl; import org.apache.accumulo.core.manager.balancer.TableStatisticsImpl; import org.apache.accumulo.core.manager.balancer.TabletServerIdImpl; @@ -78,6 +79,11 @@ public TableId getId() { TestSimpleBalancer.class.getName()); } + protected static TabletServerIdImpl createTSID(String host) { + return new TabletServerIdImpl(host, 9997, Integer.toHexString(1), + ServiceDescriptor.DEFAULT_GROUP_NAME); + } + protected static final TestTable FOO = new TestTable("foo", TableId.of("1")); protected static final TestTable BAR = new TestTable("bar", TableId.of("2")); protected static final TestTable BAZ = new TestTable("baz", TableId.of("3")); @@ -126,43 +132,40 @@ public List getOnlineTabletsForTable(TabletServerId tserver, servers.put("192.168.0.14", "r03s04"); servers.put("192.168.0.15", "r03s05"); - allTabletServers.put(new TabletServerIdImpl("192.168.0.1", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.1"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.2", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.2"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.3", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.3"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.4", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.4"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.5", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.5"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.6", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.6"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.7", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.7"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.8", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.8"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.9", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.9"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.10", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.10"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.11", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.11"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.12", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.12"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.13", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.13"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.14", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.14"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - allTabletServers.put(new TabletServerIdImpl("192.168.0.15", 9997, Integer.toHexString(1)), + allTabletServers.put(createTSID("192.168.0.15"), new TServerStatusImpl(new org.apache.accumulo.core.manager.thrift.TabletServerStatus())); - initialTableLocation.put(FOO.getTableName(), - new TabletServerIdImpl("192.168.0.1", 9997, Integer.toHexString(1))); - initialTableLocation.put(BAR.getTableName(), - new TabletServerIdImpl("192.168.0.6", 9997, Integer.toHexString(1))); - initialTableLocation.put(BAZ.getTableName(), - new TabletServerIdImpl("192.168.0.11", 9997, Integer.toHexString(1))); + initialTableLocation.put(FOO.getTableName(), createTSID("192.168.0.1")); + initialTableLocation.put(BAR.getTableName(), createTSID("192.168.0.6")); + initialTableLocation.put(BAZ.getTableName(), createTSID("192.168.0.11")); tableTablets.put(FOO.getTableName(), new ArrayList<>()); tableTablets.get(FOO.getTableName()) @@ -255,7 +258,7 @@ protected SortedMap createCurrent(int numTservers) tableMap.put(BAR.getId().canonical(), new TableStatisticsImpl(new TableInfo())); tableMap.put(BAZ.getId().canonical(), new TableStatisticsImpl(new TableInfo())); status.setTableMap(tableMap); - current.put(new TabletServerIdImpl(base + i, 9997, Integer.toHexString(1)), status); + current.put(createTSID(base + i), status); } // now put all of the tablets on one server for (Map.Entry entry : initialTableLocation.entrySet()) { diff --git a/core/src/test/java/org/apache/accumulo/core/spi/balancer/GroupBalancerTest.java b/core/src/test/java/org/apache/accumulo/core/spi/balancer/GroupBalancerTest.java index 4320d58968c..3fdc5fb57ea 100644 --- a/core/src/test/java/org/apache/accumulo/core/spi/balancer/GroupBalancerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/spi/balancer/GroupBalancerTest.java @@ -37,6 +37,7 @@ import org.apache.accumulo.core.data.TabletId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.TabletIdImpl; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.balancer.BalanceParamsImpl; import org.apache.accumulo.core.manager.balancer.TServerStatusImpl; import org.apache.accumulo.core.manager.balancer.TabletServerIdImpl; @@ -67,14 +68,17 @@ public void addTservers(String... locs) { } public void addTserver(String host, int port) { - tservers.add(new TabletServerIdImpl(host, port, Long.toHexString(6))); + tservers.add(new TabletServerIdImpl(host, port, Long.toHexString(6), + ServiceDescriptor.DEFAULT_GROUP_NAME)); } public void addTablet(String er, String host, int port) { - TabletServerId tsi = new TabletServerIdImpl(host, port, Long.toHexString(6)); + TabletServerId tsi = new TabletServerIdImpl(host, port, Long.toHexString(6), + ServiceDescriptor.DEFAULT_GROUP_NAME); tabletLocs.put( new TabletIdImpl(new KeyExtent(TableId.of("b"), er == null ? null : new Text(er), null)), - new TabletServerIdImpl(host, port, Long.toHexString(6))); + new TabletServerIdImpl(host, port, Long.toHexString(6), + ServiceDescriptor.DEFAULT_GROUP_NAME)); tservers.add(tsi); } diff --git a/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java b/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java index c70841e9821..0f4887a6696 100644 --- a/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java @@ -46,7 +46,6 @@ import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent; import org.apache.accumulo.core.manager.balancer.AssignmentParamsImpl; import org.apache.accumulo.core.manager.balancer.BalanceParamsImpl; -import org.apache.accumulo.core.manager.balancer.TabletServerIdImpl; import org.apache.accumulo.core.manager.balancer.TabletStatisticsImpl; import org.apache.accumulo.core.spi.balancer.data.TServerStatus; import org.apache.accumulo.core.spi.balancer.data.TabletMigration; @@ -163,42 +162,27 @@ public void testSplitCurrentByRegexUsingHostname() { assertTrue(groups.containsKey(FOO.getTableName())); SortedMap fooHosts = groups.get(FOO.getTableName()); assertEquals(5, fooHosts.size()); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.1", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.2", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.3", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.4", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.5", 9997, Integer.toHexString(1)))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.1"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.2"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.3"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.4"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.5"))); assertTrue(groups.containsKey(BAR.getTableName())); SortedMap barHosts = groups.get(BAR.getTableName()); assertEquals(5, barHosts.size()); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.6", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.7", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.8", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.9", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.10", 9997, Integer.toHexString(1)))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.6"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.7"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.8"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.9"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.10"))); assertTrue(groups.containsKey(DEFAULT_POOL)); SortedMap defHosts = groups.get(DEFAULT_POOL); assertEquals(5, defHosts.size()); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.11", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.12", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.13", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.14", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.15", 9997, Integer.toHexString(1)))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.11"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.12"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.13"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.14"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.15"))); } @Test @@ -217,61 +201,36 @@ public void testSplitCurrentByRegexUsingOverlappingPools() { for (String pool : new String[] {FOO.getTableName(), DEFAULT_POOL}) { SortedMap fooHosts = groups.get(pool); assertEquals(15, fooHosts.size()); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.1", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.2", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.3", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.4", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.5", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.6", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.7", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.8", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.9", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.10", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.11", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.12", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.13", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.14", 9997, Integer.toHexString(1)))); - assertTrue(fooHosts - .containsKey(new TabletServerIdImpl("192.168.0.15", 9997, Integer.toHexString(1)))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.1"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.2"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.3"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.4"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.5"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.6"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.7"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.8"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.9"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.10"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.11"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.12"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.13"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.14"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.15"))); } assertTrue(groups.containsKey(BAR.getTableName())); SortedMap barHosts = groups.get(BAR.getTableName()); assertEquals(10, barHosts.size()); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.1", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.2", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.3", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.4", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.5", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.6", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.7", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.8", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.9", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.10", 9997, Integer.toHexString(1)))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.1"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.2"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.3"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.4"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.5"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.6"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.7"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.8"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.9"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.10"))); } @Test @@ -292,42 +251,27 @@ public void testSplitCurrentByRegexUsingIP() { assertTrue(groups.containsKey(FOO.getTableName())); SortedMap fooHosts = groups.get(FOO.getTableName()); assertEquals(5, fooHosts.size()); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.1", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.2", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.3", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.4", 9997, Integer.toHexString(1)))); - assertTrue( - fooHosts.containsKey(new TabletServerIdImpl("192.168.0.5", 9997, Integer.toHexString(1)))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.1"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.2"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.3"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.4"))); + assertTrue(fooHosts.containsKey(createTSID("192.168.0.5"))); assertTrue(groups.containsKey(BAR.getTableName())); SortedMap barHosts = groups.get(BAR.getTableName()); assertEquals(5, barHosts.size()); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.6", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.7", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.8", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.9", 9997, Integer.toHexString(1)))); - assertTrue( - barHosts.containsKey(new TabletServerIdImpl("192.168.0.10", 9997, Integer.toHexString(1)))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.6"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.7"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.8"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.9"))); + assertTrue(barHosts.containsKey(createTSID("192.168.0.10"))); assertTrue(groups.containsKey(DEFAULT_POOL)); SortedMap defHosts = groups.get(DEFAULT_POOL); assertEquals(5, defHosts.size()); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.11", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.12", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.13", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.14", 9997, Integer.toHexString(1)))); - assertTrue( - defHosts.containsKey(new TabletServerIdImpl("192.168.0.15", 9997, Integer.toHexString(1)))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.11"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.12"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.13"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.14"))); + assertTrue(defHosts.containsKey(createTSID("192.168.0.15"))); } @Test diff --git a/core/src/test/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancerTest.java b/core/src/test/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancerTest.java index 9b7c2b7a56f..c58e3ceb38b 100644 --- a/core/src/test/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancerTest.java @@ -37,6 +37,7 @@ import org.apache.accumulo.core.data.TabletId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.TabletIdImpl; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.balancer.BalanceParamsImpl; import org.apache.accumulo.core.manager.balancer.TServerStatusImpl; import org.apache.accumulo.core.manager.balancer.TabletServerIdImpl; @@ -103,9 +104,15 @@ public void setUp() { @Test public void testAssignMigrations() { - servers.put(new TabletServerIdImpl("127.0.0.1", 1234, "a"), new FakeTServer()); - servers.put(new TabletServerIdImpl("127.0.0.2", 1234, "b"), new FakeTServer()); - servers.put(new TabletServerIdImpl("127.0.0.3", 1234, "c"), new FakeTServer()); + servers.put( + new TabletServerIdImpl("127.0.0.1", 1234, "a", ServiceDescriptor.DEFAULT_GROUP_NAME), + new FakeTServer()); + servers.put( + new TabletServerIdImpl("127.0.0.2", 1234, "b", ServiceDescriptor.DEFAULT_GROUP_NAME), + new FakeTServer()); + servers.put( + new TabletServerIdImpl("127.0.0.3", 1234, "c", ServiceDescriptor.DEFAULT_GROUP_NAME), + new FakeTServer()); List metadataTable = new ArrayList<>(); String table = "t1"; metadataTable.add(makeTablet(table, null, null)); @@ -180,7 +187,8 @@ SortedMap getAssignments(Map(); - - var obj1 = new TestObj(1); - var obj1_dupe = new TestObj(1); - assertSame(obj1, obj1); - assertNotSame(obj1, obj1_dupe); - assertEquals(obj1, obj1_dupe); - assertEquals(obj1.hashCode(), obj1_dupe.hashCode()); - - // verify object gets added to the intern pool - assertSame(obj1, interner.intern(obj1)); - assertEquals(1, interner.size()); - - // verify equivalent, but not the same object, gets deduplicated - assertSame(obj1, interner.intern(obj1_dupe)); - assertEquals(1, interner.size()); - - // verify second object grows the intern pool size - var obj2 = new TestObj(2); - assertNotSame(obj1, obj2); - assertNotEquals(obj1, obj2); - var intern2 = interner.intern(obj2); - assertEquals(2, interner.size()); - - // sanity check to ensure we got the same object back for obj2, and it's not mangled with obj1 - assertSame(obj2, intern2); - assertNotEquals(obj1, intern2); - } - - @Test - @Timeout(20) - public void testInternsGetGarbageCollected() { - var interner = new Interner(); - assertEquals(0, interner.size()); // ensure empty - - // add one and keep a strong reference - var obj1 = interner.intern(new TestObj(1)); - assertEquals(1, interner.size()); - - // try to add a second, weakly referenced object until it sticks (may be GC'd between checks) - do { - interner.intern(new TestObj(2)); - } while (interner.size() != 2); - - // best effort to GC until the weakly reachable object goes away or until test times out - do { - System.gc(); - } while (interner.size() != 1); - - // ensure obj1 is still interned (because we kept a strong reference) - assertSame(obj1, interner.intern(new TestObj(1))); - - // ensure second test object is entirely new (previous ones should have been GC'd) - var obj2 = new TestObj(2); - assertSame(obj2, interner.intern(obj2)); - assertEquals(2, interner.size()); - } - -} diff --git a/server/base/src/main/java/org/apache/accumulo/server/AbstractServer.java b/server/base/src/main/java/org/apache/accumulo/server/AbstractServer.java index 89666879837..bb1d156cd6c 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/AbstractServer.java +++ b/server/base/src/main/java/org/apache/accumulo/server/AbstractServer.java @@ -27,6 +27,8 @@ import org.apache.accumulo.core.cli.ConfigOpts; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.SiteConfiguration; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.metrics.MetricsUtil; import org.apache.accumulo.core.trace.TraceUtil; @@ -44,6 +46,7 @@ public abstract class AbstractServer implements AutoCloseable, MetricsProducer, private final ServerContext context; protected final String applicationName; private final String hostname; + private final String resourceGroupName; private final ProcessMetrics processMetrics; @@ -52,6 +55,7 @@ protected AbstractServer(String appName, ConfigOpts opts, String[] args) { opts.parseArgs(appName, args); var siteConfig = opts.getSiteConfiguration(); this.hostname = siteConfig.get(Property.GENERAL_PROCESS_BIND_ADDRESS); + this.resourceGroupName = getResourceGroupNamePropertyValue(siteConfig); SecurityUtil.serverLogin(siteConfig); context = new ServerContext(siteConfig); Logger log = LoggerFactory.getLogger(getClass()); @@ -72,6 +76,14 @@ protected AbstractServer(String appName, ConfigOpts opts, String[] args) { processMetrics = new ProcessMetrics(context); } + protected String getResourceGroupNamePropertyValue(SiteConfiguration conf) { + return ServiceDescriptor.DEFAULT_GROUP_NAME; + } + + public String getResourceGroupName() { + return resourceGroupName; + } + /** * Run this server in a main thread */ diff --git a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java index 6b56edc6ff8..1a0cecbac2e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java +++ b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java @@ -36,6 +36,7 @@ import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily; @@ -229,6 +230,18 @@ public List check(Environment env, Mutation mutation) { } catch (IllegalArgumentException e) { violations = addViolation(violations, 9); } + } else if (columnFamily.equals(CurrentLocationColumnFamily.NAME) + || columnFamily.equals(LastLocationColumnFamily.NAME) + || columnFamily.equals(FutureLocationColumnFamily.NAME)) { + if (!columnQualifier.toString().isEmpty()) { + violations = addViolation(violations, 11); + } else { + try { + TServerInstance.fromString(new String(columnUpdate.getValue(), UTF_8)); + } catch (IllegalStateException e) { + violations = addViolation(violations, 12); + } + } } else if (columnFamily.equals(BulkFileColumnFamily.NAME)) { if (!columnUpdate.isDeleted() && !checkedBulk) { // splits, which also write the time reference, are allowed to write this reference even @@ -359,6 +372,10 @@ public String getViolationDescription(short violationCode) { return "Malformed operation id"; case 10: return "Malformed hosting goal"; + case 11: + return "location information in column qualifier"; + case 12: + return "location information in wrong format"; } return null; } diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java index b2cc9336de0..1e8849ac96f 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java @@ -192,7 +192,7 @@ public Map> getAllMarkers() throws WalMarkerException try { String path = root(); for (String child : zoo.getChildren(path)) { - TServerInstance inst = new TServerInstance(child); + TServerInstance inst = TServerInstance.fromString(child); List logs = result.computeIfAbsent(inst, k -> new ArrayList<>()); // This function is called by the Accumulo GC which deletes WAL markers. Therefore we do not diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java b/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java index e4af38f93f0..d2f6a07eb6c 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java @@ -39,6 +39,7 @@ import org.apache.accumulo.core.fate.zookeeper.ZooCache.ZcStat; import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.lock.ServiceLockData; +import org.apache.accumulo.core.lock.ServiceLockData.ThriftService; import org.apache.accumulo.core.manager.thrift.TabletServerStatus; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.rpc.ThriftUtil; @@ -325,8 +326,10 @@ private synchronized void checkServer(final Set updates, } } else { locklessServers.remove(zPath); - HostAndPort client = sld.orElseThrow().getAddress(ServiceLockData.ThriftService.TSERV); - TServerInstance instance = new TServerInstance(client, stat.getEphemeralOwner()); + HostAndPort client = sld.orElseThrow().getAddress(ThriftService.TSERV); + String resourceGroup = sld.orElseThrow().getGroup(ThriftService.TSERV); + TServerInstance instance = + new TServerInstance(client, stat.getEphemeralOwner(), resourceGroup); if (info == null) { updates.add(instance); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java index 0189b9294ae..ef1e3bb07e7 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java @@ -93,7 +93,7 @@ public void suspend(Collection tablets, var location = tm.getLocation(); HostAndPort server = null; if (location != null) { - server = location.getHostAndPort(); + server = location.getServerInstance().getHostAndPort(); } TabletLogger.suspended(tm.getExtent(), server, suspensionTimestamp, TimeUnit.MILLISECONDS, logsForDeadServers.size()); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java index 814c4ee2241..431a86ccdff 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java @@ -61,7 +61,6 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; import org.apache.accumulo.core.metadata.schema.TabletMetadata; -import org.apache.accumulo.core.util.AddressUtil; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.slf4j.Logger; @@ -92,7 +91,7 @@ private static void setCurrentServers(final IteratorSetting cfg, if (goodServers != null) { List servers = new ArrayList<>(); for (TServerInstance server : goodServers) { - servers.add(server.getHostPortSession()); + servers.add(server.getHostPortSessionGroup()); } cfg.addOption(SERVERS_OPTION, Joiner.on(",").join(servers)); } @@ -177,16 +176,9 @@ private static Set parseTableIDs(final String tableIDs) { private static Set parseServers(final String servers) { Set result = new HashSet<>(); if (servers != null) { - // parse "host:port[INSTANCE]" if (!servers.isEmpty()) { - for (String part : servers.split(",")) { - String[] parts = part.split("\\[", 2); - String hostport = parts[0]; - String instance = parts[1]; - if (instance != null && instance.endsWith("]")) { - instance = instance.substring(0, instance.length() - 1); - } - result.add(new TServerInstance(AddressUtil.parseAddress(hostport, false), instance)); + for (String server : servers.split(",")) { + result.add(TServerInstance.fromString(server)); } } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java index 45e6664f465..360cb6f5002 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java @@ -89,8 +89,8 @@ public Ample.ConditionalTabletMutator requireLocation(Location location) { Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate."); Preconditions.checkArgument(location.getType() == TabletMetadata.LocationType.FUTURE || location.getType() == TabletMetadata.LocationType.CURRENT); - Condition c = new Condition(getLocationFamily(location.getType()), location.getSession()) - .setValue(location.getHostPort()); + Condition c = new Condition(getLocationFamily(location.getType()), "") + .setValue(location.getServerInstance().toString()); mutation.addCondition(c); return this; } diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java index 0e3a48102ed..01f865ca517 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java @@ -170,15 +170,15 @@ protected String getLocationFamily(LocationType type) { @Override public T putLocation(Location location) { Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate."); - mutation.put(getLocationFamily(location.getType()), location.getSession(), - location.getHostPort()); + mutation.put(getLocationFamily(location.getType()), "", + location.getServerInstance().toString()); return getThis(); } @Override public T deleteLocation(Location location) { Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate."); - mutation.putDelete(getLocationFamily(location.getType()), location.getSession()); + mutation.putDelete(getLocationFamily(location.getType()), ""); return getThis(); } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java index 4320f61e2ef..af495d7f883 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java @@ -183,10 +183,11 @@ private static KeyExtent fixSplit(ServerContext context, TableId tableId, Text m } } - private static TServerInstance getTServerInstance(String address, ServiceLock zooLock) { + private static TServerInstance getTServerInstance(String address, ServiceLock zooLock, + String resourceGroupName) { while (true) { try { - return new TServerInstance(address, zooLock.getSessionId()); + return new TServerInstance(address, zooLock.getSessionId(), resourceGroupName); } catch (KeeperException | InterruptedException e) { log.error("{}", e.getMessage(), e); } @@ -197,7 +198,8 @@ private static TServerInstance getTServerInstance(String address, ServiceLock zo public static void replaceDatafiles(ServerContext context, KeyExtent extent, Set datafilesToDelete, Set scanFiles, Optional path, Long compactionId, DataFileValue size, String address, - Location lastLocation, ServiceLock zooLock, Optional ecid) { + Location lastLocation, ServiceLock zooLock, Optional ecid, + String resourceGroupName) { context.getAmple().putGcCandidates(extent.tableId(), datafilesToDelete); @@ -214,7 +216,7 @@ public static void replaceDatafiles(ServerContext context, KeyExtent extent, tablet.putCompactionId(compactionId); } - updateLastForCompactionMode(context, tablet, lastLocation, address, zooLock); + updateLastForCompactionMode(context, tablet, lastLocation, address, zooLock, resourceGroupName); if (ecid.isPresent()) { tablet.deleteExternalCompaction(ecid.orElseThrow()); @@ -231,7 +233,7 @@ public static void replaceDatafiles(ServerContext context, KeyExtent extent, public static Optional updateTabletDataFile(ServerContext context, KeyExtent extent, ReferencedTabletFile newDatafile, DataFileValue dfv, MetadataTime time, String address, ServiceLock zooLock, Set unusedWalLogs, Location lastLocation, - long flushId) { + long flushId, String resourceGroupName) { TabletMutator tablet = context.getAmple().mutateTablet(extent); // if there are no entries, the path doesn't get stored in metadata table, only the flush ID @@ -243,7 +245,8 @@ public static Optional updateTabletDataFile(ServerContext cont tablet.putTime(time); newFile = Optional.of(newDatafile.insert()); - updateLastForCompactionMode(context, tablet, lastLocation, address, zooLock); + updateLastForCompactionMode(context, tablet, lastLocation, address, zooLock, + resourceGroupName); } tablet.putFlushId(flushId); @@ -286,12 +289,13 @@ public static void updateLastForAssignmentMode(ClientContext context, * @param address The server address * @param zooLock The zookeeper lock */ - public static void updateLastForCompactionMode(ClientContext context, TabletMutator tabletMutator, - Location lastLocation, String address, ServiceLock zooLock) { + private static void updateLastForCompactionMode(ClientContext context, + TabletMutator tabletMutator, Location lastLocation, String address, ServiceLock zooLock, + String resourceGroupName) { // if the location mode is 'compaction', then preserve the current compaction location in the // last location value if ("compaction".equals(context.getConfiguration().get(Property.TSERV_LAST_LOCATION_MODE))) { - Location newLocation = Location.last(getTServerInstance(address, zooLock)); + Location newLocation = Location.last(getTServerInstance(address, zooLock, resourceGroupName)); updateLocation(tabletMutator, lastLocation, newLocation); } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java index a77fe3000ab..7086c6cb20e 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java @@ -27,6 +27,7 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily; @@ -207,14 +208,16 @@ public void testBulkFileCheck() { // active txid, mutation that looks like a load m = new Mutation(new Text("0;foo")); m.put(BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("5")); - m.put(CurrentLocationColumnFamily.NAME, new Text("789"), new Value("127.0.0.1:9997")); + m.put(CurrentLocationColumnFamily.NAME, new Text(""), + new Value(new TServerInstance("127.0.0.1:9997", "789", "default").toString())); violations = mc.check(createEnv(), m); assertNull(violations); // inactive txid, mutation that looks like a load m = new Mutation(new Text("0;foo")); m.put(BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("12345")); - m.put(CurrentLocationColumnFamily.NAME, new Text("789"), new Value("127.0.0.1:9997")); + m.put(CurrentLocationColumnFamily.NAME, new Text(""), + new Value(new TServerInstance("127.0.0.1:9997", "789", "default").toString())); violations = mc.check(createEnv(), m); assertNull(violations); diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/LiveTServerSetTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/LiveTServerSetTest.java index f67b7cb80da..b3206400a94 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/manager/LiveTServerSetTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/LiveTServerSetTest.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.Map; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.manager.LiveTServerSet.Listener; @@ -41,8 +42,9 @@ public void testSessionIds() { Map servers = new HashMap<>(); TServerConnection mockConn = EasyMock.createMock(TServerConnection.class); - TServerInfo server1 = new TServerInfo( - new TServerInstance(HostAndPort.fromParts("localhost", 1234), "5555"), mockConn); + TServerInfo server1 = + new TServerInfo(new TServerInstance(HostAndPort.fromParts("localhost", 1234), "5555", + ServiceDescriptor.DEFAULT_GROUP_NAME), mockConn); servers.put("server1", server1); LiveTServerSet tservers = new LiveTServerSet(EasyMock.createMock(ServerContext.class), diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java index 90272b3bfd8..e779b4660a8 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java @@ -44,6 +44,7 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.StoredTabletFile; @@ -266,8 +267,8 @@ public void testRootTabletStateStore() throws DistributedStoreException { ZooTabletStateStore tstore = new ZooTabletStateStore(context); KeyExtent root = RootTable.EXTENT; String sessionId = "this is my unique session data"; - TServerInstance server = - new TServerInstance(HostAndPort.fromParts("127.0.0.1", 10000), sessionId); + TServerInstance server = new TServerInstance(HostAndPort.fromParts("127.0.0.1", 10000), + sessionId, ServiceDescriptor.DEFAULT_GROUP_NAME); List assignments = Collections.singletonList(new Assignment(root, server, null)); tstore.setFutureLocations(assignments); int count = 0; diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java index e09cca2b34e..cd673d7a8e6 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java @@ -40,6 +40,7 @@ import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction; import org.apache.accumulo.core.metadata.StoredTabletFile; +import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily; @@ -90,9 +91,11 @@ private SortedMap createMetadataEntryKV(KeyExtent extent) { mutation.at().family(DataFileColumnFamily.NAME).qualifier(tf2.getMetaUpdateDelete()) .put(dfv2.encode()); - mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("s001").put("server1:8555"); + mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server1:8555", "s001", "default").toString()); - mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555"); + mutation.at().family(LastLocationColumnFamily.NAME).qualifier("") + .put(new TServerInstance("server2:8555", "s000", "default").toString()); LogEntry le1 = new LogEntry(extent, 55, "lf1"); mutation.at().family(le1.getColumnFamily()).qualifier(le1.getColumnQualifier()) diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/ManagerMetadataUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/ManagerMetadataUtilTest.java index 748dffaa657..49b505a7c26 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/ManagerMetadataUtilTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/ManagerMetadataUtilTest.java @@ -23,6 +23,7 @@ import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; @@ -35,9 +36,11 @@ public class ManagerMetadataUtilTest { private AccumuloConfiguration conf; private ClientContext context; private Ample.TabletMutator tabletMutator; - private final TServerInstance server1 = new TServerInstance("127.0.0.1:10000", 0); + private final TServerInstance server1 = + new TServerInstance("127.0.0.1:10000", 0, ServiceDescriptor.DEFAULT_GROUP_NAME); private final Location last1 = Location.last(server1); - private final TServerInstance server2 = new TServerInstance("127.0.0.2:10000", 1); + private final TServerInstance server2 = + new TServerInstance("127.0.0.2:10000", 1, ServiceDescriptor.DEFAULT_GROUP_NAME); private final Location last2 = Location.last(server2); @BeforeEach diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index 159de1504d1..3936f7b9981 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -59,6 +59,7 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.NamespaceId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; @@ -130,7 +131,6 @@ public class Compactor extends AbstractServer implements MetricsProducer, Compac private final UUID compactorId = UUID.randomUUID(); private final AccumuloConfiguration aconf; - private final String queueName; protected final AtomicReference currentCompactionId = new AtomicReference<>(); private final CompactionWatcher watcher; @@ -152,7 +152,6 @@ protected Compactor(ConfigOpts opts, String[] args) { protected Compactor(ConfigOpts opts, String[] args, AccumuloConfiguration conf) { super("compactor", opts, args); aconf = conf == null ? super.getConfiguration() : conf; - queueName = aconf.get(Property.COMPACTOR_QUEUE_NAME); setupSecurity(); watcher = new CompactionWatcher(aconf); var schedExecutor = @@ -166,6 +165,11 @@ public AccumuloConfiguration getConfiguration() { return aconf; } + @Override + protected String getResourceGroupNamePropertyValue(SiteConfiguration conf) { + return conf.get(Property.COMPACTOR_QUEUE_NAME); + } + @Override public void registerMetrics(MeterRegistry registry) { super.registerMetrics(registry); @@ -245,7 +249,7 @@ protected void announceExistence(HostAndPort clientAddress) ZooReaderWriter zoo = getContext().getZooReaderWriter(); String compactorQueuePath = - getContext().getZooKeeperRoot() + Constants.ZCOMPACTORS + "/" + this.queueName; + getContext().getZooKeeperRoot() + Constants.ZCOMPACTORS + "/" + getResourceGroupName(); String zPath = compactorQueuePath + "/" + hostPort; try { @@ -280,8 +284,8 @@ public void unableToMonitorLockNode(final Exception e) { for (int i = 0; i < 25; i++) { zoo.putPersistentData(zPath, new byte[0], NodeExistsPolicy.SKIP); - if (compactorLock.tryLock(lw, - new ServiceLockData(compactorId, hostPort, ThriftService.COMPACTOR, this.queueName))) { + if (compactorLock.tryLock(lw, new ServiceLockData(compactorId, hostPort, + ThriftService.COMPACTOR, getResourceGroupName()))) { LOG.debug("Obtained Compactor lock {}", compactorLock.getLockPath()); return; } @@ -435,7 +439,7 @@ protected TExternalCompactionJob getNextJob(Supplier uuid) throws RetriesE LOG.trace("Attempting to get next job, eci = {}", eci); currentCompactionId.set(eci); return coordinatorClient.getCompactionJob(TraceUtil.traceInfo(), - getContext().rpcCreds(), queueName, + getContext().rpcCreds(), getResourceGroupName(), ExternalCompactionUtil.getHostPortString(compactorAddress.getAddress()), eci.toString()); } catch (Exception e) { @@ -518,7 +522,7 @@ protected Runnable createCompactionJob(final TExternalCompactionJob job, job.getIteratorSettings().getIterators() .forEach(tis -> iters.add(SystemIteratorUtil.toIteratorSetting(tis))); - ExtCEnv cenv = new ExtCEnv(JOB_HOLDER, queueName); + ExtCEnv cenv = new ExtCEnv(JOB_HOLDER, getResourceGroupName()); FileCompactor compactor = new FileCompactor(getContext(), extent, files, outputFile, job.isPropagateDeletes(), cenv, iters, aConfig, tConfig.getCryptoService(), pausedMetrics); @@ -566,7 +570,8 @@ protected Supplier getNextId() { protected long getWaitTimeBetweenCompactionChecks() { // get the total number of compactors assigned to this queue - int numCompactors = ExternalCompactionUtil.countCompactors(queueName, getContext()); + int numCompactors = + ExternalCompactionUtil.countCompactors(getResourceGroupName(), getContext()); // Aim for around 3 compactors checking in every second long sleepTime = numCompactors * 1000L / 3; // Ensure a compactor sleeps at least around a second @@ -598,7 +603,7 @@ public void run() { try { MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, - clientAddress); + clientAddress, getResourceGroupName()); pausedMetrics = new PausedCompactionMetrics(); MetricsUtil.initializeProducers(this, pausedMetrics); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException @@ -621,7 +626,7 @@ public void run() { try { job = getNextJob(getNextId()); if (!job.isSetExternalCompactionId()) { - LOG.trace("No external compactions in queue {}", this.queueName); + LOG.trace("No external compactions in queue {}", getResourceGroupName()); UtilWaitThread.sleep(getWaitTimeBetweenCompactionChecks()); continue; } diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 1946ffee3b0..958711e1462 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@ -157,7 +157,8 @@ public void run() { } try { - MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, address); + MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, address, + getResourceGroupName()); MetricsUtil.initializeProducers(this, new GcMetrics(this)); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException @@ -355,8 +356,8 @@ public void unableToMonitorLockNode(final Exception e) { while (true) { ServiceLock lock = new ServiceLock(getContext().getZooReaderWriter().getZooKeeper(), path, zooLockUUID); - if (lock.tryLock(lockWatcher, - new ServiceLockData(zooLockUUID, addr.toString(), ThriftService.GC))) { + if (lock.tryLock(lockWatcher, new ServiceLockData(zooLockUUID, addr.toString(), + ThriftService.GC, getResourceGroupName()))) { log.debug("Got GC ZooKeeper lock"); return; } diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java index dc5802eef41..67d53e0ac44 100644 --- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java +++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java @@ -46,8 +46,10 @@ public class GarbageCollectWriteAheadLogsTest { - private final TServerInstance server1 = new TServerInstance("localhost:1234[SESSION]"); - private final TServerInstance server2 = new TServerInstance("localhost:1234[OTHERSESS]"); + private final TServerInstance server1 = + TServerInstance.fromString("localhost:1234#SESSION#default"); + private final TServerInstance server2 = + TServerInstance.fromString("localhost:1234#OTHERSESS#default"); private final UUID id = UUID.randomUUID(); private final Map> markers = Collections.singletonMap(server1, Collections.singletonList(id)); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index 4796a1aebc9..ad46832df2a 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -1189,7 +1189,7 @@ public void run() { try { MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, - sa.getAddress()); + sa.getAddress(), getResourceGroupName()); ManagerMetrics mm = new ManagerMetrics(getConfiguration(), this); MetricsUtil.initializeProducers(this, mm); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException @@ -1337,7 +1337,7 @@ boolean canSuspendTablets() { ServiceDescriptors descriptors = new ServiceDescriptors(); for (ThriftService svc : new ThriftService[] {ThriftService.MANAGER, ThriftService.COORDINATOR}) { - descriptors.addService(new ServiceDescriptor(uuid, svc, address)); + descriptors.addService(new ServiceDescriptor(uuid, svc, address, getResourceGroupName())); } sld = new ServiceLockData(descriptors); @@ -1571,7 +1571,8 @@ private ServiceLockData getManagerLock(final ServiceLockPath zManagerLoc) ServiceDescriptors descriptors = new ServiceDescriptors(); for (ThriftService svc : new ThriftService[] {ThriftService.MANAGER, ThriftService.COORDINATOR}) { - descriptors.addService(new ServiceDescriptor(zooLockUUID, svc, managerClientAddress)); + descriptors.addService( + new ServiceDescriptor(zooLockUUID, svc, managerClientAddress, getResourceGroupName())); } ServiceLockData sld = new ServiceLockData(descriptors); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java index 44fb1ba38f5..2fc9a006960 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java @@ -469,8 +469,9 @@ private void hostSuspendedTablet(TabletLists tLists, TabletMetadata tm, Location < tableConf.getTimeInMillis(Property.TABLE_SUSPEND_DURATION)) { // Tablet is suspended. See if its tablet server is back. TServerInstance returnInstance = null; - Iterator find = tLists.destinations - .tailMap(new TServerInstance(tm.getSuspend().server, " ")).keySet().iterator(); + Iterator find = + tLists.destinations.tailMap(new TServerInstance(tm.getSuspend().server, " ", + tableConf.get(Property.TABLE_ASSIGNMENT_GROUP))).keySet().iterator(); if (find.hasNext()) { TServerInstance found = find.next(); if (found.getHostAndPort().equals(tm.getSuspend().server)) { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionFinalizer.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionFinalizer.java index 7279bc08a9a..29632153321 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionFinalizer.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/CompactionFinalizer.java @@ -122,7 +122,8 @@ private void notifyTserver(Location loc, ExternalCompactionFinalState ecfs) { Client client = null; try { - client = ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER, loc.getHostAndPort(), context); + client = ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER, + loc.getServerInstance().getHostAndPort(), context); if (ecfs.getFinalState() == FinalState.FINISHED) { LOG.debug("Notifying tserver {} that compaction {} has finished.", loc, ecfs); client.compactionJobFinished(TraceUtil.traceInfo(), context.rpcCreds(), @@ -136,7 +137,7 @@ private void notifyTserver(Location loc, ExternalCompactionFinalState ecfs) { throw new IllegalArgumentException(ecfs.getFinalState().name()); } } catch (TException e) { - LOG.warn("Failed to notify tserver {}", loc.getHostAndPort(), e); + LOG.warn("Failed to notify tserver {}", loc.getServerInstance().getHostAndPort(), e); } finally { ThriftUtil.returnClient(client, context); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/RefreshTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/RefreshTablets.java index edfa3664f81..007fd243942 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/RefreshTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/RefreshTablets.java @@ -176,8 +176,8 @@ private List sendSyncRefreshRequest(Manager manager, long tid, Locati log.trace("{} sending refresh request to {} for {} extents", FateTxId.formatTid(tid), location, extents.size()); var timeInMillis = manager.getConfiguration().getTimeInMillis(Property.MANAGER_BULK_TIMEOUT); - client = ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER, location.getHostAndPort(), - manager.getContext(), timeInMillis); + client = ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER, + location.getServerInstance().getHostAndPort(), manager.getContext(), timeInMillis); var unrefreshed = client.refreshTablets(TraceUtil.traceInfo(), manager.getContext().rpcCreds(), diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java index f1bec9dfc11..7803e2ec7bc 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tserverOps/ShutdownTServer.java @@ -42,17 +42,19 @@ public class ShutdownTServer extends ManagerRepo { private static final Logger log = LoggerFactory.getLogger(ShutdownTServer.class); private final HostAndPort hostAndPort; private final String serverSession; + private final String resourceGroup; private final boolean force; public ShutdownTServer(TServerInstance server, boolean force) { this.hostAndPort = server.getHostAndPort(); this.serverSession = server.getSession(); + this.resourceGroup = server.getGroup(); this.force = force; } @Override public long isReady(long tid, Manager manager) { - TServerInstance server = new TServerInstance(hostAndPort, serverSession); + TServerInstance server = new TServerInstance(hostAndPort, serverSession, resourceGroup); // suppress assignment of tablets to the server if (force) { return 0; diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java index a02021b4dd6..d69b9459dbd 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java @@ -240,7 +240,7 @@ public void testCoordinatorColdStart() throws Exception { assertNotNull(t); assertEquals(1, t.size()); TServerInstance queuedTsi = t.iterator().next(); - assertEquals(tsi.getHostPortSession(), queuedTsi.getHostPortSession()); + assertEquals(tsi.getHostPortSessionGroup(), queuedTsi.getHostPortSessionGroup()); assertEquals(1, coordinator.getIndex().size()); assertTrue(coordinator.getIndex().containsKey(queuedTsi)); Set i = coordinator.getIndex().get(queuedTsi); @@ -298,7 +298,7 @@ public void testCoordinatorRestartNoRunningCompactions() throws Exception { assertNotNull(t); assertEquals(1, t.size()); TServerInstance queuedTsi = t.iterator().next(); - assertEquals(instance.getHostPortSession(), queuedTsi.getHostPortSession()); + assertEquals(instance.getHostPortSessionGroup(), queuedTsi.getHostPortSessionGroup()); assertEquals(1, coordinator.getIndex().size()); assertTrue(coordinator.getIndex().containsKey(queuedTsi)); Set i = coordinator.getIndex().get(queuedTsi); @@ -364,7 +364,7 @@ public void testCoordinatorRestartOneRunningCompaction() throws Exception { assertNotNull(t); assertEquals(1, t.size()); TServerInstance queuedTsi = t.iterator().next(); - assertEquals(instance.getHostPortSession(), queuedTsi.getHostPortSession()); + assertEquals(instance.getHostPortSessionGroup(), queuedTsi.getHostPortSessionGroup()); assertEquals(1, coordinator.getIndex().size()); assertTrue(coordinator.getIndex().containsKey(queuedTsi)); Set i = coordinator.getIndex().get(queuedTsi); @@ -435,7 +435,7 @@ public void testGetCompactionJob() throws Exception { assertNotNull(t); assertEquals(1, t.size()); TServerInstance queuedTsi = t.iterator().next(); - assertEquals(tsi.getHostPortSession(), queuedTsi.getHostPortSession()); + assertEquals(tsi.getHostPortSessionGroup(), queuedTsi.getHostPortSessionGroup()); assertEquals(1, coordinator.getIndex().size()); assertTrue(coordinator.getIndex().containsKey(queuedTsi)); Set i = coordinator.getIndex().get(queuedTsi); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/QueueSummariesTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/QueueSummariesTest.java index 7a213149cd4..3278e3840a4 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/QueueSummariesTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/QueueSummariesTest.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Set; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.tabletserver.thrift.TCompactionQueueSummary; import org.apache.accumulo.manager.compaction.QueueSummaries.PrioTserver; @@ -33,7 +34,7 @@ public class QueueSummariesTest { private TServerInstance ntsi(String tserver) { - return new TServerInstance(tserver + ":9997", 0); + return new TServerInstance(tserver + ":9997", 0, ServiceDescriptor.DEFAULT_GROUP_NAME); } private PrioTserver npt(String tserver, short prio) { diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/ShutdownTServerTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/ShutdownTServerTest.java index 9aff9a89faa..62250a7f47c 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/ShutdownTServerTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/ShutdownTServerTest.java @@ -26,6 +26,7 @@ import java.util.HashMap; import org.apache.accumulo.core.fate.Repo; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.thrift.TableInfo; import org.apache.accumulo.core.manager.thrift.TabletServerStatus; import org.apache.accumulo.core.metadata.TServerInstance; @@ -42,7 +43,8 @@ public class ShutdownTServerTest { @Test public void testSingleShutdown() throws Exception { HostAndPort hap = HostAndPort.fromParts("localhost", 1234); - final TServerInstance tserver = new TServerInstance(hap, "fake"); + final TServerInstance tserver = + new TServerInstance(hap, "fake", ServiceDescriptor.DEFAULT_GROUP_NAME); final boolean force = false; final ShutdownTServer op = new ShutdownTServer(tserver, force); diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java index 7e1242def63..30f3480a5a8 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java @@ -825,8 +825,8 @@ private void getMonitorLock() throws KeeperException, InterruptedException { while (true) { MoniterLockWatcher monitorLockWatcher = new MoniterLockWatcher(); monitorLock = new ServiceLock(zoo.getZooKeeper(), monitorLockPath, zooLockUUID); - monitorLock.lock(monitorLockWatcher, - new ServiceLockData(zooLockUUID, getHostname(), ThriftService.NONE)); + monitorLock.lock(monitorLockWatcher, new ServiceLockData(zooLockUUID, getHostname(), + ThriftService.NONE, getResourceGroupName())); monitorLockWatcher.waitForChange(); diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java index 39a6b51cf7b..ff8d22a8d60 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java @@ -156,7 +156,7 @@ public TabletServers getParticipatingTabletServers(@PathParam("tableId") @NotNul final TabletMetadata tm = scanner.next().getTabletMetadata(); if (tm.hasCurrent()) { try { - locs.add(tm.getLocation().getHostPort()); + locs.add(tm.getLocation().getServerInstance().getHostPort()); } catch (Exception ex) { scanner.close(); return tabletServers; diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java index 102267aaf70..258a2919caf 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java @@ -53,6 +53,7 @@ import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.thrift.InitialMultiScan; import org.apache.accumulo.core.dataImpl.thrift.InitialScan; @@ -190,8 +191,6 @@ private TabletMetadataLoader(Ample ample) { private ZooCache managerLockCache; - private final String groupName; - public ScanServer(ConfigOpts opts, String[] args) { super("sserver", opts, args); @@ -236,8 +235,6 @@ public ScanServer(ConfigOpts opts, String[] args) { delegate = newThriftScanClientHandler(new WriteTracker()); - this.groupName = getConfiguration().get(Property.SSERV_GROUP_NAME); - ThreadPools.watchCriticalScheduledTask(getContext().getScheduledExecutor() .scheduleWithFixedDelay(() -> cleanUpReservedFiles(scanServerReservationExpiration), scanServerReservationExpiration, scanServerReservationExpiration, @@ -245,6 +242,11 @@ public ScanServer(ConfigOpts opts, String[] args) { } + @Override + protected String getResourceGroupNamePropertyValue(SiteConfiguration conf) { + return conf.get(Property.SSERV_GROUP_NAME); + } + @VisibleForTesting protected ThriftScanClientHandler newThriftScanClientHandler(WriteTracker writeTracker) { return new ThriftScanClientHandler(this, writeTracker); @@ -327,7 +329,7 @@ public void unableToMonitorLockNode(final Exception e) { zoo.putPersistentData(zLockPath.toString(), new byte[0], NodeExistsPolicy.SKIP); if (scanServerLock.tryLock(lw, new ServiceLockData(serverLockUUID, getClientAddressString(), - ThriftService.TABLET_SCAN, this.groupName))) { + ThriftService.TABLET_SCAN, getResourceGroupName()))) { LOG.debug("Obtained scan server lock {}", scanServerLock.getLockPath()); return scanServerLock; } @@ -357,7 +359,7 @@ public void run() { try { MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, - clientAddress); + clientAddress, getResourceGroupName()); scanMetrics = new TabletServerScanMetrics(); MetricsUtil.initializeProducers(this, scanMetrics); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index 805d32a01b9..077f122b3aa 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@ -73,6 +73,7 @@ import org.apache.accumulo.core.clientImpl.DurabilityImpl; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.InstanceId; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; @@ -368,9 +369,15 @@ private void logBusyTablets(List> busyTablets, } else { authKeyWatcher = null; } + config(); } + @Override + protected String getResourceGroupNamePropertyValue(SiteConfiguration conf) { + return conf.get(Property.TSERV_GROUP_NAME); + } + public InstanceId getInstanceID() { return getContext().getInstanceID(); } @@ -672,8 +679,8 @@ public void unableToMonitorLockNode(final Exception e) { for (ThriftService svc : new ThriftService[] {ThriftService.CLIENT, ThriftService.TABLET_INGEST, ThriftService.TABLET_MANAGEMENT, ThriftService.TABLET_SCAN, ThriftService.TSERV}) { - descriptors - .addService(new ServiceDescriptor(tabletServerUUID, svc, getClientAddressString())); + descriptors.addService(new ServiceDescriptor(tabletServerUUID, svc, + getClientAddressString(), getResourceGroupName())); } if (tabletServerLock.tryLock(lw, new ServiceLockData(descriptors))) { @@ -719,8 +726,8 @@ public void run() { } try { - MetricsUtil.initializeMetrics(context.getConfiguration(), this.applicationName, - clientAddress); + MetricsUtil.initializeMetrics(context.getConfiguration(), this.applicationName, clientAddress, + getResourceGroupName()); metrics = new TabletServerMetrics(this); updateMetrics = new TabletServerUpdateMetrics(); @@ -945,7 +952,7 @@ public TServerInstance getTabletSession() { } try { - return new TServerInstance(address, tabletServerLock.getSessionId()); + return new TServerInstance(address, tabletServerLock.getSessionId(), getResourceGroupName()); } catch (Exception ex) { log.warn("Unable to read session from tablet server lock" + ex); return null; diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java index e2843890f7f..d544db1302f 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java @@ -112,8 +112,8 @@ public void run() { } try { - TServerInstance instance = - new TServerInstance(server.clientAddress, server.getLock().getSessionId()); + TServerInstance instance = new TServerInstance(server.clientAddress, + server.getLock().getSessionId(), server.getResourceGroupName()); // ELASTICITY_TODO: Modify Tablet to keep a reference to TableMetadata so that we // can get rid of TabletMetadataImposter TabletMetadata tm = new TabletMetadataImposter(extent, null, Location.current(instance), null, diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java index 8d1171b4221..1b34968ea78 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java @@ -473,7 +473,8 @@ Optional bringMajorCompactionOnline(Set oldD ManagerMetadataUtil.replaceDatafiles(tablet.getContext(), extent, oldDatafiles, filesInUseByScans, newFile, compactionIdToWrite, dfv, tablet.getTabletServer().getClientAddressString(), lastLocation, - tablet.getTabletServer().getLock(), ecid); + tablet.getTabletServer().getLock(), ecid, + tablet.getTabletServer().getResourceGroupName()); tablet.setLastCompactionID(compactionIdToWrite); removeFilesAfterScan(filesInUseByScans); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java index 538c2387c76..3260683e4d5 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java @@ -2045,7 +2045,7 @@ public Optional updateTabletDataFile(long maxCommittedTime, return ManagerMetadataUtil.updateTabletDataFile(getTabletServer().getContext(), extent, newDatafile, dfv, tabletTime.getMetadataTime(persistedTime), tabletServer.getClientAddressString(), tabletServer.getLock(), unusedWalLogs, - lastLocation, flushId); + lastLocation, flushId, tabletServer.getResourceGroupName()); } } diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java index 346854c003c..5c01acc088b 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.FutureLocationColumnFamily; @@ -96,19 +97,26 @@ public void testBadTabletMetadata() throws Exception { TabletColumnFamily.encodePrevEndRow(null).get()); put(tabletMeta, "1<", ServerColumnFamily.DIRECTORY_COLUMN, "t1".getBytes()); put(tabletMeta, "1<", ServerColumnFamily.TIME_COLUMN, "M0".getBytes()); - put(tabletMeta, "1<", FutureLocationColumnFamily.NAME, "4", "127.0.0.1:9997"); + put(tabletMeta, "1<", FutureLocationColumnFamily.NAME, "", + new TServerInstance("127.0.0.1:9997", 4, "default").toString()); - TServerInstance tsi = new TServerInstance("127.0.0.1:9997", 4); + TServerInstance tsi = + new TServerInstance("127.0.0.1:9997", 4, ServiceDescriptor.DEFAULT_GROUP_NAME); TabletMetadata tm = TabletMetadata.convertRow(tabletMeta.entrySet().iterator(), EnumSet.allOf(ColumnType.class), true, false); assertTrue(checkTabletMetadata(ke, tsi, tm)); - assertFail(tabletMeta, ke, new TServerInstance("127.0.0.1:9998", 4)); - assertFail(tabletMeta, ke, new TServerInstance("127.0.0.1:9998", 5)); - assertFail(tabletMeta, ke, new TServerInstance("127.0.0.1:9997", 5)); - assertFail(tabletMeta, ke, new TServerInstance("127.0.0.2:9997", 4)); - assertFail(tabletMeta, ke, new TServerInstance("127.0.0.2:9997", 5)); + assertFail(tabletMeta, ke, + new TServerInstance("127.0.0.1:9998", 4, ServiceDescriptor.DEFAULT_GROUP_NAME)); + assertFail(tabletMeta, ke, + new TServerInstance("127.0.0.1:9998", 5, ServiceDescriptor.DEFAULT_GROUP_NAME)); + assertFail(tabletMeta, ke, + new TServerInstance("127.0.0.1:9997", 5, ServiceDescriptor.DEFAULT_GROUP_NAME)); + assertFail(tabletMeta, ke, + new TServerInstance("127.0.0.2:9997", 4, ServiceDescriptor.DEFAULT_GROUP_NAME)); + assertFail(tabletMeta, ke, + new TServerInstance("127.0.0.2:9997", 5, ServiceDescriptor.DEFAULT_GROUP_NAME)); assertFail(tabletMeta, new KeyExtent(TableId.of("1"), null, new Text("m")), tsi); @@ -120,12 +128,12 @@ public void testBadTabletMetadata() throws Exception { assertFail(tabletMeta, ke, tsi, newKey("1<", ServerColumnFamily.TIME_COLUMN)); - assertFail(tabletMeta, ke, tsi, newKey("1<", FutureLocationColumnFamily.NAME, "4")); + assertFail(tabletMeta, ke, tsi, newKey("1<", FutureLocationColumnFamily.NAME, "")); TreeMap copy = new TreeMap<>(tabletMeta); put(copy, "1<", CurrentLocationColumnFamily.NAME, "4", "127.0.0.1:9997"); assertFail(copy, ke, tsi); - assertFail(copy, ke, tsi, newKey("1<", FutureLocationColumnFamily.NAME, "4")); + assertFail(copy, ke, tsi, newKey("1<", FutureLocationColumnFamily.NAME, "")); copy = new TreeMap<>(tabletMeta); put(copy, "1<", CurrentLocationColumnFamily.NAME, "5", "127.0.0.1:9998"); diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java index 71b2a18b75d..9b0877f3952 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java +++ b/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java @@ -427,7 +427,7 @@ public Factory location(Location location) { if (location == null) { this.location = "None"; } else { - String server = location.getHostPort(); + String server = location.getServerInstance().getHostPort(); this.location = location.getType() + ":" + server; } return this; diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java index 583819b96b9..7672ee710c9 100644 --- a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java +++ b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java @@ -34,6 +34,8 @@ import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; +import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletState; import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; import org.apache.accumulo.shell.Shell; @@ -91,7 +93,8 @@ protected List getMetadataInfo(Shell shellState, KeyExtent ke1 = new KeyExtent(tableId, new Text("a"), null); KeyExtent ke2 = new KeyExtent(tableId, new Text("m"), new Text("a")); KeyExtent ke3 = new KeyExtent(tableId, null, new Text("m")); - Location loc = Location.current("localhost", ""); + Location loc = Location + .current(new TServerInstance("localhost:9997", "", ServiceDescriptor.DEFAULT_GROUP_NAME)); ListTabletsCommand.TabletRowInfo.Factory factory = new ListTabletsCommand.TabletRowInfo.Factory(tableName, ke1).dir("t-dir1").numFiles(1) .numWalLogs(1).numEntries(1).size(100).status(TabletState.HOSTED.toString()) @@ -168,7 +171,8 @@ public void builderTest() { Text startRow = new Text("a"); Text endRow = new Text("z"); KeyExtent ke = new KeyExtent(id, endRow, startRow); - Location loc = Location.current("localhost", ""); + Location loc = Location + .current(new TServerInstance("localhost:9997", "", ServiceDescriptor.DEFAULT_GROUP_NAME)); ListTabletsCommand.TabletRowInfo.Factory factory = new ListTabletsCommand.TabletRowInfo.Factory("aName", ke).numFiles(1).numWalLogs(2) .numEntries(3).size(4).status(TabletState.HOSTED.toString()).location(loc) @@ -184,7 +188,7 @@ public void builderTest() { assertEquals("4", info.getSize(false)); assertEquals(4, info.size); assertEquals("HOSTED", info.status); - assertEquals("CURRENT:localhost", info.location); + assertEquals("CURRENT:localhost:9997", info.location); assertEquals(TableId.of("123"), info.tableId); assertEquals(startRow + " " + endRow, info.getTablet()); assertTrue(info.tableExists); diff --git a/test/src/main/java/org/apache/accumulo/test/LocatorIT.java b/test/src/main/java/org/apache/accumulo/test/LocatorIT.java index bf090bcb33c..8e537f18418 100644 --- a/test/src/main/java/org/apache/accumulo/test/LocatorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/LocatorIT.java @@ -96,7 +96,7 @@ public void testBasic() throws Exception { final Predicate alwaysHostedAndCurrentNotNull = t -> t.getHostingGoal() == TabletHostingGoal.ALWAYS && t.hasCurrent() - && t.getLocation().getHostAndPort() != null; + && t.getLocation().getServerInstance().getHostAndPort() != null; try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { String tableName = getUniqueNames(1)[0]; diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java index 5c25fd9fd70..37edf32c098 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java @@ -33,6 +33,7 @@ import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.StoredTabletFile; @@ -69,8 +70,8 @@ public void testLocations() throws Exception { var e1 = new KeyExtent(tid, new Text("c"), null); - var ts1 = new TServerInstance("localhost:9997", 5000L); - var ts2 = new TServerInstance("localhost:9997", 6000L); + var ts1 = new TServerInstance("localhost:9997", 5000L, ServiceDescriptor.DEFAULT_GROUP_NAME); + var ts2 = new TServerInstance("localhost:9997", 6000L, ServiceDescriptor.DEFAULT_GROUP_NAME); var context = cluster.getServerContext(); @@ -146,8 +147,8 @@ public void testFiles() throws Exception { var e1 = new KeyExtent(tid, new Text("c"), null); - var ts1 = new TServerInstance("localhost:9997", 5000L); - var ts2 = new TServerInstance("localhost:9997", 6000L); + var ts1 = new TServerInstance("localhost:9997", 5000L, ServiceDescriptor.DEFAULT_GROUP_NAME); + var ts2 = new TServerInstance("localhost:9997", 6000L, ServiceDescriptor.DEFAULT_GROUP_NAME); var context = cluster.getServerContext(); @@ -279,8 +280,8 @@ public void testMultipleExtents() throws Exception { var e3 = new KeyExtent(tid, new Text("j"), new Text("f")); var e4 = new KeyExtent(tid, null, new Text("j")); - var ts1 = new TServerInstance("localhost:9997", 5000L); - var ts2 = new TServerInstance("localhost:9997", 6000L); + var ts1 = new TServerInstance("localhost:9997", 5000L, ServiceDescriptor.DEFAULT_GROUP_NAME); + var ts2 = new TServerInstance("localhost:9997", 6000L, ServiceDescriptor.DEFAULT_GROUP_NAME); var context = cluster.getServerContext(); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java index 6e326de445a..af5a9113e0c 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java @@ -67,7 +67,8 @@ public void test() throws Exception { newTablet = ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); } while (!newTablet.hasCurrent()); // this would be null if the mode was not "assign" - assertEquals(newTablet.getLocation().getHostPort(), newTablet.getLast().getHostPort()); + assertEquals(newTablet.getLocation().getServerInstance().getHostPort(), + newTablet.getLast().getServerInstance().getHostPort()); // put something in it try (BatchWriter bw = c.createBatchWriter(tableName)) { @@ -82,8 +83,10 @@ public void test() throws Exception { // last location should not be set yet TabletMetadata unflushed = ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); - assertEquals(newTablet.getLocation().getHostPort(), unflushed.getLocation().getHostPort()); - assertEquals(newTablet.getLocation().getHostPort(), unflushed.getLast().getHostPort()); + assertEquals(newTablet.getLocation().getServerInstance().getHostPort(), + unflushed.getLocation().getServerInstance().getHostPort()); + assertEquals(newTablet.getLocation().getServerInstance().getHostPort(), + unflushed.getLast().getServerInstance().getHostPort()); assertTrue(newTablet.hasCurrent()); // take the tablet offline @@ -92,7 +95,8 @@ public void test() throws Exception { ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertNull(offline.getLocation()); assertFalse(offline.hasCurrent()); - assertEquals(newTablet.getLocation().getHostPort(), offline.getLast().getHostPort()); + assertEquals(newTablet.getLocation().getServerInstance().getHostPort(), + offline.getLast().getServerInstance().getHostPort()); // put it back online, should have the same last location c.tableOperations().online(tableName, true); @@ -100,7 +104,8 @@ public void test() throws Exception { ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(online.hasCurrent()); assertNotNull(online.getLocation()); - assertEquals(newTablet.getLast().getHostPort(), online.getLast().getHostPort()); + assertEquals(newTablet.getLast().getServerInstance().getHostPort(), + online.getLast().getServerInstance().getHostPort()); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java b/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java index 471aedf566d..8e12e97eaf8 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java @@ -81,7 +81,8 @@ public void test() throws Exception { // no last location should be set yet TabletMetadata unflushed = ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); - assertEquals(newTablet.getLocation().getHostPort(), unflushed.getLocation().getHostPort()); + assertEquals(newTablet.getLocation().getServerInstance().getHostPort(), + unflushed.getLocation().getServerInstance().getHostPort()); assertNull(unflushed.getLast()); assertTrue(newTablet.hasCurrent()); @@ -90,8 +91,10 @@ public void test() throws Exception { TabletMetadata flushed = ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); - assertEquals(newTablet.getLocation().getHostPort(), flushed.getLocation().getHostPort()); - assertEquals(flushed.getLocation().getHostPort(), flushed.getLast().getHostPort()); + assertEquals(newTablet.getLocation().getServerInstance().getHostPort(), + flushed.getLocation().getServerInstance().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + flushed.getLast().getServerInstance().getHostPort()); assertTrue(newTablet.hasCurrent()); // take the tablet offline @@ -99,7 +102,8 @@ public void test() throws Exception { TabletMetadata offline = ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertNull(offline.getLocation()); - assertEquals(flushed.getLocation().getHostPort(), offline.getLast().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + offline.getLast().getServerInstance().getHostPort()); // put it back online, should have the same last location c.tableOperations().online(tableName, true); @@ -107,7 +111,8 @@ public void test() throws Exception { ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(online.hasCurrent()); assertNotNull(online.getLocation()); - assertEquals(offline.getLast().getHostPort(), online.getLast().getHostPort()); + assertEquals(offline.getLast().getServerInstance().getHostPort(), + online.getLast().getServerInstance().getHostPort()); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java index 93626de9580..699b23c7e08 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java @@ -140,7 +140,8 @@ public void test() throws Exception { TabletMetadata flushed = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(flushed.hasCurrent()); assertNotNull(flushed.getLocation()); - assertEquals(flushed.getLocation().getHostPort(), flushed.getLast().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + flushed.getLast().getServerInstance().getHostPort()); assertFalse(flushed.getLocation().getType().equals(LocationType.FUTURE)); assertEquals(TabletHostingGoal.ONDEMAND, flushed.getHostingGoal()); @@ -149,7 +150,8 @@ public void test() throws Exception { TabletMetadata offline = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertFalse(offline.hasCurrent()); assertNull(offline.getLocation()); - assertEquals(flushed.getLocation().getHostPort(), offline.getLast().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + offline.getLast().getServerInstance().getHostPort()); assertEquals(TabletHostingGoal.ONDEMAND, offline.getHostingGoal()); // put it back online @@ -157,7 +159,8 @@ public void test() throws Exception { TabletMetadata online = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(online.hasCurrent()); assertNotNull(online.getLocation()); - assertEquals(online.getLocation().getHostPort(), online.getLast().getHostPort()); + assertEquals(online.getLocation().getServerInstance().getHostPort(), + online.getLast().getServerInstance().getHostPort()); assertEquals(TabletHostingGoal.ONDEMAND, online.getHostingGoal()); // set the hosting goal to always @@ -172,7 +175,8 @@ public void test() throws Exception { final TabletMetadata always = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(alwaysHostedOrCurrentNotNull.test(always)); assertTrue(always.hasCurrent()); - assertEquals(flushed.getLocation().getHostPort(), always.getLast().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + always.getLast().getServerInstance().getHostPort()); assertEquals(TabletHostingGoal.ALWAYS, always.getHostingGoal()); // set the hosting goal to never @@ -185,7 +189,8 @@ public void test() throws Exception { final TabletMetadata never = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(neverHostedOrCurrentNull.test(never)); assertNull(never.getLocation()); - assertEquals(flushed.getLocation().getHostPort(), never.getLast().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + never.getLast().getServerInstance().getHostPort()); assertEquals(TabletHostingGoal.NEVER, never.getHostingGoal()); // set the hosting goal to ondemand @@ -198,7 +203,8 @@ public void test() throws Exception { final TabletMetadata ondemand = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); assertTrue(ondemandHosted.test(ondemand)); assertNull(ondemand.getLocation()); - assertEquals(flushed.getLocation().getHostPort(), ondemand.getLast().getHostPort()); + assertEquals(flushed.getLocation().getServerInstance().getHostPort(), + ondemand.getLast().getServerInstance().getHostPort()); assertEquals(TabletHostingGoal.ONDEMAND, ondemand.getHostingGoal()); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/RegexGroupBalanceIT.java b/test/src/main/java/org/apache/accumulo/test/functional/RegexGroupBalanceIT.java index b7194b86bd2..6327f73a42a 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/RegexGroupBalanceIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/RegexGroupBalanceIT.java @@ -194,8 +194,8 @@ private Table getCounts(AccumuloClient client, String } else { group = group.substring(tableId.canonical().length() + 1).substring(0, 2); } - String loc = new TServerInstance(entry.getValue(), entry.getKey().getColumnQualifier()) - .getHostPortSession(); + String loc = + TServerInstance.fromString(entry.getValue().toString()).getHostPortSessionGroup(); MutableInt count = groupLocationCounts.get(group, loc); if (count == null) { diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java index 379c259494d..fc3e563e83e 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java @@ -51,6 +51,7 @@ import org.apache.accumulo.core.lock.ServiceLock.LockLossReason; import org.apache.accumulo.core.lock.ServiceLock.LockWatcher; import org.apache.accumulo.core.lock.ServiceLockData; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.lock.ServiceLockData.ThriftService; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.ReferencedTabletFile; @@ -211,7 +212,8 @@ private void splitPartiallyAndRecover(ServerContext context, KeyExtent extent, K lowDatafileSizes, highDatafileSizes, highDatafilesToRemove); MetadataTableUtil.splitTablet(high, extent.prevEndRow(), splitRatio, context, zl, Set.of()); - TServerInstance instance = new TServerInstance(location, zl.getSessionId()); + TServerInstance instance = + new TServerInstance(location, zl.getSessionId(), ServiceDescriptor.DEFAULT_GROUP_NAME); Assignment assignment = new Assignment(high, instance, null); TabletMutator tabletMutator = context.getAmple().mutateTablet(extent); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java index 1b3a14df5e5..1b2ecbc10c3 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java @@ -57,6 +57,7 @@ import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.fate.zookeeper.ZooUtil; import org.apache.accumulo.core.lock.ServiceLock; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.manager.thrift.ManagerState; @@ -375,7 +376,8 @@ public Set onlineTabletServers() { var zPath = ServiceLock.path(ZooUtil.getRoot(context.instanceOperations().getInstanceId()) + Constants.ZTSERVERS + "/" + tserver); long sessionId = ServiceLock.getSessionId(context.getZooCache(), zPath); - tservers.add(new TServerInstance(tserver, sessionId)); + tservers + .add(new TServerInstance(tserver, sessionId, ServiceDescriptor.DEFAULT_GROUP_NAME)); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/test/src/main/java/org/apache/accumulo/test/manager/MergeStateIT.java b/test/src/main/java/org/apache/accumulo/test/manager/MergeStateIT.java index d49c932f2d2..2a12aa3394b 100644 --- a/test/src/main/java/org/apache/accumulo/test/manager/MergeStateIT.java +++ b/test/src/main/java/org/apache/accumulo/test/manager/MergeStateIT.java @@ -39,6 +39,7 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.manager.thrift.ManagerState; import org.apache.accumulo.core.metadata.MetadataTable; @@ -69,8 +70,8 @@ public class MergeStateIT extends ConfigurableMacBase { private static class MockCurrentState implements CurrentState { - TServerInstance someTServer = - new TServerInstance(HostAndPort.fromParts("127.0.0.1", 1234), 0x123456); + TServerInstance someTServer = new TServerInstance(HostAndPort.fromParts("127.0.0.1", 1234), + 0x123456, ServiceDescriptor.DEFAULT_GROUP_NAME); MergeInfo mergeInfo; MockCurrentState(MergeInfo info) { diff --git a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java index 125af5b23c8..fb4d6fcb517 100644 --- a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java @@ -53,6 +53,7 @@ import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.TServerInstance; @@ -191,7 +192,8 @@ public void shutdownAndResumeTserver() throws Exception { ClientTabletCache.LocationNeed.REQUIRED); // add it to the set of servers with metadata metadataServerSet.add(new TServerInstance(tab.getTserverLocation().orElseThrow(), - Long.valueOf(tab.getTserverSession().orElseThrow(), 16))); + Long.valueOf(tab.getTserverSession().orElseThrow(), 16), + ServiceDescriptor.DEFAULT_GROUP_NAME)); } } @@ -206,7 +208,7 @@ public void shutdownAndResumeTserver() throws Exception { Collections.shuffle(tserversList, random); for (int i1 = 0; i1 < count; ++i1) { - final String tserverName = tserversList.get(i1).getHostPortSession(); + final String tserverName = tserversList.get(i1).getHostPortSessionGroup(); ThriftClientTypes.MANAGER.executeVoid(ctx, client -> { log.info("Sending shutdown command to {} via ManagerClientService", tserverName); client.shutdownTabletServer(null, ctx.rpcCreds(), tserverName, false); @@ -443,7 +445,7 @@ private void scan(ClientContext ctx, String tableName, String metaName) { suspended.put(tm.getSuspend().server, ke); ++suspendedCount; } else if (tm.hasCurrent()) { - hosted.put(tm.getLocation().getHostAndPort(), ke); + hosted.put(tm.getLocation().getServerInstance().getHostAndPort(), ke); ++hostedCount; } else if (tm.getLocation() != null && tm.getLocation().getType().equals(LocationType.FUTURE)) { diff --git a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java index 5d2be5df56b..e47001de011 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java @@ -53,6 +53,7 @@ import org.apache.accumulo.core.dataImpl.thrift.TSummaries; import org.apache.accumulo.core.dataImpl.thrift.TSummaryRequest; import org.apache.accumulo.core.dataImpl.thrift.UpdateErrors; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.thrift.TabletServerStatus; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.TServerInstance; @@ -358,7 +359,8 @@ public static void main(String[] args) throws Exception { List assignments = new ArrayList<>(); try (var s = new TabletManagementScanner(context, tableRange, MetadataTable.NAME)) { long randomSessionID = opts.port; - TServerInstance instance = new TServerInstance(addr, randomSessionID); + TServerInstance instance = + new TServerInstance(addr, randomSessionID, ServiceDescriptor.DEFAULT_GROUP_NAME); while (s.hasNext()) { TabletMetadata next = s.next().getTabletMetadata(); diff --git a/test/src/test/java/org/apache/accumulo/test/ChaoticLoadBalancerTest.java b/test/src/test/java/org/apache/accumulo/test/ChaoticLoadBalancerTest.java index 8b803e70895..e740146a33b 100644 --- a/test/src/test/java/org/apache/accumulo/test/ChaoticLoadBalancerTest.java +++ b/test/src/test/java/org/apache/accumulo/test/ChaoticLoadBalancerTest.java @@ -34,6 +34,7 @@ import org.apache.accumulo.core.data.TabletId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.TabletIdImpl; +import org.apache.accumulo.core.lock.ServiceLockData.ServiceDescriptor; import org.apache.accumulo.core.manager.balancer.AssignmentParamsImpl; import org.apache.accumulo.core.manager.balancer.BalanceParamsImpl; import org.apache.accumulo.core.manager.balancer.TServerStatusImpl; @@ -95,9 +96,15 @@ public List getOnlineTabletsForTable(TabletServerId tserver, T @Test public void testAssignMigrations() { servers.clear(); - servers.put(new TabletServerIdImpl("127.0.0.1", 1234, "a"), new FakeTServer()); - servers.put(new TabletServerIdImpl("127.0.0.1", 1235, "b"), new FakeTServer()); - servers.put(new TabletServerIdImpl("127.0.0.1", 1236, "c"), new FakeTServer()); + servers.put( + new TabletServerIdImpl("127.0.0.1", 1234, "a", ServiceDescriptor.DEFAULT_GROUP_NAME), + new FakeTServer()); + servers.put( + new TabletServerIdImpl("127.0.0.1", 1235, "b", ServiceDescriptor.DEFAULT_GROUP_NAME), + new FakeTServer()); + servers.put( + new TabletServerIdImpl("127.0.0.1", 1236, "c", ServiceDescriptor.DEFAULT_GROUP_NAME), + new FakeTServer()); Map metadataTable = new TreeMap<>(); String table = "t1"; metadataTable.put(makeTablet(table, null, null), null); @@ -134,7 +141,8 @@ public void testUnevenAssignment() { servers.clear(); for (char c : "abcdefghijklmnopqrstuvwxyz".toCharArray()) { String cString = Character.toString(c); - TabletServerId tsi = new TabletServerIdImpl("127.0.0.1", c, cString); + TabletServerId tsi = + new TabletServerIdImpl("127.0.0.1", c, cString, ServiceDescriptor.DEFAULT_GROUP_NAME); FakeTServer fakeTServer = new FakeTServer(); servers.put(tsi, fakeTServer); fakeTServer.tablets.add(makeTablet(cString, null, null));