Handle RFC 6761 special-use domain names in Dns resolution

src/libraries/System.Net.NameResolution/src/System/Net/Dns.cs

@@ -416,6 +416,71 @@ private static bool ValidateAddressFamily(ref AddressFamily addressFamily, strin
             return true;
         }
 
+        private const string Localhost = "localhost";
+        private const string InvalidDomain = "invalid";
+
+        /// <summary>
+        /// Checks if the given host name matches a reserved name or is a subdomain of it.
+        /// For example, IsReservedName("foo.localhost", "localhost") returns true.
+        /// Also handles trailing dots: IsReservedName("foo.localhost.", "localhost") returns true.
+        /// Returns false for malformed hostnames (starting with dot or containing consecutive dots).
+        /// </summary>
+        private static bool IsReservedName(string hostName, string reservedName)
+        {
+            // Reject malformed hostnames - let OS resolver handle them (and reject them)
+            if (hostName.StartsWith('.') || hostName.Contains("..", StringComparison.Ordinal))
+            {
+                return false;
+            }
+
+            // Strip trailing dot if present (DNS root notation)
+            ReadOnlySpan<char> hostSpan = hostName.AsSpan();
+            if (hostSpan.EndsWith('.'))
+            {
+                hostSpan = hostSpan.Slice(0, hostSpan.Length - 1);
+            }
+
+            // Matches "reservedName" exactly, or "*.reservedName" (subdomain)
+            return hostSpan.EndsWith(reservedName, StringComparison.OrdinalIgnoreCase) &&
+                   (hostSpan.Length == reservedName.Length ||
+                    hostSpan[hostSpan.Length - reservedName.Length - 1] == '.');
+        }
+
+        /// <summary>
+        /// Checks if the given host name is a subdomain of localhost (e.g., "foo.localhost").
+        /// Plain "localhost" or "localhost." returns false.
+        /// </summary>
+        private static bool IsLocalhostSubdomain(string hostName)
+        {
+            // Strip trailing dot for length comparison
+            int length = hostName.Length;
+            if (hostName.EndsWith('.'))
+            {
+                length--;
+            }
+
+            // Must be longer than "localhost" (not just equal with trailing dot)
+            return length > Localhost.Length && IsReservedName(hostName, Localhost);
+        }
+
+        /// <summary>
+        /// Tries to handle RFC 6761 "invalid" domain names.
+        /// Returns true if the host name is an invalid domain (exception will be set).
+        /// </summary>
+        private static bool TryHandleRfc6761InvalidDomain(string hostName, out SocketException? exception)
+        {
+            // RFC 6761 Section 6.4: "invalid" and "*.invalid" must always return NXDOMAIN.
+            if (IsReservedName(hostName, InvalidDomain))
+            {
+                if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(hostName, "RFC 6761: Returning NXDOMAIN for 'invalid' domain");
+                exception = new SocketException((int)SocketError.HostNotFound);
+                return true;
+            }
+
+            exception = null;
+            return false;
+        }
+
         private static object GetHostEntryOrAddressesCore(string hostName, bool justAddresses, AddressFamily addressFamily, NameResolutionActivity? activityOrDefault = default)
         {
             ValidateHostName(hostName);
@@ -429,32 +494,66 @@ private static object GetHostEntryOrAddressesCore(string hostName, bool justAddr
             // NameResolutionActivity may have already been set if we're being called from RunAsync.
             NameResolutionActivity activity = activityOrDefault ?? NameResolutionTelemetry.Log.BeforeResolution(hostName);
 
-            object result;
+            // RFC 6761 Section 6.4: "invalid" domains must return NXDOMAIN.
+            if (TryHandleRfc6761InvalidDomain(hostName, out SocketException? invalidDomainException))
+            {
+                NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: null, exception: invalidDomainException);
+                throw invalidDomainException!;
+            }
+
+            bool fallbackToLocalhost = false;
+            object? result = null;
             try
             {
                 SocketError errorCode = NameResolutionPal.TryGetAddrInfo(hostName, justAddresses, addressFamily, out string? newHostName, out string[] aliases, out IPAddress[] addresses, out int nativeErrorCode);
 
                 if (errorCode != SocketError.Success)
                 {
-                    if (NetEventSource.Log.IsEnabled()) NetEventSource.Error(hostName, $"{hostName} DNS lookup failed with {errorCode}");
-                    throw CreateException(errorCode, nativeErrorCode);
+                    // RFC 6761 Section 6.3: If localhost subdomain fails, fall back to resolving plain "localhost".
+                    if (IsLocalhostSubdomain(hostName))
+                    {
+                        if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(hostName, "RFC 6761: Localhost subdomain resolution failed, falling back to 'localhost'");
+                        NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: null, exception: CreateException(errorCode, nativeErrorCode));
+                        fallbackToLocalhost = true;
+                    }
+                    else
+                    {
+                        if (NetEventSource.Log.IsEnabled()) NetEventSource.Error(hostName, $"{hostName} DNS lookup failed with {errorCode}");
+                        throw CreateException(errorCode, nativeErrorCode);
+                    }
+                }
+                else if (addresses.Length == 0 && IsLocalhostSubdomain(hostName))
+                {
+                    // RFC 6761 Section 6.3: If localhost subdomain returns empty addresses, fall back to plain "localhost".
+                    if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(hostName, "RFC 6761: Localhost subdomain returned empty, falling back to 'localhost'");
+                    NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: justAddresses ? addresses : (object)new IPHostEntry { AddressList = addresses, HostName = newHostName!, Aliases = aliases }, exception: null);
+                    fallbackToLocalhost = true;
                 }
 
-                result = justAddresses ? (object)
-                    addresses :
-                    new IPHostEntry
-                    {
-                        AddressList = addresses,
-                        HostName = newHostName!,
-                        Aliases = aliases
-                    };
+                if (!fallbackToLocalhost)
+                {
+                    result = justAddresses ? (object)
+                        addresses :
+                        new IPHostEntry
+                        {
+                            AddressList = addresses,
+                            HostName = newHostName!,
+                            Aliases = aliases
+                        };
+                }
             }
             catch (Exception ex) when (LogFailure(hostName, activity, ex))
             {
                 Debug.Fail("LogFailure should return false");
                 throw;
             }
 
+            if (fallbackToLocalhost)
+            {
+                return GetHostEntryOrAddressesCore(Localhost, justAddresses, addressFamily);
+            }
+
+            Debug.Assert(result is not null);
             NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: result);
 
             return result;
@@ -588,6 +687,23 @@ private static Task GetHostEntryOrAddressesCoreAsync(string hostName, bool justR
             }
             else
             {
+                // Validate hostname before any processing
+                ValidateHostName(hostName);
+
+                // RFC 6761 Section 6.4: "invalid" domains must return NXDOMAIN.
+                if (TryHandleRfc6761InvalidDomain(hostName, out SocketException? invalidDomainException))
+                {
+                    NameResolutionActivity activity = NameResolutionTelemetry.Log.BeforeResolution(hostName);
+                    NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: null, exception: invalidDomainException);
+                    return justAddresses ? (Task)
+                        Task.FromException<IPAddress[]>(invalidDomainException!) :
+                        Task.FromException<IPHostEntry>(invalidDomainException!);
+                }
+
+                // For localhost subdomains (RFC 6761 Section 6.3), we try the OS resolver first.
+                // If it fails or returns empty, we fall back to resolving plain "localhost".
+                // This fallback logic is handled in GetHostEntryOrAddressesCore and GetAddrInfoWithTelemetryAsync.
+
                 if (NameResolutionPal.SupportsGetAddrInfoAsync)
                 {
 #pragma warning disable CS0162 // Unreachable code detected -- SupportsGetAddrInfoAsync is a constant on *nix.
@@ -596,10 +712,11 @@ private static Task GetHostEntryOrAddressesCoreAsync(string hostName, bool justR
                     // instead of calling the synchronous version in the ThreadPool.
                     // If it fails, we will fall back to ThreadPool as well.
 
-                    ValidateHostName(hostName);
-
+                    // Always use the telemetry-enabled path for localhost subdomains to ensure fallback handling.
+                    // For other hostnames, use the non-telemetry path if diagnostics are disabled.
+                    bool isLocalhostSubdomain = IsLocalhostSubdomain(hostName);
                     Task? t;
-                    if (NameResolutionTelemetry.AnyDiagnosticsEnabled())
+                    if (NameResolutionTelemetry.AnyDiagnosticsEnabled() || isLocalhostSubdomain)
                     {
                         t = justAddresses
                             ? GetAddrInfoWithTelemetryAsync<IPAddress[]>(hostName, justAddresses, family, cancellationToken)
@@ -653,31 +770,69 @@ private static Task GetHostEntryOrAddressesCoreAsync(string hostName, bool justR
 
             if (task != null)
             {
-                return CompleteAsync(task, hostName, startingTimestamp);
+                bool isLocalhostSubdomain = IsLocalhostSubdomain(hostName);
+                return CompleteAsync(task, hostName, justAddresses, addressFamily, isLocalhostSubdomain, startingTimestamp, cancellationToken);
             }
 
             // If resolution even did not start don't bother with telemetry.
             // We will retry on thread-pool.
             return null;
 
-            static async Task<T> CompleteAsync(Task task, string hostName, long startingTimeStamp)
+            static async Task<T> CompleteAsync(Task task, string hostName, bool justAddresses, AddressFamily addressFamily, bool isLocalhostSubdomain, long startingTimeStamp, CancellationToken cancellationToken)
             {
                 NameResolutionActivity activity = NameResolutionTelemetry.Log.BeforeResolution(hostName, startingTimeStamp);
                 Exception? exception = null;
                 T? result = null;
+                bool fallbackOccurred = false;
                 try
                 {
                     result = await ((Task<T>)task).ConfigureAwait(false);
+
+                    // RFC 6761 Section 6.3: If localhost subdomain returns empty addresses, fall back to plain "localhost".
+                    if (isLocalhostSubdomain && result is IPAddress[] addresses && addresses.Length == 0)
+                    {
+                        if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(hostName, "RFC 6761: Localhost subdomain returned empty, falling back to 'localhost'");
+                        NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: result, exception: null);
+                        fallbackOccurred = true;
+
+                        // result is IPAddress[] so justAddresses is guaranteed true here.
+                        return await ((Task<T>)(Task)Dns.GetHostAddressesAsync(Localhost, addressFamily, cancellationToken)).ConfigureAwait(false);
+                    }
+
+                    if (isLocalhostSubdomain && result is IPHostEntry entry && entry.AddressList.Length == 0)
+                    {
+                        if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(hostName, "RFC 6761: Localhost subdomain returned empty, falling back to 'localhost'");
+                        NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: result, exception: null);
+                        fallbackOccurred = true;
+
+                        // result is IPHostEntry so justAddresses is guaranteed false here.
+                        return await ((Task<T>)(Task)Dns.GetHostEntryAsync(Localhost, addressFamily, cancellationToken)).ConfigureAwait(false);
+                    }
+
                     return result;
                 }
+                catch (SocketException ex) when (isLocalhostSubdomain && !fallbackOccurred)
+                {
+                    // RFC 6761 Section 6.3: If localhost subdomain fails, fall back to resolving plain "localhost".
+                    if (NetEventSource.Log.IsEnabled()) NetEventSource.Info(hostName, "RFC 6761: Localhost subdomain resolution failed, falling back to 'localhost'");
+                    NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: null, exception: ex);
+                    fallbackOccurred = true;
+
+                    return await ((Task<T>)(justAddresses
+                        ? (Task)Dns.GetHostAddressesAsync(Localhost, addressFamily, cancellationToken)
+                        : Dns.GetHostEntryAsync(Localhost, addressFamily, cancellationToken))).ConfigureAwait(false);
+                }
                 catch (Exception ex)
                 {
                     exception = ex;
                     throw;
                 }
                 finally
                 {
-                    NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: result, exception: exception);
+                    if (!fallbackOccurred)
+                    {
+                        NameResolutionTelemetry.Log.AfterResolution(hostName, activity, answer: result, exception: exception);
+                    }
                 }
             }
         }