diff --git a/src/Microsoft.AspNet.Server.Kestrel/Http/Frame.cs b/src/Microsoft.AspNet.Server.Kestrel/Http/Frame.cs index 008e1a905..522feb36e 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Http/Frame.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Http/Frame.cs @@ -70,22 +70,25 @@ public abstract partial class Frame : FrameContext, IFrameControl private readonly Action _prepareRequest; private readonly string _pathBase; + protected readonly IStringCache _stringCache; public Frame(ConnectionContext context) - : this(context, remoteEndPoint: null, localEndPoint: null, prepareRequest: null) + : this(context, remoteEndPoint: null, localEndPoint: null, prepareRequest: null, stringCache: null) { } public Frame(ConnectionContext context, IPEndPoint remoteEndPoint, IPEndPoint localEndPoint, - Action prepareRequest) + Action prepareRequest, + IStringCache stringCache) : base(context) { _remoteEndPoint = remoteEndPoint; _localEndPoint = localEndPoint; _prepareRequest = prepareRequest; _pathBase = context.ServerAddress.PathBase; + _stringCache = stringCache; FrameControl = this; Reset(); @@ -702,7 +705,7 @@ protected bool TakeStartLine(SocketInput input) { return false; } - var method = begin.GetAsciiString(scan); + var method = begin.GetAsciiString(scan, _stringCache); scan.Take(); begin = scan; @@ -726,7 +729,7 @@ protected bool TakeStartLine(SocketInput input) { return false; } - queryString = begin.GetAsciiString(scan); + queryString = begin.GetAsciiString(scan, _stringCache); } scan.Take(); @@ -735,7 +738,7 @@ protected bool TakeStartLine(SocketInput input) { return false; } - var httpVersion = begin.GetAsciiString(scan); + var httpVersion = begin.GetAsciiString(scan, _stringCache); scan.Take(); if (scan.Take() != '\n') @@ -756,7 +759,7 @@ protected bool TakeStartLine(SocketInput input) else { // URI wasn't encoded, parse as ASCII - requestUrlPath = pathBegin.GetAsciiString(pathEnd); + requestUrlPath = pathBegin.GetAsciiString(pathEnd, _stringCache); } consumed = scan; @@ -809,7 +812,7 @@ private bool RequestUrlStartsWithPathBase(string requestUrl, out bool caseMatche return true; } - public static bool TakeMessageHeaders(SocketInput input, FrameRequestHeaders requestHeaders) + public static bool TakeMessageHeaders(SocketInput input, FrameRequestHeaders requestHeaders, IStringCache stringCache) { var scan = input.ConsumingStart(); var consumed = scan; @@ -900,7 +903,7 @@ public static bool TakeMessageHeaders(SocketInput input, FrameRequestHeaders req } var name = beginName.GetArraySegment(endName); - var value = beginValue.GetAsciiString(endValue); + var value = beginValue.GetAsciiString(endValue, stringCache); if (wrapping) { value = value.Replace("\r\n", " "); diff --git a/src/Microsoft.AspNet.Server.Kestrel/Http/FrameOfT.cs b/src/Microsoft.AspNet.Server.Kestrel/Http/FrameOfT.cs index 4410cdc8c..b6fe22c1d 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Http/FrameOfT.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Http/FrameOfT.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Microsoft.AspNet.Hosting.Server; using Microsoft.AspNet.Http.Features; +using Microsoft.AspNet.Server.Kestrel.Infrastructure; using Microsoft.Extensions.Logging; namespace Microsoft.AspNet.Server.Kestrel.Http @@ -16,7 +17,7 @@ public class Frame : Frame public Frame(IHttpApplication application, ConnectionContext context) - : this(application, context, remoteEndPoint: null, localEndPoint: null, prepareRequest: null) + : this(application, context, remoteEndPoint: null, localEndPoint: null, prepareRequest: null, stringCache: null) { } @@ -24,8 +25,9 @@ public Frame(IHttpApplication application, ConnectionContext context, IPEndPoint remoteEndPoint, IPEndPoint localEndPoint, - Action prepareRequest) - : base(context, remoteEndPoint, localEndPoint, prepareRequest) + Action prepareRequest, + IStringCache stringCache) + : base(context, remoteEndPoint, localEndPoint, prepareRequest, stringCache) { _application = application; } @@ -42,6 +44,8 @@ public override async Task RequestProcessingAsync() { while (!_requestProcessingStopping) { + _stringCache?.MarkStart(); + while (!_requestProcessingStopping && !TakeStartLine(SocketInput)) { if (SocketInput.RemoteIntakeFin) @@ -51,7 +55,7 @@ public override async Task RequestProcessingAsync() await SocketInput; } - while (!_requestProcessingStopping && !TakeMessageHeaders(SocketInput, _requestHeaders)) + while (!_requestProcessingStopping && !TakeMessageHeaders(SocketInput, _requestHeaders, _stringCache)) { if (SocketInput.RemoteIntakeFin) { diff --git a/src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs b/src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs index 64c5808ef..7659fbba1 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs @@ -17,6 +17,7 @@ public class SocketOutput : ISocketOutput private const int _maxPendingWrites = 3; private const int _maxBytesPreCompleted = 65536; private const int _initialTaskQueues = 64; + private const int _maxPooledWriteContexts = 32; private static WaitCallback _returnBlocks = (state) => ReturnBlocks((MemoryPoolBlock2)state); @@ -39,6 +40,7 @@ public class SocketOutput : ISocketOutput // This locks access to to all of the below fields private readonly object _contextLock = new object(); + private bool _isDisposed = false; // The number of write operations that have been scheduled so far // but have not completed. @@ -49,6 +51,7 @@ public class SocketOutput : ISocketOutput private WriteContext _nextWriteContext; private readonly Queue> _tasksPending; private readonly Queue> _tasksCompleted; + private readonly Queue _writeContextPool; public SocketOutput( KestrelThread thread, @@ -67,6 +70,7 @@ public SocketOutput( _threadPool = threadPool; _tasksPending = new Queue>(_initialTaskQueues); _tasksCompleted = new Queue>(_initialTaskQueues); + _writeContextPool = new Queue(_maxPooledWriteContexts); _head = memory.Lease(); _tail = _head; @@ -93,7 +97,14 @@ public Task WriteAsync( { if (_nextWriteContext == null) { - _nextWriteContext = new WriteContext(this); + if (_writeContextPool.Count > 0) + { + _nextWriteContext = _writeContextPool.Dequeue(); + } + else + { + _nextWriteContext = new WriteContext(this); + } } if (socketShutdownSend) @@ -272,9 +283,12 @@ private void WriteAllPending() } // This is called on the libuv event loop - private void OnWriteCompleted(int bytesWritten, int status, Exception error) + private void OnWriteCompleted(WriteContext writeContext) { - _log.ConnectionWriteCallback(_connectionId, status); + var bytesWritten = writeContext.ByteCount; + var status = writeContext.WriteStatus; + var error = writeContext.WriteError; + if (error != null) { @@ -288,6 +302,7 @@ private void OnWriteCompleted(int bytesWritten, int status, Exception error) lock (_contextLock) { + PoolWriteContext(writeContext); if (_nextWriteContext != null) { scheduleWrite = true; @@ -330,11 +345,11 @@ private void OnWriteCompleted(int bytesWritten, int status, Exception error) } } + _log.ConnectionWriteCallback(_connectionId, status); + if (scheduleWrite) { - // ScheduleWrite(); - // on right thread, fairness issues? - WriteAllPending(); + ScheduleWrite(); } _tasksCompleted.Clear(); @@ -368,6 +383,32 @@ private void ReturnAllBlocks() } } + private void PoolWriteContext(WriteContext writeContext) + { + // called inside _contextLock + if (!_isDisposed && _writeContextPool.Count < _maxPooledWriteContexts) + { + writeContext.Reset(); + _writeContextPool.Enqueue(writeContext); + } + else + { + writeContext.Dispose(); + } + } + + private void Dispose() + { + lock (_contextLock) + { + _isDisposed = true; + while (_writeContextPool.Count > 0) + { + _writeContextPool.Dequeue().Dispose(); + } + } + } + void ISocketOutput.Write(ArraySegment buffer, bool immediate) { var task = WriteAsync(buffer, immediate); @@ -387,14 +428,14 @@ Task ISocketOutput.WriteAsync(ArraySegment buffer, bool immediate, Cancell return WriteAsync(buffer, immediate); } - private class WriteContext + private class WriteContext : IDisposable { private static WaitCallback _returnWrittenBlocks = (state) => ReturnWrittenBlocks((MemoryPoolBlock2)state); private MemoryPoolIterator2 _lockedStart; private MemoryPoolIterator2 _lockedEnd; private int _bufferCount; - private int _byteCount; + public int ByteCount; public SocketOutput Self; @@ -404,11 +445,15 @@ private class WriteContext public int WriteStatus; public Exception WriteError; + private UvWriteReq _writeReq; + public int ShutdownSendStatus; public WriteContext(SocketOutput self) { Self = self; + _writeReq = new UvWriteReq(Self._log); + _writeReq.Init(Self._thread.Loop); } /// @@ -418,18 +463,19 @@ public void DoWriteIfNeeded() { LockWrite(); - if (_byteCount == 0 || Self._socket.IsClosed) + if (ByteCount == 0 || Self._socket.IsClosed) { DoShutdownIfNeeded(); return; } - var writeReq = new UvWriteReq(Self._log); - writeReq.Init(Self._thread.Loop); + // Sample values locally in case write completes inline + // to allow block to be Reset and still complete this function + var lockedEndBlock = _lockedEnd.Block; + var lockedEndIndex = _lockedEnd.Index; - writeReq.Write(Self._socket, _lockedStart, _lockedEnd, _bufferCount, (_writeReq, status, error, state) => + _writeReq.Write(Self._socket, _lockedStart, _lockedEnd, _bufferCount, (_writeReq, status, error, state) => { - _writeReq.Dispose(); var _this = (WriteContext)state; _this.ScheduleReturnFullyWrittenBlocks(); _this.WriteStatus = status; @@ -437,8 +483,8 @@ public void DoWriteIfNeeded() _this.DoShutdownIfNeeded(); }, this); - Self._head = _lockedEnd.Block; - Self._head.Start = _lockedEnd.Index; + Self._head = lockedEndBlock; + Self._head.Start = lockedEndIndex; } /// @@ -471,21 +517,28 @@ public void DoShutdownIfNeeded() /// public void DoDisconnectIfNeeded() { - if (SocketDisconnect == false || Self._socket.IsClosed) + if (SocketDisconnect == false) + { + Complete(); + return; + } + else if (Self._socket.IsClosed) { + Self.Dispose(); Complete(); return; } Self._socket.Dispose(); Self.ReturnAllBlocks(); + Self.Dispose(); Self._log.ConnectionStop(Self._connectionId); Complete(); } public void Complete() { - Self.OnWriteCompleted(_byteCount, WriteStatus, WriteError); + Self.OnWriteCompleted(this); } private void ScheduleReturnFullyWrittenBlocks() @@ -537,23 +590,44 @@ private void LockWrite() if (_lockedStart.Block == _lockedEnd.Block) { - _byteCount = _lockedEnd.Index - _lockedStart.Index; + ByteCount = _lockedEnd.Index - _lockedStart.Index; _bufferCount = 1; return; } - _byteCount = _lockedStart.Block.Data.Offset + _lockedStart.Block.Data.Count - _lockedStart.Index; + ByteCount = _lockedStart.Block.Data.Offset + _lockedStart.Block.Data.Count - _lockedStart.Index; _bufferCount = 1; for (var block = _lockedStart.Block.Next; block != _lockedEnd.Block; block = block.Next) { - _byteCount += block.Data.Count; + ByteCount += block.Data.Count; _bufferCount++; } - _byteCount += _lockedEnd.Index - _lockedEnd.Block.Data.Offset; + ByteCount += _lockedEnd.Index - _lockedEnd.Block.Data.Offset; _bufferCount++; } + + public void Reset() + { + _lockedStart = default(MemoryPoolIterator2); + _lockedEnd = default(MemoryPoolIterator2); + _bufferCount = 0; + ByteCount = 0; + + SocketShutdownSend = false; + SocketDisconnect = false; + + WriteStatus = 0; + WriteError = null; + + ShutdownSendStatus = 0; + } + + public void Dispose() + { + _writeReq.Dispose(); + } } } } diff --git a/src/Microsoft.AspNet.Server.Kestrel/IKestrelServerInformation.cs b/src/Microsoft.AspNet.Server.Kestrel/IKestrelServerInformation.cs index 870fe2e53..fa12d548a 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/IKestrelServerInformation.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/IKestrelServerInformation.cs @@ -11,6 +11,12 @@ public interface IKestrelServerInformation bool NoDelay { get; set; } + bool StringCacheOnConnection { get; set; } + + int StringCacheMaxStrings { get; set; } + + int StringCacheMaxStringLength { get; set; } + IConnectionFilter ConnectionFilter { get; set; } } } diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/IStringCache.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/IStringCache.cs new file mode 100644 index 000000000..435617eab --- /dev/null +++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/IStringCache.cs @@ -0,0 +1,8 @@ +namespace Microsoft.AspNet.Server.Kestrel.Infrastructure +{ + public interface IStringCache + { + void MarkStart(); + unsafe string GetString(char* data, uint hash, int length); + } +} \ No newline at end of file diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/KestrelThread.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/KestrelThread.cs index ee43b9f60..de603b445 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/KestrelThread.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/KestrelThread.cs @@ -18,16 +18,21 @@ namespace Microsoft.AspNet.Server.Kestrel /// public class KestrelThread { + // maximum times the work queues swapped and are processed in a single pass + // as completing a task may immediately have write data to put on the network + // otherwise it needs to wait till the next pass of the libuv loop + private const int _maxLoops = 8; + private static Action _threadCallbackAdapter = (callback, state) => ((Action)callback).Invoke((KestrelThread)state); private KestrelEngine _engine; private readonly IApplicationLifetime _appLifetime; private Thread _thread; private UvLoopHandle _loop; private UvAsyncHandle _post; - private Queue _workAdding = new Queue(); - private Queue _workRunning = new Queue(); - private Queue _closeHandleAdding = new Queue(); - private Queue _closeHandleRunning = new Queue(); + private Queue _workAdding = new Queue(1024); + private Queue _workRunning = new Queue(1024); + private Queue _closeHandleAdding = new Queue(256); + private Queue _closeHandleRunning = new Queue(256); private object _workSync = new Object(); private bool _stopImmediate = false; private bool _initCompleted = false; @@ -249,11 +254,17 @@ private void ThreadStart(object parameter) private void OnPost() { - DoPostWork(); - DoPostCloseHandle(); + var loopsRemaining = _maxLoops; + bool wasWork; + do + { + wasWork = DoPostWork(); + wasWork = DoPostCloseHandle() || wasWork; + loopsRemaining--; + } while (wasWork && loopsRemaining > 0); } - private void DoPostWork() + private bool DoPostWork() { Queue queue; lock (_workSync) @@ -262,6 +273,9 @@ private void DoPostWork() _workAdding = _workRunning; _workRunning = queue; } + + bool wasWork = queue.Count > 0; + while (queue.Count != 0) { var work = queue.Dequeue(); @@ -286,8 +300,10 @@ private void DoPostWork() } } } + + return wasWork; } - private void DoPostCloseHandle() + private bool DoPostCloseHandle() { Queue queue; lock (_workSync) @@ -296,6 +312,9 @@ private void DoPostCloseHandle() _closeHandleAdding = _closeHandleRunning; _closeHandleRunning = queue; } + + bool wasWork = queue.Count > 0; + while (queue.Count != 0) { var closeHandle = queue.Dequeue(); @@ -309,6 +328,8 @@ private void DoPostCloseHandle() throw; } } + + return wasWork; } private struct Work diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2Extensions.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2Extensions.cs index d1f83c3d6..c5f2850c2 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2Extensions.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2Extensions.cs @@ -11,80 +11,128 @@ public static class MemoryPoolIterator2Extensions private const int _maxStackAllocBytes = 16384; private static Encoding _utf8 = Encoding.UTF8; + private static uint _startHash; - private static unsafe string GetAsciiStringStack(byte[] input, int inputOffset, int length) + static MemoryPoolIterator2Extensions() + { + using (var rnd = System.Security.Cryptography.RandomNumberGenerator.Create()) + { + var randomBytes = new byte[8]; + rnd.GetBytes(randomBytes); + _startHash = + (randomBytes[0]) | + (((uint)randomBytes[1]) << 8) | + (((uint)randomBytes[2]) << 16) | + (((uint)randomBytes[3]) << 24); + } + } + + private static unsafe string GetAsciiStringStack(byte* input, int length, IStringCache stringCache) { // avoid declaring other local vars, or doing work with stackalloc // to prevent the .locals init cil flag , see: https://github.com/dotnet/coreclr/issues/1279 char* output = stackalloc char[length]; - return GetAsciiStringImplementation(output, input, inputOffset, length); + return GetAsciiStringImplementation(output, input, length, stringCache); } - private static unsafe string GetAsciiStringImplementation(char* output, byte[] input, int inputOffset, int length) + private static unsafe string GetAsciiStringImplementation(char* outputStart, byte* input, int length, IStringCache stringCache) { - for (var i = 0; i < length; i++) + var hash = _startHash; + + var output = outputStart; + var i = 0; + var lengthMinusSpan = length - 3; + for (; i < lengthMinusSpan; i += 4) + { + // span hashing with fix https://github.com/dotnet/corefxlab/pull/455 + hash = hash * 31 + *((uint*)input); + + *(output) = (char)*(input); + *(output + 1) = (char)*(input + 1); + *(output + 2) = (char)*(input + 2); + *(output + 3) = (char)*(input + 3); + output += 4; + input += 4; + } + for (; i < length; i++) { - output[i] = (char)input[inputOffset + i]; + hash = hash * 31 + *((char*)input); + *(output++) = (char)*(input++); } - return new string(output, 0, length); + return stringCache?.GetString(outputStart, hash, length) ?? new string(outputStart, 0, length); } - private static unsafe string GetAsciiStringStack(MemoryPoolBlock2 start, MemoryPoolIterator2 end, int inputOffset, int length) + private static unsafe string GetAsciiStringStack(MemoryPoolBlock2 start, MemoryPoolIterator2 end, int inputOffset, int length, IStringCache stringCache) { // avoid declaring other local vars, or doing work with stackalloc // to prevent the .locals init cil flag , see: https://github.com/dotnet/coreclr/issues/1279 char* output = stackalloc char[length]; - return GetAsciiStringImplementation(output, start, end, inputOffset, length); + return GetAsciiStringImplementation(output, start, end, inputOffset, length, stringCache); } - private unsafe static string GetAsciiStringHeap(MemoryPoolBlock2 start, MemoryPoolIterator2 end, int inputOffset, int length) + private unsafe static string GetAsciiStringHeap(MemoryPoolBlock2 start, MemoryPoolIterator2 end, int inputOffset, int length, IStringCache stringCache) { var buffer = new char[length]; fixed (char* output = buffer) { - return GetAsciiStringImplementation(output, start, end, inputOffset, length); + return GetAsciiStringImplementation(output, start, end, inputOffset, length, stringCache); } } - private static unsafe string GetAsciiStringImplementation(char* output, MemoryPoolBlock2 start, MemoryPoolIterator2 end, int inputOffset, int length) + private static unsafe string GetAsciiStringImplementation(char* outputStart, MemoryPoolBlock2 start, MemoryPoolIterator2 end, int inputOffset, int length, IStringCache stringCache) { - var outputOffset = 0; + var hash = _startHash; + + var output = outputStart; + var block = start; var remaining = length; var endBlock = end.Block; var endIndex = end.Index; - while (true) + while (remaining > 0) { int following = (block != endBlock ? block.End : endIndex) - inputOffset; if (following > 0) { - var input = block.Array; - for (var i = 0; i < following; i++) + fixed (byte* blockStart = block.Array) { - output[i + outputOffset] = (char)input[i + inputOffset]; - } + var input = blockStart + inputOffset; + var i = 0; + var followingMinusSpan = following - 3; + for (; i < followingMinusSpan; i += 4) + { + // span hashing with fix https://github.com/dotnet/corefxlab/pull/455 + hash = hash * 31 + *((uint*)input); + *(output) = (char)*(input); + *(output + 1) = (char)*(input + 1); + *(output + 2) = (char)*(input + 2); + *(output + 3) = (char)*(input + 3); + output += 4; + input += 4; + } + for (; i < following; i++) + { + hash = hash * 31 + *((char*)input); + *(output++) = (char)*(input++); + } + } remaining -= following; - outputOffset += following; - } - - if (remaining == 0) - { - return new string(output, 0, length); } block = block.Next; - inputOffset = block.Start; + inputOffset = block?.Start??0; } + return stringCache?.GetString(outputStart, hash, length) ?? new string(outputStart, 0, length); } - public static string GetAsciiString(this MemoryPoolIterator2 start, MemoryPoolIterator2 end) + public unsafe static string GetAsciiString(this MemoryPoolIterator2 start, MemoryPoolIterator2 end, IStringCache stringCache) { if (start.IsDefault || end.IsDefault) { @@ -93,20 +141,28 @@ public static string GetAsciiString(this MemoryPoolIterator2 start, MemoryPoolIt var length = start.GetLength(end); + if (length <= 0) + { + return default(string); + } + // Bytes out of the range of ascii are treated as "opaque data" // and kept in string as a char value that casts to same input byte value // https://tools.ietf.org/html/rfc7230#section-3.2.4 if (end.Block == start.Block) { - return GetAsciiStringStack(start.Block.Array, start.Index, length); + fixed (byte* input = start.Block.Array) + { + return GetAsciiStringStack(input + start.Index, length, stringCache); + } } if (length > _maxStackAllocBytes) { - return GetAsciiStringHeap(start.Block, end, start.Index, length); + return GetAsciiStringHeap(start.Block, end, start.Index, length, stringCache); } - return GetAsciiStringStack(start.Block, end, start.Index, length); + return GetAsciiStringStack(start.Block, end, start.Index, length, stringCache); } public static string GetUtf8String(this MemoryPoolIterator2 start, MemoryPoolIterator2 end) @@ -120,9 +176,14 @@ public static string GetUtf8String(this MemoryPoolIterator2 start, MemoryPoolIte return _utf8.GetString(start.Block.Array, start.Index, end.Index - start.Index); } - var decoder = _utf8.GetDecoder(); - var length = start.GetLength(end); + + if (length <= 0) + { + return default(string); + } + + var decoder = _utf8.GetDecoder(); var charLength = length * 2; var chars = new char[charLength]; var charIndex = 0; diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/StringCache.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/StringCache.cs new file mode 100644 index 000000000..c611e6e03 --- /dev/null +++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/StringCache.cs @@ -0,0 +1,128 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.AspNet.Server.Kestrel.Infrastructure +{ + public class StringCache : IStringCache + { + private int _maxCached; + private int _maxCachedStringLength; + + private readonly uint[] _hashes; + private readonly int[] _lastUse; + private readonly string[] _strings; + + private int _currentUse = 0; + + // x64 int array byte size (28 + length * 4) rounded up to 8 bytes + // x86 int array byte size (12 + length * 4) rounded up to 4 bytes + // Array of 25 ints is 2 consecutive cache lines on x64; second prefetched + // Array of 9 ints is 1 cache line on x64 + public StringCache() : this(25, 256) + { + } + + public StringCache(int maxCached, int maxCachedStringLength) + { + _maxCached = maxCached; + _maxCachedStringLength = maxCachedStringLength; + _hashes = new uint[maxCached]; + _lastUse = new int[maxCached]; + _strings = new string[maxCached]; + } + + public void MarkStart() + { + _currentUse++; + } + + public unsafe string GetString(char* data, uint hash, int length) + { + if (length > _maxCachedStringLength) + { + return new string(data, 0, length); + } + + int oldestEntry = int.MaxValue; + int oldestIndex = 0; + + for (var i = 0; i < _maxCached; i++) + { + var usage = _lastUse[i]; + if (oldestEntry > usage) + { + oldestEntry = usage; + oldestIndex = i; + } + + if (hash == _hashes[i]) + { + var cachedString = _strings[i]; + if (cachedString.Length != length) + { +#if DEBUG + Console.WriteLine($"{nameof(StringCache)} Collision differing lengths {cachedString.Length} and {length}"); +#endif + continue; + } + + fixed(char* cs = cachedString) + { + var cached = cs; + var potential = data; + + var c = 0; + var lengthMinusSpan = length - 3; + for (; c < lengthMinusSpan; c += 4) + { + if( + *(cached) != *(potential) || + *(cached + 1) != *(potential + 1) || + *(cached + 2) != *(potential + 2) || + *(cached + 3) != *(potential + 3) + ) + { +#if DEBUG + Console.WriteLine($"{nameof(StringCache)} Collision same length, differing strings"); +#endif + continue; + } + cached += 4; + potential += 4; + } + for (; c < length; c++) + { + if (*(cached++) != *(potential++)) + { +#if DEBUG + Console.WriteLine($"{nameof(StringCache)} Collision same length, differing strings"); +#endif + continue; + } + } + } + + _lastUse[i] = _currentUse; + // same string + return cachedString; + } + } + + var value = new string(data, 0, length); +#if DEBUG + if (_lastUse[oldestIndex] != 0) + { + Console.WriteLine($"{nameof(StringCache)} Evict: {_strings[oldestIndex]} {_lastUse[oldestIndex]} {_hashes[oldestIndex]}"); + Console.WriteLine($"{nameof(StringCache)} New: {value} {_currentUse} {hash}"); + } +#endif + _lastUse[oldestIndex] = _currentUse; + _hashes[oldestIndex] = hash; + _strings[oldestIndex] = value; + + return value; + } + } +} diff --git a/src/Microsoft.AspNet.Server.Kestrel/KestrelServer.cs b/src/Microsoft.AspNet.Server.Kestrel/KestrelServer.cs index 3a1295de7..53e8cfa45 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/KestrelServer.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/KestrelServer.cs @@ -60,7 +60,15 @@ public void Start(IHttpApplication application) { FrameFactory = (context, remoteEP, localEP, prepareRequest) => { - return new Frame(application, context, remoteEP, localEP, prepareRequest); + return new Frame( + application, + context, + remoteEP, + localEP, + prepareRequest, + information.StringCacheOnConnection ? + new StringCache(information.StringCacheMaxStrings, information.StringCacheMaxStringLength) : + null); }, AppLifetime = _applicationLifetime, Log = trace, diff --git a/src/Microsoft.AspNet.Server.Kestrel/KestrelServerInformation.cs b/src/Microsoft.AspNet.Server.Kestrel/KestrelServerInformation.cs index ec31f4fbf..e087aa4b9 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/KestrelServerInformation.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/KestrelServerInformation.cs @@ -16,6 +16,8 @@ public KestrelServerInformation(IConfiguration configuration, int threadCount) Addresses = GetAddresses(configuration); ThreadCount = threadCount; NoDelay = true; + + ConfigureStringCache(configuration); } public ICollection Addresses { get; } @@ -24,6 +26,12 @@ public KestrelServerInformation(IConfiguration configuration, int threadCount) public bool NoDelay { get; set; } + public bool StringCacheOnConnection { get; set; } + + public int StringCacheMaxStrings { get; set; } + + public int StringCacheMaxStringLength { get; set; } + public IConnectionFilter ConnectionFilter { get; set; } private static ICollection GetAddresses(IConfiguration configuration) @@ -39,5 +47,48 @@ private static ICollection GetAddresses(IConfiguration configuration) return addresses; } + + private void ConfigureStringCache(IConfiguration configuration) + { + bool stringCacheOnConnection; + if (bool.TryParse(configuration["server.stringCacheOnConnection"], out stringCacheOnConnection)) + { + StringCacheOnConnection = stringCacheOnConnection; + } + else + { + StringCacheOnConnection = true; + } + int stringCacheMaxStrings; + if (StringCacheOnConnection && int.TryParse(configuration["server.stringCacheMaxStrings"], out stringCacheMaxStrings)) + { + if (stringCacheMaxStrings <= 0) + { + throw new ArgumentOutOfRangeException(nameof(stringCacheMaxStrings), + stringCacheMaxStrings, + "StringCacheMaxStrings must be positive."); + } + StringCacheMaxStrings = stringCacheMaxStrings; + } + else + { + StringCacheMaxStrings = 25; + } + int stringCacheMaxStringLength; + if (StringCacheOnConnection && int.TryParse(configuration["server.stringCacheMaxStringLength"], out stringCacheMaxStringLength)) + { + if (stringCacheMaxStringLength <= 0) + { + throw new ArgumentOutOfRangeException(nameof(stringCacheMaxStringLength), + stringCacheMaxStringLength, + "StringCacheMaxStringLength must be positive."); + } + StringCacheMaxStringLength = stringCacheMaxStringLength; + } + else + { + StringCacheMaxStringLength = 256; + } + } } } diff --git a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteReq.cs b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteReq.cs index 2695036f3..21a1aaf77 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteReq.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteReq.cs @@ -14,7 +14,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking /// public class UvWriteReq : UvRequest { - private readonly static Libuv.uv_write_cb _uv_write_cb = UvWriteCb; + private readonly static Libuv.uv_write_cb _uv_write_cb = (IntPtr ptr, int status) => UvWriteCb(ptr, status); private IntPtr _bufs; @@ -22,7 +22,7 @@ public class UvWriteReq : UvRequest private object _state; private const int BUFFER_COUNT = 4; - private List _pins = new List(); + private List _pins = new List(BUFFER_COUNT + 1); public UvWriteReq(IKestrelTrace logger) : base(logger) { diff --git a/test/Microsoft.AspNet.Server.KestrelTests/AsciiDecoder.cs b/test/Microsoft.AspNet.Server.KestrelTests/AsciiDecoder.cs index 26c034d2d..0ffa94958 100644 --- a/test/Microsoft.AspNet.Server.KestrelTests/AsciiDecoder.cs +++ b/test/Microsoft.AspNet.Server.KestrelTests/AsciiDecoder.cs @@ -13,6 +13,7 @@ public class AsciiDecoderTests [Fact] private void FullByteRangeSupported() { + var stringCache = new StringCache(); var byteRange = Enumerable.Range(0, 255).Select(x => (byte)x).ToArray(); var mem = MemoryPoolBlock2.Create(new ArraySegment(byteRange), IntPtr.Zero, null, null); @@ -21,7 +22,7 @@ private void FullByteRangeSupported() var begin = mem.GetIterator(); var end = GetIterator(begin, byteRange.Length); - var s = begin.GetAsciiString(end); + var s = begin.GetAsciiString(end, stringCache); Assert.Equal(s.Length, byteRange.Length); @@ -37,6 +38,7 @@ private void FullByteRangeSupported() [Fact] private void MultiBlockProducesCorrectResults() { + var stringCache = new StringCache(); var byteRange = Enumerable.Range(0, 512 + 64).Select(x => (byte)x).ToArray(); var expectedByteRange = byteRange .Concat(byteRange) @@ -60,7 +62,7 @@ private void MultiBlockProducesCorrectResults() var begin = mem0.GetIterator(); var end = GetIterator(begin, expectedByteRange.Length); - var s = begin.GetAsciiString(end); + var s = begin.GetAsciiString(end, stringCache); Assert.Equal(s.Length, expectedByteRange.Length); @@ -76,6 +78,7 @@ private void MultiBlockProducesCorrectResults() [Fact] private void HeapAllocationProducesCorrectResults() { + var stringCache = new StringCache(); var byteRange = Enumerable.Range(0, 16384 + 64).Select(x => (byte)x).ToArray(); var expectedByteRange = byteRange.Concat(byteRange).ToArray(); @@ -89,7 +92,7 @@ private void HeapAllocationProducesCorrectResults() var begin = mem0.GetIterator(); var end = GetIterator(begin, expectedByteRange.Length); - var s = begin.GetAsciiString(end); + var s = begin.GetAsciiString(end, stringCache); Assert.Equal(s.Length, expectedByteRange.Length); diff --git a/test/Microsoft.AspNet.Server.KestrelTests/FrameTests.cs b/test/Microsoft.AspNet.Server.KestrelTests/FrameTests.cs index cc9bc2661..4f1a45b08 100644 --- a/test/Microsoft.AspNet.Server.KestrelTests/FrameTests.cs +++ b/test/Microsoft.AspNet.Server.KestrelTests/FrameTests.cs @@ -50,6 +50,7 @@ public void ChunkedPrefixMustBeHexCrLfWithoutLeadingZeros(int dataCount, string public void EmptyHeaderValuesCanBeParsed(string rawHeaders, int numHeaders) { var trace = new KestrelTrace(new TestKestrelTrace()); + var stringCache = new StringCache(); var ltp = new LoggingThreadPool(trace); var socketInput = new SocketInput(new MemoryPool2(), ltp); var headerCollection = new FrameRequestHeaders(); @@ -59,7 +60,7 @@ public void EmptyHeaderValuesCanBeParsed(string rawHeaders, int numHeaders) Buffer.BlockCopy(headerArray, 0, inputBuffer.Data.Array, inputBuffer.Data.Offset, headerArray.Length); socketInput.IncomingComplete(headerArray.Length, null); - var success = Frame.TakeMessageHeaders(socketInput, headerCollection); + var success = Frame.TakeMessageHeaders(socketInput, headerCollection, stringCache); Assert.True(success); Assert.Equal(numHeaders, headerCollection.Count()); diff --git a/test/Microsoft.AspNet.Server.KestrelTests/TestServer.cs b/test/Microsoft.AspNet.Server.KestrelTests/TestServer.cs index 42efcb379..7271855c2 100644 --- a/test/Microsoft.AspNet.Server.KestrelTests/TestServer.cs +++ b/test/Microsoft.AspNet.Server.KestrelTests/TestServer.cs @@ -35,7 +35,7 @@ public void Create(RequestDelegate app, ServiceContext context, string serverAdd { context.FrameFactory = (connectionContext, remoteEP, localEP, prepareRequest) => { - return new Frame(new DummyApplication(app), connectionContext, remoteEP, localEP, prepareRequest); + return new Frame(new DummyApplication(app), connectionContext, remoteEP, localEP, prepareRequest, stringCache: null); }; _engine = new KestrelEngine(context); _engine.Start(1); diff --git a/test/Microsoft.AspNet.Server.KestrelTests/TestServiceContext.cs b/test/Microsoft.AspNet.Server.KestrelTests/TestServiceContext.cs index 54dfc3c3b..9ffcbacff 100644 --- a/test/Microsoft.AspNet.Server.KestrelTests/TestServiceContext.cs +++ b/test/Microsoft.AspNet.Server.KestrelTests/TestServiceContext.cs @@ -31,7 +31,7 @@ public RequestDelegate App _app = value; FrameFactory = (connectionContext, remoteEP, localEP, prepareRequest) => { - return new Frame(new DummyApplication(_app), connectionContext, remoteEP, localEP, prepareRequest); + return new Frame(new DummyApplication(_app), connectionContext, remoteEP, localEP, prepareRequest, new StringCache(25, 256)); }; } }