Skip to content

402 - Pre PR #434

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Hirogen merged 4 commits into from
Jul 7, 2025
Merged

402 - Pre PR #434

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
204b0f8
Merge pull request #339 from gureedo/about-fix
Hirogen Mar 12, 2025
b7c088b
Merge pull request #361 from LogExperts/Development
Hirogen Mar 20, 2025
3d46e96
Merge pull request #442 from LogExperts/Development
Hirogen Jul 6, 2025
a39d287
Changed the LogFileReader constructor in order to better encapsulate …
Jul 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 64 additions & 92 deletions src/LogExpert.Core/Classes/Log/LogfileReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,38 +14,33 @@ public class LogfileReader : IAutoLogLineColumnizerCallback, IDisposable
{
#region Fields

private static readonly ILogger _logger = LogManager.GetCurrentClassLogger();

private readonly GetLogLineFx _logLineFx;
private static readonly Logger _logger = LogManager.GetCurrentClassLogger();

private readonly string _fileName;
private readonly int _max_buffers;
private readonly int _maxLinesPerBuffer;

private readonly object _monitor = new();
private readonly MultiFileOptions _multiFileOptions;
private readonly IPluginRegistry _pluginRegistry;
private readonly CancellationTokenSource _cts = new();
private readonly bool _useNewReader;

private IList<LogBuffer> _bufferList;
private ReaderWriterLock _bufferListLock;
private bool _contentDeleted;
private int _currLineCount;
private ReaderWriterLock _disposeLock;
private EncodingOptions _encodingOptions;
private long _fileLength;

private Task _garbageCollectorTask;
private Task _monitorTask;
private readonly CancellationTokenSource _cts = new();

private bool _isDeleted;
private bool _isFailModeCheckCallPending;
private bool _isFastFailOnGetLogLine;
private bool _isLineCountDirty = true;
private IList<ILogFileInfo> _logFileInfoList = [];
private Dictionary<int, LogBufferCacheEntry> _lruCacheDict;

private ReaderWriterLock _lruCacheDictLock;

private bool _shouldStop;
private bool _disposed;
private ILogFileInfo _watchedILogFileInfo;
Expand All @@ -54,85 +49,66 @@ public class LogfileReader : IAutoLogLineColumnizerCallback, IDisposable

#region cTor

public LogfileReader (string fileName, EncodingOptions encodingOptions, bool multiFile, int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions, IPluginRegistry pluginRegistry)
/// Public constructor for single file.
public LogfileReader (string fileName, EncodingOptions encodingOptions, bool multiFile, int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions, bool useNewReader, IPluginRegistry pluginRegistry)
: this([fileName], encodingOptions, multiFile, bufferCount, linesPerBuffer, multiFileOptions, useNewReader, pluginRegistry)
{
if (fileName == null)
{
return;
}

_fileName = fileName;
EncodingOptions = encodingOptions;
IsMultiFile = multiFile;
_max_buffers = bufferCount;
_maxLinesPerBuffer = linesPerBuffer;
_multiFileOptions = multiFileOptions;
_pluginRegistry = pluginRegistry;
_logLineFx = GetLogLineInternal;
_disposed = false;

InitLruBuffers();

if (multiFile)
{
var info = GetLogFileInfo(fileName);
RolloverFilenameHandler rolloverHandler = new(info, _multiFileOptions);
var nameList = rolloverHandler.GetNameList(_pluginRegistry);

ILogFileInfo fileInfo = null;
foreach (var name in nameList)
{
fileInfo = AddFile(name);
}

_watchedILogFileInfo = fileInfo; // last added file in the list is the watched file
}
else
{
_watchedILogFileInfo = AddFile(fileName);
}
}

StartGCThread();
/// Public constructor for multiple files.
public LogfileReader (string[] fileNames, EncodingOptions encodingOptions, int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions, bool useNewReader, IPluginRegistry pluginRegistry)
: this(fileNames, encodingOptions, true, bufferCount, linesPerBuffer, multiFileOptions, useNewReader, pluginRegistry)
{
// In this overload, we assume multiFile is always true.
}

public LogfileReader (string[] fileNames, EncodingOptions encodingOptions, int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions, IPluginRegistry pluginRegistry)
// Single private constructor that contains the common initialization logic.
private LogfileReader (string[] fileNames, EncodingOptions encodingOptions, bool multiFile, int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions, bool useNewReader, IPluginRegistry pluginRegistry)
{
// Validate input: at least one file must be provided.
if (fileNames == null || fileNames.Length < 1)
{
return;
throw new ArgumentException("Must provide at least one file.", nameof(fileNames));
}

_useNewReader = useNewReader;
EncodingOptions = encodingOptions;
IsMultiFile = true;
_max_buffers = bufferCount;
_maxLinesPerBuffer = linesPerBuffer;
_multiFileOptions = multiFileOptions;
_pluginRegistry = pluginRegistry;
_logLineFx = GetLogLineInternal;
_disposed = false;

InitLruBuffers();

ILogFileInfo fileInfo = null;
foreach (var name in fileNames)

IsMultiFile = multiFile || fileNames.Length == 1;
_fileName = fileNames[0];

IEnumerable<string> names = IsMultiFile
// For multi-file rollover mode: get rollover names.
? new RolloverFilenameHandler(GetLogFileInfo(_fileName), _multiFileOptions).GetNameList(_pluginRegistry)
: [_fileName];

foreach (var name in names)
{
fileInfo = AddFile(name);
}

if (IsMultiFile)
{
// Use the full name of the last file as _fileName.
_fileName = fileInfo.FullName;
}

_watchedILogFileInfo = fileInfo;
_fileName = fileInfo.FullName;

StartGCThread();
}

#endregion

#region Delegates

private delegate Task<ILogLine> GetLogLineFx (int lineNum);

#endregion

#region Events

public event EventHandler<LogEventArgs> FileSizeChanged;
Expand Down Expand Up @@ -165,7 +141,7 @@ public int LineCount

return _currLineCount;
}
set => _currLineCount = value;
private set => _currLineCount = value;
}

public bool IsMultiFile { get; }
Expand All @@ -174,13 +150,15 @@ public int LineCount

public long FileSize { get; private set; }

//TODO: Change to private field. No need for a property.
public bool IsXmlMode { get; set; }

//TODO: Change to private field. No need for a property.
public IXmlLogConfiguration XmlLogConfig { get; set; }

public IPreProcessColumnizer PreProcessColumnizer { get; set; }

public EncodingOptions EncodingOptions
private EncodingOptions EncodingOptions
{
get => _encodingOptions;
set
Expand All @@ -195,15 +173,14 @@ public EncodingOptions EncodingOptions
}
}

public bool UseNewReader { get; set; }

#endregion

#region Public methods

/// <summary>
/// Public for unit test reasons
/// </summary>
//TODO: Make this private
public void ReadFiles ()
{
FileSize = 0;
Expand Down Expand Up @@ -254,6 +231,7 @@ public void ReadFiles ()
/// Public for unit tests.
/// </summary>
/// <returns></returns>
//TODO: Make this private
public int ShiftBuffers ()
{
_logger.Info(CultureInfo.InvariantCulture, "ShiftBuffers() begin for {0}{1}", _fileName, IsMultiFile ? " (MultiFile)" : "");
Expand Down Expand Up @@ -335,10 +313,10 @@ public int ShiftBuffers ()
if (lostILogFileInfoList.Count > 0)
{
_logger.Info(CultureInfo.InvariantCulture, "Deleting buffers for lost files");
foreach (var ILogFileInfo in lostILogFileInfoList)
foreach (var logFileInfo in lostILogFileInfoList)
{
//this.ILogFileInfoList.Remove(ILogFileInfo);
var lastBuffer = DeleteBuffersForInfo(ILogFileInfo, false);
//this.ILogFileInfoList.Remove(logFileInfo);
var lastBuffer = DeleteBuffersForInfo(logFileInfo, false);
if (lastBuffer != null)
{
offset += lastBuffer.StartLine + lastBuffer.LineCount;
Expand Down Expand Up @@ -366,7 +344,7 @@ public int ShiftBuffers ()
foreach (var ILogFileInfo in readNewILogFileInfoList)
{
DeleteBuffersForInfo(ILogFileInfo, true);
//this.ILogFileInfoList.Remove(ILogFileInfo);
//this.ILogFileInfoList.Remove(logFileInfo);
}

_logger.Info(CultureInfo.InvariantCulture, "Deleting buffers for the watched file");
Expand All @@ -375,9 +353,9 @@ public int ShiftBuffers ()
_logger.Info(CultureInfo.InvariantCulture, "Re-Reading files");
foreach (var ILogFileInfo in readNewILogFileInfoList)
{
//ILogFileInfo.OpenFile();
//logFileInfo.OpenFile();
ReadToBufferList(ILogFileInfo, 0, LineCount);
//this.ILogFileInfoList.Add(ILogFileInfo);
//this.ILogFileInfoList.Add(logFileInfo);
newFileInfoList.Add(ILogFileInfo);
}

Expand Down Expand Up @@ -422,7 +400,7 @@ public async Task<ILogLine> GetLogLineWithWait (int lineNum)

if (!_isFastFailOnGetLogLine)
{
var task = Task.Run(() => _logLineFx(lineNum));
var task = Task.Run(() => GetLogLineInternal(lineNum));
if (task.Wait(WAIT_TIME))
{
result = task.Result;
Expand All @@ -440,7 +418,7 @@ public async Task<ILogLine> GetLogLineWithWait (int lineNum)
if (!_isFailModeCheckCallPending)
{
_isFailModeCheckCallPending = true;
var logLine = await _logLineFx(lineNum);
var logLine = await GetLogLineInternal(lineNum);
GetLineFinishedCallback(logLine);
}
}
Expand Down Expand Up @@ -927,7 +905,7 @@ private void ReadToBufferList (ILogFileInfo logFileInfo, long filePos, int start
using var fileStream = logFileInfo.OpenStream();
try
{
using var reader = GetLogStreamReader(fileStream, EncodingOptions, UseNewReader);
using var reader = GetLogStreamReader(fileStream, EncodingOptions, _useNewReader);
reader.Position = filePos;
_fileLength = logFileInfo.Length;

Expand Down Expand Up @@ -980,7 +958,6 @@ private void ReadToBufferList (ILogFileInfo logFileInfo, long filePos, int start

while (ReadLine(reader, logBuffer.StartLine + logBuffer.LineCount, logBuffer.StartLine + logBuffer.LineCount + droppedLines, out var line))
{
LogLine logLine = new();
if (_shouldStop)
{
Monitor.Exit(logBuffer);
Expand Down Expand Up @@ -1011,8 +988,7 @@ private void ReadToBufferList (ILogFileInfo logFileInfo, long filePos, int start
lineCount = 1;
}

logLine.FullLine = line;
logLine.LineNumber = logBuffer.StartLine + logBuffer.LineCount;
LogLine logLine = new(line, logBuffer.StartLine + logBuffer.LineCount);

logBuffer.AddLine(logLine, filePos);
filePos = reader.Position;
Expand Down Expand Up @@ -1319,7 +1295,7 @@ private void ReReadBuffer (LogBuffer logBuffer)

try
{
var reader = GetLogStreamReader(fileStream, EncodingOptions, UseNewReader);
var reader = GetLogStreamReader(fileStream, EncodingOptions, _useNewReader);

var filePos = logBuffer.StartPos;
reader.Position = logBuffer.StartPos;
Expand All @@ -1341,11 +1317,7 @@ private void ReReadBuffer (LogBuffer logBuffer)
continue;
}

LogLine logLine = new()
{
FullLine = line,
LineNumber = logBuffer.StartLine + logBuffer.LineCount
};
LogLine logLine = new(line, logBuffer.StartLine + logBuffer.LineCount);

logBuffer.AddLine(logLine, filePos);
filePos = reader.Position;
Expand Down Expand Up @@ -1578,7 +1550,6 @@ private void FireChangeEvent ()
if (IsMultiFile)
{
var offset = ShiftBuffers();
//this.currFileSize = newSize; // removed because ShiftBuffers() calls ReadToBuffer() which will set the actual read size
args.FileSize = newSize;
args.LineCount = LineCount;
args.IsRollover = true;
Expand Down Expand Up @@ -1785,6 +1756,8 @@ private void DumpBufferInfos (LogBuffer buffer)

#endregion

#region IDisposable Support

public void Dispose ()
{
Dispose(true);
Expand All @@ -1811,6 +1784,9 @@ protected virtual void Dispose (bool disposing)
Dispose(false);
}

#endregion IDisposable Support

#region Event Handlers
protected virtual void OnFileSizeChanged (LogEventArgs e)
{
FileSizeChanged?.Invoke(this, e);
Expand Down Expand Up @@ -1842,16 +1818,12 @@ protected virtual void OnRespawned ()
Respawned?.Invoke(this, EventArgs.Empty);
}

private class LogLine : ILogLine
{
#region Properties

public string FullLine { get; set; }
#endregion Event Handlers

public int LineNumber { get; set; }

string ITextValue.Text => FullLine;

#endregion
#region Records
private record LogLine (string FullLine, int LineNumber) : ILogLine
{
public string Text => FullLine;
}
}
#endregion Records
}
2 changes: 1 addition & 1 deletion src/LogExpert.Tests/BufferShiftTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public void TestShiftBuffers1 ()
};

PluginRegistry.PluginRegistry.Instance.Create(TestDirectory.FullName, 500);
LogfileReader reader = new(files.Last.Value, encodingOptions, true, 40, 50, options, PluginRegistry.PluginRegistry.Instance);
LogfileReader reader = new(files.Last.Value, encodingOptions, true, 40, 50, options, false, PluginRegistry.PluginRegistry.Instance);
reader.ReadFiles();

IList<ILogFileInfo> lil = reader.GetLogFileInfoList();
Expand Down
2 changes: 1 addition & 1 deletion src/LogExpert.Tests/CSVColumnizerTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ public void Instantiat_CSVFile_BuildCorrectColumnizer (string filename, string[]
{
CsvColumnizer.CsvColumnizer csvColumnizer = new();
var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, filename);
LogfileReader reader = new(path, new EncodingOptions(), false, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance);
LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), false, PluginRegistry.PluginRegistry.Instance);
reader.ReadFiles();
ILogLine line = reader.GetLogLine(0);
IColumnizedLogLine logline = new ColumnizedLogLine();
Expand Down
2 changes: 1 addition & 1 deletion src/LogExpert.Tests/ColumnizerPickerTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public void FindReplacementForAutoColumnizer_ValidTextFile_ReturnCorrectColumniz
string fileName, Type columnizerType)
{
var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, fileName);
LogfileReader reader = new(path, new EncodingOptions(), false, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance);
LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), false, PluginRegistry.PluginRegistry.Instance);
reader.ReadFiles();

Mock<ILogLineColumnizer> autoColumnizer = new();
Expand Down
2 changes: 1 addition & 1 deletion src/LogExpert.Tests/JsonColumnizerTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ public void GetColumnNames_HappyFile_ColumnNameMatches (string fileName, string
{
var jsonColumnizer = new JsonColumnizer.JsonColumnizer();
var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, fileName);
LogfileReader reader = new(path, new EncodingOptions(), false, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance);
LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), false, PluginRegistry.PluginRegistry.Instance);
reader.ReadFiles();

ILogLine line = reader.GetLogLine(0);
Expand Down
Loading
Loading