diff --git a/src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs b/src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs index c0c7d21e76..893754c2d0 100644 --- a/src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs +++ b/src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs @@ -34,6 +34,12 @@ public record FetchedCrossLinks /// public FrozenSet? CodexRepositories { get; init; } + /// + /// True when all declared repositories resolved without falling back to placeholder data. + /// When false, callers should avoid caching so a subsequent reload retries the fetch. + /// + public bool IsComplete { get; init; } = true; + public static FetchedCrossLinks Empty { get; } = new() { DeclaredRepositories = [], diff --git a/src/Elastic.Documentation.Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs b/src/Elastic.Documentation.Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs index 0af5a856b4..a72f789ac6 100644 --- a/src/Elastic.Documentation.Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs +++ b/src/Elastic.Documentation.Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs @@ -33,30 +33,36 @@ public override async Task FetchCrossLinks(Cancel ctx) var publicReader = linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous(); var useDualRegistry = configuration.Registry != DocSetRegistry.Public && _codexReader is not null; + // Fetch each registry once up front so per-repository lookups don't trigger N S3 round-trips. + var publicRegistry = await TryGetRegistry(publicReader, ctx); + var codexRegistry = useDualRegistry ? await TryGetRegistry(_codexReader!, ctx) : null; + var hadFetchFailures = false; + foreach (var entry in configuration.CrossLinkEntries) { _ = declaredRepositories.Add(entry.Repository); var isCodexEntry = useDualRegistry && entry.Registry != DocSetRegistry.Public; var reader = isCodexEntry ? _codexReader! : publicReader; + var registry = isCodexEntry ? codexRegistry : publicRegistry; if (isCodexEntry) _ = codexRepositories.Add(entry.Repository); try { - var linkReference = await FetchCrossLinksFromReader(reader, entry.Repository, this, ctx); + if (registry is null || !registry.Repositories.TryGetValue(entry.Repository, out var repoBranches)) + throw new Exception($"Repository {entry.Repository} not found in link index"); + + var linkIndexEntry = GetNextContentSourceLinkIndexEntry(repoBranches, entry.Repository); + var linkReference = await FetchLinkIndexEntryFromReader(reader, entry.Repository, linkIndexEntry, ctx); + linkReferences.Add(entry.Repository, linkReference); + linkIndexEntries.Add(entry.Repository, linkIndexEntry); registryUrlsByRepository[entry.Repository] = reader.RegistryUrl; - - var registry = await reader.GetRegistry(ctx); - if (registry.Repositories.TryGetValue(entry.Repository, out var repoBranches)) - { - var linkIndexEntry = GetNextContentSourceLinkIndexEntry(repoBranches, entry.Repository); - linkIndexEntries.Add(entry.Repository, linkIndexEntry); - } } catch (Exception ex) { + hadFetchFailures = true; _logger.LogWarning(ex, "Error fetching link data for repository '{Repository}'. Cross-links to this repository may not resolve correctly.", entry.Repository); _ = registryUrlsByRepository.TryAdd(entry.Repository, reader.RegistryUrl); @@ -86,6 +92,24 @@ public override async Task FetchCrossLinks(Cancel ctx) LinkIndexEntries = linkIndexEntries.ToFrozenDictionary(), RegistryUrlsByRepository = registryUrlsByRepository.ToFrozenDictionary(), CodexRepositories = codexRepositories.Count > 0 ? codexRepositories.ToFrozenSet() : null, + IsComplete = !hadFetchFailures, }; } + + private async Task TryGetRegistry(ILinkIndexReader reader, Cancel ctx) + { + try + { + return await reader.GetRegistry(ctx); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to fetch link index registry from {RegistryUrl}", reader.RegistryUrl); + return null; + } + } } diff --git a/src/tooling/docs-builder/Http/ReloadGeneratorService.cs b/src/tooling/docs-builder/Http/ReloadGeneratorService.cs index 1cf66d4d90..e3896eb858 100644 --- a/src/tooling/docs-builder/Http/ReloadGeneratorService.cs +++ b/src/tooling/docs-builder/Http/ReloadGeneratorService.cs @@ -2,6 +2,7 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.Collections.Frozen; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Westwind.AspNetCore.LiveReload; @@ -28,13 +29,18 @@ public sealed class ReloadGeneratorService( ILogger logger ) : IHostedService, IDisposable { + private static readonly FrozenSet AssetExtensions = new[] + { + ".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", + ".yml", ".yaml", ".toml" + }.ToFrozenSet(StringComparer.OrdinalIgnoreCase); + private FileSystemWatcher? _watcher; private CancellationTokenSource? _serviceCts; private ReloadableGeneratorState ReloadableGenerator { get; } = reloadableGenerator; private InMemoryBuildState InMemoryBuildState { get; } = inMemoryBuildState; private ILogger Logger { get; } = logger; - //debounce reload requests due to many file changes private readonly Debouncer _debouncer = new(TimeSpan.FromMilliseconds(200)); public async Task StartAsync(Cancel cancellationToken) @@ -78,6 +84,8 @@ await Task.WhenAll( watcher.Filters.Add("docset.yml"); watcher.Filters.Add("_docset.yml"); watcher.Filters.Add("toc.yml"); + foreach (var ext in AssetExtensions) + watcher.Filters.Add($"*{ext}"); watcher.IncludeSubdirectories = true; watcher.EnableRaisingEvents = true; _watcher = watcher; @@ -88,18 +96,17 @@ private void Reload(bool reloadConfiguration = false) var token = _serviceCts?.Token ?? Cancel.None; _ = _debouncer.ExecuteAsync(async ctx => { - var sourcePath = ReloadableGenerator.Generator.Context.DocumentationSourceDirectory.FullName; - - // Start in-memory validation build (runs in parallel) - var validationTask = InMemoryBuildState.StartBuildAsync(sourcePath, ctx); - - // Wait for live reload to complete, then refresh the browser immediately await ReloadableGenerator.ReloadAsync(ctx, reloadConfiguration); Logger.LogInformation("Reload complete!"); _ = LiveReloadMiddleware.RefreshWebSocketRequest(); - // Wait for validation build to complete - await validationTask; + // Only run the full validation build for structural changes (config/toc edits, file add/delete). + // Content-only .md edits are picked up on the next request via ParseFullAsync. + if (reloadConfiguration) + { + var sourcePath = ReloadableGenerator.Generator.Context.DocumentationSourceDirectory.FullName; + await InMemoryBuildState.StartBuildAsync(sourcePath, ctx); + } }, token); } @@ -124,6 +131,9 @@ private static bool ShouldIgnorePath(string path) => private static bool IsConfigFile(string path) => path.EndsWith("docset.yml") || path.EndsWith("toc.yml"); + private static bool IsAssetFile(string path) => + AssetExtensions.Contains(Path.GetExtension(path)); + private void OnChanged(object sender, FileSystemEventArgs e) { if (e.ChangeType != WatcherChangeTypes.Changed) @@ -138,6 +148,8 @@ private void OnChanged(object sender, FileSystemEventArgs e) Reload(reloadConfiguration: true); else if (e.FullPath.EndsWith(".md")) Reload(); + else if (IsAssetFile(e.FullPath)) + _ = LiveReloadMiddleware.RefreshWebSocketRequest(); #if DEBUG if (e.FullPath.EndsWith(".cshtml")) _ = LiveReloadMiddleware.RefreshWebSocketRequest(); @@ -152,6 +164,8 @@ private void OnCreated(object sender, FileSystemEventArgs e) Logger.LogInformation("Created: {FullPath}", e.FullPath); if (e.FullPath.EndsWith(".md") || IsConfigFile(e.FullPath)) Reload(reloadConfiguration: true); + else if (IsAssetFile(e.FullPath)) + _ = LiveReloadMiddleware.RefreshWebSocketRequest(); } private void OnDeleted(object sender, FileSystemEventArgs e) @@ -162,6 +176,8 @@ private void OnDeleted(object sender, FileSystemEventArgs e) Logger.LogInformation("Deleted: {FullPath}", e.FullPath); if (e.FullPath.EndsWith(".md") || IsConfigFile(e.FullPath)) Reload(reloadConfiguration: true); + else if (IsAssetFile(e.FullPath)) + _ = LiveReloadMiddleware.RefreshWebSocketRequest(); } private void OnRenamed(object sender, RenamedEventArgs e) @@ -174,6 +190,8 @@ private void OnRenamed(object sender, RenamedEventArgs e) Logger.LogInformation(" New: {NewFullPath}", e.FullPath); if (e.FullPath.EndsWith(".md") || e.OldFullPath.EndsWith(".md") || IsConfigFile(e.FullPath) || IsConfigFile(e.OldFullPath)) Reload(reloadConfiguration: true); + else if (IsAssetFile(e.FullPath) || IsAssetFile(e.OldFullPath)) + _ = LiveReloadMiddleware.RefreshWebSocketRequest(); #if DEBUG if (e.FullPath.EndsWith(".cshtml")) _ = LiveReloadMiddleware.RefreshWebSocketRequest(); diff --git a/src/tooling/docs-builder/Http/ReloadableGeneratorState.cs b/src/tooling/docs-builder/Http/ReloadableGeneratorState.cs index 70acd4d757..11314ed6e6 100644 --- a/src/tooling/docs-builder/Http/ReloadableGeneratorState.cs +++ b/src/tooling/docs-builder/Http/ReloadableGeneratorState.cs @@ -27,8 +27,9 @@ public class ReloadableGeneratorState : IDisposable private readonly ILoggerFactory _logFactory; private readonly BuildContext _context; private readonly bool _isWatchBuild; - private readonly DocSetConfigurationCrossLinkFetcher _crossLinkFetcher; - private readonly ILinkIndexReader? _codexReader; + private DocSetConfigurationCrossLinkFetcher _crossLinkFetcher; + private ILinkIndexReader? _codexReader; + private FetchedCrossLinks? _cachedCrossLinks; public ReloadableGeneratorState(ILoggerFactory logFactory, IDirectoryInfo sourcePath, @@ -66,11 +67,29 @@ bool isWatchBuild public async Task ReloadAsync(Cancel ctx, bool reloadConfiguration = true) { + // Content-only changes (e.g. .md edits) don't need a full rebuild: + // RenderLayout -> ParseFullAsync reads fresh content from disk on each request. + if (!reloadConfiguration && _cachedCrossLinks is not null) + return; + SourcePath.Refresh(); OutputPath.Refresh(); if (reloadConfiguration) + { _context.ReloadConfiguration(); - var crossLinks = await _crossLinkFetcher.FetchCrossLinks(ctx); + (_codexReader as IDisposable)?.Dispose(); + _codexReader = _context.Configuration.Registry != DocSetRegistry.Public + ? new GitLinkIndexReader(_context.Configuration.Registry.ToStringFast(true), FileSystemFactory.AppData) + : null; + _crossLinkFetcher = new DocSetConfigurationCrossLinkFetcher(_logFactory, _context.Configuration, codexLinkIndexReader: _codexReader); + } + var crossLinks = _cachedCrossLinks; + if (crossLinks is null || reloadConfiguration) + { + crossLinks = await _crossLinkFetcher.FetchCrossLinks(ctx); + // Only cache successful fetches so transient failures get retried on the next reload. + _cachedCrossLinks = crossLinks.IsComplete ? crossLinks : null; + } IUriEnvironmentResolver? uriResolver = crossLinks.CodexRepositories is not null ? new CodexAwareUriResolver(crossLinks.CodexRepositories) : null;