chore: Fix internal exception on TOC loading

src/Docfx.Build/TableOfContents/TocHelper.cs

@@ -6,15 +6,14 @@
 using Docfx.Common;
 using Docfx.DataContracts.Common;
 using Docfx.Plugins;
+using YamlDotNet.Core.Events;
+using YamlDotNet.Core;
+using Constants = Docfx.DataContracts.Common.Constants;
 
 namespace Docfx.Build.TableOfContents;
 
 public static class TocHelper
 {
-    private static readonly YamlDeserializerWithFallback _deserializer =
-        YamlDeserializerWithFallback.Create<List<TocItemViewModel>>()
-        .WithFallback<TocItemViewModel>();
-
     internal static List<FileModel> ResolveToc(ImmutableList<FileModel> models)
     {
         var tocCache = new Dictionary<string, TocItemInfo>(FilePathComparer.OSPlatformSensitiveStringComparer);
@@ -63,21 +62,22 @@ public static TocItemViewModel LoadSingleToc(string file)
         var fileType = Utility.GetTocFileType(file);
         try
         {
-            if (fileType == TocFileType.Markdown)
-            {
-                return new()
-                {
-                    Items = MarkdownTocReader.LoadToc(EnvironmentContext.FileAbstractLayer.ReadAllText(file), file)
-                };
-            }
-            else if (fileType == TocFileType.Yaml)
+            switch (fileType)
             {
-                return _deserializer.Deserialize(file) switch
-                {
-                    List<TocItemViewModel> vm => new() { Items = vm },
-                    TocItemViewModel root => root,
-                    _ => throw new NotSupportedException($"{file} is not a valid TOC file."),
-                };
+                case TocFileType.Markdown:
+                    return new()
+                    {
+                        Items = MarkdownTocReader.LoadToc(EnvironmentContext.FileAbstractLayer.ReadAllText(file), file)
+                    };
+                case TocFileType.Yaml:
+                    {
+                        using var reader = EnvironmentContext.FileAbstractLayer.OpenReadText(file);
+                        return IsListTocItems(reader)
+                            ? new TocItemViewModel { Items = YamlUtility.Deserialize<List<TocItemViewModel>>(reader), }
+                            : YamlUtility.Deserialize<TocItemViewModel>(reader);
+                    }
+                default:
+                    throw new NotSupportedException($"{file} is not a valid TOC file, supported TOC files should be either \"{Constants.TableOfContents.MarkdownTocFileName}\" or \"{Constants.TableOfContents.YamlTocFileName}\".");
             }
         }
         catch (Exception e)
@@ -86,7 +86,45 @@ public static TocItemViewModel LoadSingleToc(string file)
             Logger.LogError(message, code: ErrorCodes.Toc.InvalidTocFile);
             throw new DocumentException(message, e);
         }
+    }
+
+    private static bool IsListTocItems(StreamReader reader)
+    {
+        try
+        {
+            var scanner = new Scanner(reader, skipComments: true);
+            var parser = new Parser(scanner);
+
+            return parser.TryConsume<StreamStart>(out var _)
+                && parser.TryConsume<DocumentStart>(out var _)
+                && parser.TryConsume<SequenceStart>(out var _);
+        }
+        finally
+        {
+            ResetStreamReader(reader);
+        }
+    }
+
+    private static void ResetStreamReader(StreamReader reader)
+    {
+        // Reset stream position and discard StreamReader's buffered data.
+        reader.BaseStream.Seek(0, SeekOrigin.Begin);
+        reader.DiscardBufferedData();
+
+        // Gets current encoding's BOM bytes.
+        var bomBytes = reader.CurrentEncoding.Preamble;
+        if (bomBytes.IsEmpty)
+            return;
+
+        // Try to skip BOM bytes. (StreamReader don't skip BOM bytes automatically when calling DiscardBufferedData) 
+        Span<byte> bytes = stackalloc byte[bomBytes.Length];
+        reader.BaseStream.Read(bytes);
+
+        if (bomBytes.SequenceEqual(bytes))
+            return;
 
-        throw new NotSupportedException($"{file} is not a valid TOC file, supported TOC files should be either \"{Constants.TableOfContents.MarkdownTocFileName}\" or \"{Constants.TableOfContents.YamlTocFileName}\".");
+        // Reset stream position. If header bytes are not matched to expected BOM bytes. 
+        // This logics are required for StreamReader that is created with `detectEncodingFromByteOrderMarks: true`.
+        reader.BaseStream.Seek(0, SeekOrigin.Begin);
     }
 }

test/Docfx.Build.Tests/TocHelperTest.cs

@@ -0,0 +1,124 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Text;
+using Docfx.Common;
+using Docfx.DataContracts.Common;
+using FluentAssertions;
+using Xunit;
+
+namespace Docfx.Build.TableOfContents.Tests;
+
+public class TocHelperTest
+{
+    [Fact]
+    public void TestItemDeserialization()
+    {
+        // Arrange
+        var item = new TocItemViewModel
+        {
+            Items =
+            [
+                new TocItemViewModel { Uid = "item1" },
+                new TocItemViewModel { Uid = "item2" }
+            ],
+        };
+
+        var yaml = ToYaml(item);
+        var filePath = Path.Combine(Path.GetTempPath(), "toc.yml");
+        File.WriteAllText(filePath, yaml, new UTF8Encoding(false));
+
+        try
+        {
+            // Act
+            var result = TocHelper.LoadSingleToc(filePath);
+
+            // Assert
+            result.Should().BeEquivalentTo(item);
+        }
+        finally
+        {
+            File.Delete(filePath);
+        }
+    }
+
+    [Fact]
+    public void TestListDeserialization()
+    {
+        // Arrange
+        var items = new TocItemViewModel[]
+        {
+            new TocItemViewModel { Uid = "item1" },
+            new TocItemViewModel { Uid = "item2" },
+        };
+
+        var yaml = ToYaml(items);
+        var filePath = Path.Combine(Path.GetTempPath(), "toc.yml");
+        File.WriteAllText(filePath, yaml);
+
+        try
+        {
+            // Act
+            var result = TocHelper.LoadSingleToc(filePath);
+
+            // Assert
+            result.Uid.Should().BeNull();
+            result.Href.Should().BeNull();
+            result.Items.Should().BeEquivalentTo(items);
+        }
+        finally
+        {
+            File.Delete(filePath);
+        }
+    }
+
+    [Fact]
+    public void TestItemDeserializationWithEncoding()
+    {
+        // Arrange
+        var item = new TocItemViewModel
+        {
+            Items =
+            [
+                new TocItemViewModel { Uid = "item1" },
+                new TocItemViewModel { Uid = "item2" }
+            ],
+        };
+
+        var yaml = ToYaml(item);
+
+        foreach (var encoding in Encodings)
+        {
+            var filePath = Path.Combine(Path.GetTempPath(), "toc.yml");
+            File.WriteAllText(filePath, yaml, encoding);
+
+            try
+            {
+                // Act
+                var result = TocHelper.LoadSingleToc(filePath);
+
+                // Assert
+                result.Should().BeEquivalentTo(item);
+            }
+            finally
+            {
+                File.Delete(filePath);
+            }
+        }
+    }
+
+    private static readonly Encoding[] Encodings =
+    [
+        new UTF8Encoding(false),
+        new UTF8Encoding(true),
+        Encoding.Unicode,
+        Encoding.BigEndianUnicode,
+    ];
+
+    private static string ToYaml<T>(T model)
+    {
+        using StringWriter sw = new StringWriter();
+        YamlUtility.Serialize(sw, model);
+        return sw.ToString();
+    }
+}

test/docfx.Tests/Api.verified.cs

@@ -2123,13 +2123,6 @@ public static class XrefUtility
     {
         public static bool TryGetXrefStringValue(this Docfx.Plugins.XRefSpec spec, string key, out string value) { }
     }
-    public class YamlDeserializerWithFallback
-    {
-        public object Deserialize(System.Func<System.IO.TextReader> reader) { }
-        public object Deserialize(string filePath) { }
-        public Docfx.Common.YamlDeserializerWithFallback WithFallback<T>() { }
-        public static Docfx.Common.YamlDeserializerWithFallback Create<T>() { }
-    }
     public static class YamlMime
     {
         public const string ManagedReference = "YamlMime:ManagedReference";