Unit tests that aim to verify the behavior and correctness of the sampling pipeline under various conditions

LLama.Rag/IWebScraper.cs

@@ -0,0 +1,15 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using HtmlAgilityPack;
+
+namespace LLama.Rag
+{
+    public interface IWebScraper
+    {
+        HashSet<string> VisitedUrls { get; }
+        List<HtmlDocument> Documents { get; }
+
+        Task<List<string>> ExtractVisibleTextAsync(int minWordLength, bool checkSentences, bool explodeParagraphs);
+        Task<List<string>> ExtractParagraphsAsync();
+    }
+}

LLama.Rag/LLama.Rag.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.72" />
+  </ItemGroup>
+
+</Project>

LLama.Rag/Rag.cs

@@ -0,0 +1,53 @@
+using System;
+using System.Threading.Tasks;
+
+namespace LLama.Rag
+{
+    public class Rag
+    {
+        public static async Task Main(string[] args)
+        {
+            try
+            {
+                Console.WriteLine("Initializing WebScraper...");
+
+                string startUrl = "https://en.wikipedia.org/wiki/Aluminium_alloy";
+                int depth = 0; // Scrape only the provided webpage and no links. 
+                int minWordLength = 4; // Minimum word count for a text block to be extracted.
+                bool checkSentences = false;
+                bool explodeParagraphs = true;
+
+                WebScraper webScraper = await WebScraper.CreateAsync(startUrl, depth);
+
+                Console.WriteLine("WebScraper initialized successfully.");
+                Console.WriteLine("Extracting visible text...");
+
+                var documentText = webScraper.ExtractVisibleTextAsync(minWordLength, checkSentences, explodeParagraphs);
+
+                Console.WriteLine($"Extracted {documentText.Result.Count} blocks of text.");
+
+                if (documentText.Result.Count == 0)
+                {
+                    Console.WriteLine("Warning: No text was extracted. Try lowering minWordLength or changing extraction settings.");
+                }
+
+                foreach (string text in documentText.Result)
+                {
+                    Console.WriteLine("Extracted Block:");
+                    Console.WriteLine(text);
+                    Console.WriteLine(""); // Space between blocks for readability
+                }
+
+                Console.WriteLine("Scraping complete.");
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"An error occurred: {ex.Message}");
+                Console.WriteLine($"StackTrace: {ex.StackTrace}");
+            }
+
+            Console.WriteLine("Press any key to exit...");
+            Console.ReadKey();
+        }
+    }
+}

LLama.Rag/WebScraper.cs

@@ -0,0 +1,144 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.Threading.Tasks;
+using System.Text.RegularExpressions;
+using HtmlAgilityPack;
+using System.Web;
+
+namespace LLama.Rag
+{
+    class WebScraper : IWebScraper
+    {
+        private static readonly HttpClient httpClient = new HttpClient();
+        public HashSet<string> VisitedUrls { get; } = new HashSet<string>();
+        public List<HtmlDocument> Documents { get; } = new List<HtmlDocument>();
+
+        private WebScraper() { }
+
+        public static async Task<WebScraper> CreateAsync(string url, int queryDepth)
+        {
+            WebScraper instance = new WebScraper();
+            await instance.FetchContentAsynch(url, queryDepth);
+            return instance;
+        }
+
+        private async Task FetchContentAsynch(string url, int queryDepth)
+        {
+            if (queryDepth < 0 || VisitedUrls.Contains(url)) return;
+
+            try
+            {
+                VisitedUrls.Add(url);
+                string pageContent = await httpClient.GetStringAsync(url);
+                HtmlDocument doc = new HtmlDocument();
+                doc.LoadHtml(pageContent);
+                Documents.Add(doc);
+
+                if (queryDepth > 0)
+                {
+                    var links = ExtractLinks(doc, url);
+                    var tasks = links.Select(link => FetchContentAsynch(link, queryDepth - 1));
+                    await Task.WhenAll(tasks);
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error scraping {url}: {ex.Message}");
+            }
+        }
+
+        private static List<string> ExtractLinks(HtmlDocument doc, string baseUrl)
+        {
+            return doc.DocumentNode
+                .SelectNodes("//body//a[@href]")?
+                .Select(node => node.GetAttributeValue("href", ""))
+                .Where(href => !string.IsNullOrEmpty(href))
+                .Select(href => NormalizeUrl(href, baseUrl))
+                .Where(link => link != null)
+                .Distinct()
+                .ToList() ?? new List<string>();
+        }
+
+        private static string NormalizeUrl(string href, string baseUrl)
+        {
+            if (href.StartsWith("http", StringComparison.OrdinalIgnoreCase))
+                return href;
+
+            if (href.StartsWith("/"))
+                return new Uri(new Uri(baseUrl), href).ToString();
+
+            return null;
+        }
+
+        public async Task<List<string>> ExtractVisibleTextAsync(int minWordLength, bool checkSentences, bool explodeParagraphs)
+        {
+            return await Task.Run(() =>
+            {
+                List<string> allDocumentText = new List<string>();
+                foreach (HtmlDocument doc in Documents)
+                {
+                    var currentDocText = doc.DocumentNode
+                        .SelectNodes("//body//*[not(ancestor::table) and not(self::script or self::style)] | //body//a[not(self::script or self::style)]")?
+                        .Select(node =>
+                        {
+                            string cleanedText = HtmlEntity.DeEntitize(node.InnerText.Trim());
+                            cleanedText = cleanedText.Replace("\t", " ");
+                            cleanedText = Regex.Replace(cleanedText, @"\s+", " ");
+                            return cleanedText;
+                        })
+                        .Where(text => !string.IsNullOrWhiteSpace(text) && text.Split(' ').Length >= minWordLength)
+                        .ToList() ?? new List<string>();
+
+                    allDocumentText.AddRange(currentDocText);
+                }
+
+                if (explodeParagraphs) allDocumentText = ExplodeParagraphs(allDocumentText, minWordLength);
+                if (checkSentences) allDocumentText = RudimentarySentenceCheck(allDocumentText);
+                return allDocumentText;
+            });
+        }
+
+        public async Task<List<string>> ExtractParagraphsAsync()
+        {
+            return await Task.Run(() =>
+            {
+                List<string> paragraphs = new List<string>();
+                foreach (HtmlDocument doc in Documents)
+                {
+                    var currentDocParagraph = doc.DocumentNode
+                        .SelectNodes("//p//text()")?
+                        .Select(node => HtmlEntity.DeEntitize(node.InnerText.Trim()))
+                        .Where(text => !string.IsNullOrWhiteSpace(text))
+                        .ToList() ?? new List<string>();
+
+                    paragraphs.AddRange(currentDocParagraph);
+                }
+                return paragraphs;
+            });
+        }
+
+        private static List<string> RudimentarySentenceCheck(List<string> sentences)
+        {
+            List<Regex> sentenceRules = new List<Regex>
+            {
+                new Regex(@"^[A-Za-z0-9]+[\w\s,;:'""-]*", RegexOptions.Compiled | RegexOptions.IgnoreCase),
+                new Regex(@"[^\W]{2,}", RegexOptions.Compiled),
+                new Regex(@"\b(\w*:?[/\w\d]+\.){2,}\d+\b", RegexOptions.Compiled)
+            };
+
+            return sentences.Where(sentence => sentenceRules.All(regex => regex.IsMatch(sentence))).ToList();
+        }
+
+        private static List<string> ExplodeParagraphs(List<string> paragraphs, int minWordLength)
+        {
+            return paragraphs
+                .SelectMany(paragraph =>
+                    Regex.Matches(paragraph, @"(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\s|^)([A-Z0-9][^.!?]*[.!?])")
+                        .Cast<Match>()
+                        .Select(m => m.Value.Trim()))
+                .ToList();
+        }
+    }
+}

LLama.Unittest/ChatSessionTests.cs

@@ -0,0 +1,31 @@
+using LLama.Common;
+using Xunit.Abstractions;
+
+namespace LLama.Unittest
+{
+    public sealed class ChatSessionTests
+        : IDisposable
+    {
+        private readonly ITestOutputHelper _testOutputHelper;
+        private readonly ModelParams _params;
+        private readonly LLamaWeights _model;
+
+        public ChatSessionTests(ITestOutputHelper testOutputHelper)
+        {
+            _testOutputHelper = testOutputHelper;
+            _params = new ModelParams(Constants.GenerativeModelPath2)
+            {
+                ContextSize = 128,
+                GpuLayerCount = Constants.CIGpuLayerCount
+            };
+            _model = LLamaWeights.LoadFromFile(_params);
+        }
+
+        public void Dispose()
+        {
+            _model.Dispose();
+        }
+
+
+    }
+}

LLama.Unittest/GrammarTest.cs

@@ -0,0 +1,63 @@
+using Xunit;
+
+namespace LLama.Sampling.Tests
+{
+    public class GrammarTests
+    {
+        [Fact]
+        public void Constructor_SetsPropertiesCorrectly()
+        {
+            // Arrange
+            var gbnf = "test_gbnf";
+            var root = "test_root";
+
+            // Act
+            var grammar = new Grammar(gbnf, root);
+
+            // Assert
+            Assert.Equal(gbnf, grammar.Gbnf);
+            Assert.Equal(root, grammar.Root);
+        }
+
+        [Fact]
+        public void ToString_ReturnsExpectedString()
+        {
+            // Arrange
+            var gbnf = "test_gbnf";
+            var root = "test_root";
+            var grammar = new Grammar(gbnf, root);
+
+            // Act
+            var toString = grammar.ToString();
+
+            // Assert
+            Assert.Equal($"Grammar {{ Gbnf = {gbnf}, Root = {root} }}", toString);
+        }
+
+        [Fact]
+        public void Equality_ChecksPropertiesCorrectly()
+        {
+            // Arrange
+            var gbnf = "test_gbnf";
+            var root = "test_root";
+            var grammar1 = new Grammar(gbnf, root);
+            var grammar2 = new Grammar(gbnf, root);
+
+            // Act and Assert
+            Assert.Equal(grammar1, grammar2);
+        }
+
+        [Fact]
+        public void Inequality_ChecksPropertiesCorrectly()
+        {
+            // Arrange
+            var gbnf = "test_gbnf";
+            var root = "test_root";
+            var grammar1 = new Grammar(gbnf, root);
+            var grammar2 = new Grammar("different_gbnf", root);
+
+            // Act and Assert
+            Assert.NotEqual(grammar1, grammar2);
+        }
+    }
+}