Refactor common caching semantics into ICacheService

This change reduces the repentitiveness of implementing request-and-cache or check-cache-else-request patterns when requesting provider metadata.  This simplified code in each provider, and also centralized testing that behavior on the CacheService class rather than duplicating cache behavior testing.

It also removes the GetCatalogAsync and GetMetadataAsync as those names were Cdnjs-specifc, and the catalog is just another form of metadata.

Testing was also largely refactored to apply fakes at the ICacheService layer instead of the underlying IWebRequestHandler, as that should be transparent to the Catalog implementations.  There are a couple exceptions to verify that no web requests are issued at all during some test scenarios.

Author: jimmylewis

src/LibraryManager.Contracts/Caching/ICacheService.cs

@@ -12,21 +12,22 @@ namespace Microsoft.Web.LibraryManager.Contracts.Caching
     public interface ICacheService
     {
         /// <summary>
-        /// Returns the provider's catalog from the provided Url to cacheFile
+        /// Gets the contents from the specified URL, or if the request fails then from a locally cached copy
         /// </summary>
-        /// <param name="url">Url to the provider catalog</param>
-        /// <param name="cacheFile">Where to store the provider catalog within the cache</param>
+        /// <param name="url">The URL to request</param>
+        /// <param name="cacheFile">The locally cached file</param>
         /// <param name="cancellationToken">Cancellation token</param>
         /// <returns></returns>
-        Task<string> GetCatalogAsync(string url, string cacheFile, CancellationToken cancellationToken);
+        Task<string> GetContentsFromUriWithCacheFallbackAsync(string url, string cacheFile, CancellationToken cancellationToken);
 
         /// <summary>
-        /// Returns library metadata from provided Url to cacheFile
+        /// Gets the contents of a local cache file, or if the file does not exist then requests it from the specified URL
         /// </summary>
-        /// <param name="url">Url to the library metadata</param>
-        /// <param name="cacheFile">Where to store the metadata file within the cache</param>
+        /// <param name="cacheFile">The locally cached file</param>
+        /// <param name="url">The URL to request</param>
         /// <param name="cancellationToken">Cancellation token</param>
         /// <returns></returns>
-        Task<string> GetMetadataAsync(string url, string cacheFile, CancellationToken cancellationToken);
+        /// <exception cref="ResourceDownloadException">Thrown when the file doesn't exist and the resource download fails</exception>
+        Task<string> GetContentsFromCachedFileWithWebRequestFallbackAsync(string cacheFile, string url, CancellationToken cancellationToken);
     }
 }

src/LibraryManager/CacheService/CacheService.cs

@@ -17,9 +17,7 @@ namespace Microsoft.Web.LibraryManager
     /// </summary>
     public class CacheService : ICacheService
     {
-        // TO DO: Move these expirations to the provider
-        private const int CatalogExpiresAfterDays = 1;
-        private const int MetadataExpiresAfterDays = 1;
+        private const int DefaultCacheExpiresAfterDays = 1;
         private const int MaxConcurrentDownloads = 10;
 
         private readonly IWebRequestHandler _requestHandler;
@@ -52,30 +50,6 @@ public static string CacheFolder
             }
         }
 
-        /// <summary>
-        /// Returns the provider's catalog from the provided Url to cacheFile
-        /// </summary>
-        /// <param name="url"></param>
-        /// <param name="cacheFile"></param>
-        /// <param name="cancellationToken"></param>
-        /// <returns></returns>
-        public async Task<string> GetCatalogAsync(string url, string cacheFile, CancellationToken cancellationToken)
-        {
-            return await GetResourceAsync(url, cacheFile, CatalogExpiresAfterDays, cancellationToken);
-        }
-
-        /// <summary>
-        /// Returns library metadata from provided Url to cacheFile
-        /// </summary>
-        /// <param name="url"></param>
-        /// <param name="cacheFile"></param>
-        /// <param name="cancellationToken"></param>
-        /// <returns></returns>
-        public async Task<string> GetMetadataAsync(string url, string cacheFile, CancellationToken cancellationToken)
-        {
-            return await GetResourceAsync(url, cacheFile, MetadataExpiresAfterDays, cancellationToken);
-        }
-
         /// <summary>
         /// Downloads a resource from specified url to a destination file
         /// </summary>
@@ -141,5 +115,45 @@ async Task DownloadFileIfNecessaryAsync(CacheFileMetadata metadata)
                 }
             }
         }
+
+        /// <inheritdoc />
+        public async Task<string> GetContentsFromUriWithCacheFallbackAsync(string url, string cacheFile, CancellationToken cancellationToken)
+        {
+            string contents;
+            try
+            {
+                contents = await GetResourceAsync(url, cacheFile, DefaultCacheExpiresAfterDays, cancellationToken);
+            }
+            catch (ResourceDownloadException)
+            {
+                // TODO: Log telemetry
+                if (File.Exists(cacheFile))
+                {
+                    contents = await FileHelpers.ReadFileAsTextAsync(cacheFile, cancellationToken);
+                }
+                else
+                {
+                    throw;
+                }
+            }
+
+            return contents;
+        }
+
+        /// <inheritdoc />
+        public async Task<string> GetContentsFromCachedFileWithWebRequestFallbackAsync(string cacheFile, string url, CancellationToken cancellationToken)
+        {
+            string contents;
+            if (File.Exists(cacheFile))
+            {
+                contents = await FileHelpers.ReadFileAsTextAsync(cacheFile, cancellationToken);
+            }
+            else
+            {
+                contents = await GetResourceAsync(url, cacheFile, DefaultCacheExpiresAfterDays, cancellationToken);
+            }
+
+            return contents;
+        }
     }
 }

src/LibraryManager/Providers/Cdnjs/CdnjsCatalog.cs

@@ -19,8 +19,8 @@ internal class CdnjsCatalog : ILibraryCatalog
     {
         // TO DO: These should become Provider properties to be passed to CacheService
         private const string FileName = "cache.json";
-        private const string RemoteApiUrl = "https://aka.ms/g8irvu";
-        private const string MetaPackageUrlFormat = "https://api.cdnjs.com/libraries/{0}"; // https://aka.ms/goycwu/{0}
+        public const string CatalogUrl = "https://api.cdnjs.com/libraries?fields=name,description,version";
+        public const string MetaPackageUrlFormat = "https://api.cdnjs.com/libraries/{0}"; // https://aka.ms/goycwu/{0}
 
         private readonly string _cacheFile;
         private readonly CdnjsProvider _provider;
@@ -245,24 +245,7 @@ private async Task<bool> EnsureCatalogAsync(CancellationToken cancellationToken)
 
             try
             {
-                string json;
-                try
-                {
-                    json = await _cacheService.GetCatalogAsync(RemoteApiUrl, _cacheFile, cancellationToken).ConfigureAwait(false);
-                }
-                catch (ResourceDownloadException)
-                {
-                    // TODO: add telemetry
-                    _provider.HostInteraction.Logger.Log(string.Format(Resources.Text.FailedToDownloadCatalog, _provider.Id), LogLevel.Operation);
-                    if (File.Exists(_cacheFile))
-                    {
-                        json = await FileHelpers.ReadFileAsTextAsync(_cacheFile, cancellationToken);
-                    }
-                    else
-                    {
-                        return false;
-                    }
-                }
+                string json = await _cacheService.GetContentsFromUriWithCacheFallbackAsync(CatalogUrl, _cacheFile, cancellationToken);
 
                 if (string.IsNullOrWhiteSpace(json))
                 {
@@ -295,23 +278,7 @@ private async Task<IEnumerable<Asset>> GetAssetsAsync(string groupName, Cancella
 
             try
             {
-                string json;
-                try
-                {
-                    json = await _cacheService.GetMetadataAsync(url, localFile, cancellationToken).ConfigureAwait(false);
-                }
-                catch (ResourceDownloadException)
-                {
-                    // TODO: Log telemetry
-                    if (File.Exists(localFile))
-                    {
-                        json = await FileHelpers.ReadFileAsTextAsync(localFile, cancellationToken);
-                    }
-                    else
-                    {
-                        throw;
-                    }
-                }
+                string json = await _cacheService.GetContentsFromUriWithCacheFallbackAsync(url, localFile, cancellationToken);
 
                 if (!string.IsNullOrEmpty(json))
                 {

src/LibraryManager/Providers/Unpkg/UnpkgCatalog.cs

@@ -48,25 +48,11 @@ public async Task<string> GetLatestVersion(string libraryName, bool includePreRe
             try
             {
                 string latestLibraryVersionUrl = string.Format(LatestLibraryVersonUrl, libraryName);
-                string latestCacheFile = Path.Combine(_cacheFolder, $"{libraryName}-{LatestVersionTag}.json");
+                string latestCacheFile = Path.Combine(_cacheFolder, libraryName, $"{LatestVersionTag}.json");
 
-                string latestJson;
-                try
-                {
-                    latestJson = await _cacheService.GetMetadataAsync(latestLibraryVersionUrl, latestCacheFile, cancellationToken);
-                }
-                catch (ResourceDownloadException)
-                {
-                    // TODO: add telemetry
-                    if (File.Exists(latestCacheFile))
-                    {
-                        latestJson = await FileHelpers.ReadFileAsTextAsync(latestCacheFile, cancellationToken);
-                    }
-                    else
-                    {
-                        throw;
-                    }
-                }
+                string latestJson = await _cacheService.GetContentsFromUriWithCacheFallbackAsync(latestLibraryVersionUrl,
+                                                                                                 latestCacheFile,
+                                                                                                 cancellationToken);
 
                 var packageObject = (JObject)JsonConvert.DeserializeObject(latestJson);
 
@@ -123,22 +109,9 @@ private async Task<IEnumerable<string>> GetLibraryFilesAsync(string libraryName,
             string libraryFileListUrl = string.Format(LibraryFileListUrlFormat, libraryName, version);
             string libraryFileListCacheFile = Path.Combine(_cacheFolder, libraryName, $"{version}-filelist.json");
 
-            string fileList;
-            try
-            {
-                fileList = await _cacheService.GetMetadataAsync(libraryFileListUrl, libraryFileListCacheFile, cancellationToken);
-            }
-            catch (ResourceDownloadException)
-            {
-                if (File.Exists(libraryFileListCacheFile))
-                {
-                    fileList = await FileHelpers.ReadFileAsTextAsync(libraryFileListCacheFile, cancellationToken);
-                }
-                else
-                {
-                    throw;
-                }
-            }
+            string fileList = await _cacheService.GetContentsFromCachedFileWithWebRequestFallbackAsync(libraryFileListCacheFile,
+                                                                                                       libraryFileListUrl,
+                                                                                                       cancellationToken);
 
             var fileListObject = (JObject)JsonConvert.DeserializeObject(fileList);
 

src/LibraryManager/Providers/jsDelivr/JsDelivrCatalog.cs

@@ -61,28 +61,13 @@ public async Task<string> GetLatestVersion(string libraryId, bool includePreRele
                 bool isGitHub = IsGitHub(libraryId);
                 string latestLibraryVersionUrl = string.Format(isGitHub ? LatestLibraryVersionUrlGH : LatestLibraryVersionUrl, name);
                 string cacheFileType = isGitHub ? "github" : "npm";
-                string latestLibraryVersionCacheFile = Path.Combine(_cacheFolder, $"{name.Replace("/", "_")}-{cacheFileType}-{LatestVersionTag}.json");
+                string latestLibraryVersionCacheFile = Path.Combine(_cacheFolder, name, $"{cacheFileType}-{LatestVersionTag}.json");
 
-                string latestVersionContent;
-                try
-                {
-                    latestVersionContent = await _cacheService.GetMetadataAsync(latestLibraryVersionUrl, latestLibraryVersionCacheFile, cancellationToken);
-                }
-                catch (ResourceDownloadException)
-                {
-                    // TODO: Log telemetry
-                    if (File.Exists(latestLibraryVersionCacheFile))
-                    {
-                        latestVersionContent = await FileHelpers.ReadFileAsTextAsync(latestLibraryVersionCacheFile, cancellationToken);
-                    }
-                    else
-                    {
-                        throw;
-                    }
-                }
+                string latestVersionContent = await _cacheService.GetContentsFromUriWithCacheFallbackAsync(latestLibraryVersionUrl,
+                                                                                                           latestLibraryVersionCacheFile,
+                                                                                                           cancellationToken);
 
                 var packageObject = (JObject)JsonConvert.DeserializeObject(latestVersionContent);
-
                 if (packageObject != null)
                 {
                     var versions = packageObject["tags"] as JObject;
@@ -129,16 +114,10 @@ private async Task<IEnumerable<string>> GetLibraryFilesAsync(string libraryId, C
 
             (string libraryName, string libraryVersion) = _libraryNamingScheme.GetLibraryNameAndVersion(libraryId);
             string libraryFileListCacheFile = Path.Combine(_cacheFolder, libraryName, $"{libraryVersion}-filelist.json");
-            string fileListJson;
-            if (File.Exists(libraryFileListCacheFile))
-            {
-                fileListJson = await FileHelpers.ReadFileAsTextAsync(libraryFileListCacheFile, cancellationToken);
-            }
-            else
-            {
-                string libraryFileListUrl = string.Format(IsGitHub(libraryId) ? LibraryFileListUrlFormatGH : LibraryFileListUrlFormat, libraryId);
-                fileListJson = await _cacheService.GetMetadataAsync(libraryFileListUrl, libraryFileListCacheFile, cancellationToken);
-            }
+            string libraryFileListUrl = string.Format(IsGitHub(libraryId) ? LibraryFileListUrlFormatGH : LibraryFileListUrlFormat, libraryId);
+            string fileListJson = await _cacheService.GetContentsFromCachedFileWithWebRequestFallbackAsync(libraryFileListCacheFile,
+                                                                                                           libraryFileListUrl,
+                                                                                                           cancellationToken);
 
             if ((JObject)JsonConvert.DeserializeObject(fileListJson) is var fileListObject)
             {
@@ -326,22 +305,8 @@ private async Task<IEnumerable<string>> GetGithubLibraryVersionsAsync(string nam
         {
             var versions = new List<string>();
             string versionsCacheFile = Path.Combine(_cacheFolder, name, "github-versions-cache.json");
-            string versionsJson;
-            try
-            {
-                versionsJson = await _cacheService.GetMetadataAsync(string.Format(LatestLibraryVersionUrlGH, name), versionsCacheFile, cancellationToken);
-            }
-            catch (ResourceDownloadException)
-            {
-                if (File.Exists(versionsCacheFile))
-                {
-                    versionsJson = await FileHelpers.ReadFileAsTextAsync(versionsCacheFile, cancellationToken);
-                }
-                else
-                {
-                    throw;
-                }
-            }
+            string versionsUrl = string.Format(LatestLibraryVersionUrlGH, name);
+            string versionsJson = await _cacheService.GetContentsFromUriWithCacheFallbackAsync(versionsUrl, versionsCacheFile, cancellationToken);
 
             var versionsObject = (JObject)JsonConvert.DeserializeObject(versionsJson);
             var versionsArray = versionsObject["versions"] as JArray;