-
Notifications
You must be signed in to change notification settings - Fork 2.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Created v2.2 folder Updated some URI properties in skill manifest to accept URI-refences Added sample with relative URLs * Added missing files * Add validation for skill manifest examples against skill manifest schemas (#6325) * Add validation for 2.2 skill manifest * Add other skill schemas and manifest versions * Fixed 2.1 and 2.2 manifest validation errors Updated test to read files directly from the skills folder to make them easier to maintain. Updated tests to use NewtonSoft and NJSonSchema validations to provide better comat Co-authored-by: Gabo Gilabert <[email protected]> * Added titles * Fixed versions Co-authored-by: Eric Dahlvang <[email protected]>
- Loading branch information
1 parent
877a9a6
commit 2362e4f
Showing
20 changed files
with
1,252 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
25 changes: 25 additions & 0 deletions
25
schemas/skills/SchemaManifestTests/SchemaManifestTests.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>netcoreapp3.1</TargetFramework> | ||
|
|
||
| <IsPackable>false</IsPackable> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" /> | ||
| <PackageReference Include="Microsoft.VisualStudio.Threading" Version="16.10.56" /> | ||
| <PackageReference Include="Newtonsoft.Json.Schema" Version="3.0.14" /> | ||
| <PackageReference Include="NJsonSchema" Version="10.4.4" /> | ||
| <PackageReference Include="xunit" Version="2.4.1" /> | ||
| <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3"> | ||
| <PrivateAssets>all</PrivateAssets> | ||
| <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> | ||
| </PackageReference> | ||
| <PackageReference Include="coverlet.collector" Version="3.0.3"> | ||
| <PrivateAssets>all</PrivateAssets> | ||
| <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> | ||
| </PackageReference> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
25 changes: 25 additions & 0 deletions
25
schemas/skills/SchemaManifestTests/SchemaManifestTests.sln
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
|
|
||
| Microsoft Visual Studio Solution File, Format Version 12.00 | ||
| # Visual Studio Version 16 | ||
| VisualStudioVersion = 16.0.31409.214 | ||
| MinimumVisualStudioVersion = 10.0.40219.1 | ||
| Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SchemaManifestTests", "SchemaManifestTests.csproj", "{53609E6C-0C2C-4DB2-864C-628BE29495E4}" | ||
| EndProject | ||
| Global | ||
| GlobalSection(SolutionConfigurationPlatforms) = preSolution | ||
| Debug|Any CPU = Debug|Any CPU | ||
| Release|Any CPU = Release|Any CPU | ||
| EndGlobalSection | ||
| GlobalSection(ProjectConfigurationPlatforms) = postSolution | ||
| {53609E6C-0C2C-4DB2-864C-628BE29495E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU | ||
| {53609E6C-0C2C-4DB2-864C-628BE29495E4}.Debug|Any CPU.Build.0 = Debug|Any CPU | ||
| {53609E6C-0C2C-4DB2-864C-628BE29495E4}.Release|Any CPU.ActiveCfg = Release|Any CPU | ||
| {53609E6C-0C2C-4DB2-864C-628BE29495E4}.Release|Any CPU.Build.0 = Release|Any CPU | ||
| EndGlobalSection | ||
| GlobalSection(SolutionProperties) = preSolution | ||
| HideSolutionNode = FALSE | ||
| EndGlobalSection | ||
| GlobalSection(ExtensibilityGlobals) = postSolution | ||
| SolutionGuid = {DB082524-0CE2-4C94-BA0E-A9FFDBE8EE42} | ||
| EndGlobalSection | ||
| EndGlobal |
111 changes: 111 additions & 0 deletions
111
schemas/skills/SchemaManifestTests/ValidateSchemaTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Threading.Tasks; | ||
| using Newtonsoft.Json.Linq; | ||
| using Newtonsoft.Json.Schema; | ||
| using NJsonSchema; | ||
| using NJsonSchema.Generation; | ||
| using Xunit; | ||
| using JsonSchema = NJsonSchema.JsonSchema; | ||
| using JsonSchemaResolver = NJsonSchema.Generation.JsonSchemaResolver; | ||
|
|
||
| namespace SchemaManifestTests | ||
| { | ||
| /// <summary> | ||
| /// Validates sample manifests against schemas. | ||
| /// </summary> | ||
| /// <remarks> | ||
| /// There are some differences on the validation provided by Newtonsoft and NJsonSchema so we use both libraries in the tests | ||
| /// to ensure better compatibility. | ||
| /// </remarks> | ||
| public class ValidateSchemaTests | ||
| { | ||
| // List of schema version folders to test. | ||
| private static readonly List<string> _schemaVersionFolders = new List<string> | ||
| { | ||
| "v2.0", | ||
| "v2.1", | ||
| "v2.2" | ||
| }; | ||
|
|
||
| // Path to the folder containing the schema files and samples (relative to where the test is executing). | ||
| private static readonly string _schemasRootFolder = Path.Combine(Directory.GetCurrentDirectory(), "../../../../"); | ||
|
|
||
| /// <summary> | ||
| /// Builds the list of manifest schemas and samples to validate from the file system. | ||
| /// </summary> | ||
| public static TheoryData<string, string> GetManifestAndSamples() | ||
| { | ||
| var manifestAndSamples = new TheoryData<string, string>(); | ||
|
|
||
| foreach (var schemaVersion in _schemaVersionFolders) | ||
| { | ||
| var schemaFolder = Path.Combine(_schemasRootFolder, schemaVersion); | ||
| var samplesFolder = Path.Combine(schemaFolder, "Samples"); | ||
| var sampleManifestFiles = Directory.GetFileSystemEntries(samplesFolder, "*.json", SearchOption.AllDirectories); | ||
| foreach (var manifestFile in sampleManifestFiles) | ||
| { | ||
| var manifestRelativePath = Path.GetRelativePath(schemaFolder, manifestFile); | ||
| manifestAndSamples.Add(schemaVersion, manifestRelativePath); | ||
| } | ||
| } | ||
|
|
||
| return manifestAndSamples; | ||
| } | ||
|
|
||
| [Theory] | ||
| [MemberData(nameof(GetManifestAndSamples))] | ||
| public async Task ValidateManifestSamplesAgainstSchemasUsingNJsonSchemaAsync(string schemaVersion, string sampleManifest) | ||
| { | ||
| // Arrange | ||
| var manifestSchemaPath = Path.Combine(_schemasRootFolder, schemaVersion, "skill-manifest.json"); | ||
| var manifestSchema = await GetSchemaAsync(manifestSchemaPath); | ||
|
|
||
| var sampleManifestPath = Path.Combine(_schemasRootFolder, schemaVersion, sampleManifest); | ||
| var sampleManifestText = await File.ReadAllTextAsync(sampleManifestPath); | ||
|
|
||
| // Act | ||
| var validationErrors = manifestSchema.Validate(sampleManifestText); | ||
|
|
||
| // Assert | ||
| Assert.Empty(validationErrors); | ||
| } | ||
|
|
||
| [Theory] | ||
| [MemberData(nameof(GetManifestAndSamples))] | ||
| public async Task ValidateManifestSamplesAgainstSchemasUsingNewtonsoftSchemaAsync(string schemaVersion, string sampleManifest) | ||
| { | ||
| // Note: you can use https://www.jsonschemavalidator.net/ for an interactive version. | ||
|
|
||
| // Arrange | ||
| var manifestSchemaPath = Path.Combine(_schemasRootFolder, schemaVersion, "skill-manifest.json"); | ||
| var manifestSchema = JSchema.Parse(await File.ReadAllTextAsync(manifestSchemaPath), new JSchemaUrlResolver()); | ||
|
|
||
| var sampleManifestPath = Path.Combine(_schemasRootFolder, schemaVersion, sampleManifest); | ||
| var json = JToken.Parse(await File.ReadAllTextAsync(sampleManifestPath)); | ||
|
|
||
| // Act | ||
| json.IsValid(manifestSchema, out IList<ValidationError> validationErrors); | ||
|
|
||
| // Assert | ||
| Assert.Empty(validationErrors); | ||
| } | ||
|
|
||
| private static async Task<JsonSchema> GetSchemaAsync(string schemaPath) | ||
| { | ||
| var rawSchemaText = await File.ReadAllTextAsync(schemaPath); | ||
|
|
||
| return await JsonSchema.FromJsonAsync(rawSchemaText, null, x => | ||
| { | ||
| var schemaResolver = new JsonSchemaResolver(x, new JsonSchemaGeneratorSettings()); | ||
| var referenceResolver = new JsonReferenceResolver(schemaResolver); | ||
| referenceResolver.AddDocumentReference("http://json-schema.org/draft-07/schema", JsonSchema.CreateAnySchema()); | ||
|
|
||
| return referenceResolver; | ||
| }); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.