Skip to content

Bugfix/crash when no key #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Bugfix/crash when no key #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
06fd242
Moved to VS 2015 project.
Apr 11, 2017
2e41fa3
Fixed path in fsproj
Apr 11, 2017
b55961c
Part 1 of falling back to source text
Apr 11, 2017
56b70c4
Cleaned up HTML Generation
Apr 11, 2017
7e107e6
Fixed a bug where the compiler assumed any non text node was a variable
Apr 12, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ bin
obj
[Pp]ackages

test2/

# Localization Compiler Files
Resources

Expand Down
1,031 changes: 1,031 additions & 0 deletions LocalizationCompiler/.vs/config/applicationhost.config
View file
Open in desktop

Large diffs are not rendered by default.

19 changes: 15 additions & 4 deletions LocalizationCompiler/Ids.Localization.Parsers.Public/Ids.Localization.Parsers.Public.fsproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
<Name>Ids.Localization.Parsers.Public</Name>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
<RestorePackages>true</RestorePackages>
<TargetFSharpCoreVersion>4.3.0.0</TargetFSharpCoreVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
Expand All @@ -36,17 +37,27 @@
<PropertyGroup>
<MinimumVisualStudioVersion Condition="'$(MinimumVisualStudioVersion)' == ''">11</MinimumVisualStudioVersion>
</PropertyGroup>
<Import Project="$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets" Condition="Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')" />
<Import Project="$(MSBuildExtensionsPath32)\..\Microsoft F#\v4.0\Microsoft.FSharp.Targets" Condition="(!Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')) And (Exists('$(MSBuildExtensionsPath32)\..\Microsoft F#\v4.0\Microsoft.FSharp.Targets'))" />
<Import Project="$(MSBuildExtensionsPath32)\FSharp\1.0\Microsoft.FSharp.Targets" Condition="(!Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')) And (!Exists('$(MSBuildExtensionsPath32)\..\Microsoft F#\v4.0\Microsoft.FSharp.Targets')) And (Exists('$(MSBuildExtensionsPath32)\FSharp\1.0\Microsoft.FSharp.Targets'))" />
<Choose>
<When Condition="'$(VisualStudioVersion)' == '11.0'">
<PropertyGroup>
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets</FSharpTargetsPath>
</PropertyGroup>
</When>
<Otherwise>
<PropertyGroup>
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\FSharp\Microsoft.FSharp.Targets</FSharpTargetsPath>
</PropertyGroup>
</Otherwise>
</Choose>
<Import Project="$(FSharpTargetsPath)" Condition="Exists('$(FSharpTargetsPath)')" />
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
<ItemGroup>
<Compile Include="LocalizationTag.fs" />
<Content Include="Scripts\jquery-2.0.0.intellisense.js" />
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
<Reference Include="FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<Reference Include="FSharp.Core, Version=$(TargetFSharpCoreVersion), Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<Private>True</Private>
</Reference>
<Reference Include="mscorlib" />
Expand Down
7 changes: 7 additions & 0 deletions LocalizationCompiler/Ids.Localization.Parsers.Public/LocalizationTag.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,11 @@ type LocalizationTag = {
source : LocalizationSourcePart seq;
translatorNote : string;
isInAttribute : bool;
}

type IdLocalizationTag = {
id : string;
source : LocalizationSourcePart seq;
translatorNote : string;
isInAttribute : bool;
}
19 changes: 15 additions & 4 deletions LocalizationCompiler/Ids.Localization.Parsers/Ids.Localization.Parsers.fsproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
<Name>Ids.Localization.Parsers</Name>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
<RestorePackages>true</RestorePackages>
<TargetFSharpCoreVersion>4.3.0.0</TargetFSharpCoreVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
Expand Down Expand Up @@ -42,7 +43,7 @@
<HintPath>..\packages\FParsec.0.9.2.0\lib\net40\FParsecCS.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<Reference Include="FSharp.Core, Version=$(TargetFSharpCoreVersion), Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<Private>True</Private>
</Reference>
<Reference Include="mscorlib" />
Expand All @@ -64,9 +65,19 @@
<PropertyGroup>
<MinimumVisualStudioVersion Condition="'$(MinimumVisualStudioVersion)' == ''">11</MinimumVisualStudioVersion>
</PropertyGroup>
<Import Project="$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets" Condition="Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')" />
<Import Project="$(MSBuildExtensionsPath32)\..\Microsoft F#\v4.0\Microsoft.FSharp.Targets" Condition="(!Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')) And (Exists('$(MSBuildExtensionsPath32)\..\Microsoft F#\v4.0\Microsoft.FSharp.Targets'))" />
<Import Project="$(MSBuildExtensionsPath32)\FSharp\1.0\Microsoft.FSharp.Targets" Condition="(!Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')) And (!Exists('$(MSBuildExtensionsPath32)\..\Microsoft F#\v4.0\Microsoft.FSharp.Targets')) And (Exists('$(MSBuildExtensionsPath32)\FSharp\1.0\Microsoft.FSharp.Targets'))" />
<Choose>
<When Condition="'$(VisualStudioVersion)' == '11.0'">
<PropertyGroup>
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets</FSharpTargetsPath>
</PropertyGroup>
</When>
<Otherwise>
<PropertyGroup>
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\FSharp\Microsoft.FSharp.Targets</FSharpTargetsPath>
</PropertyGroup>
</Otherwise>
</Choose>
<Import Project="$(FSharpTargetsPath)" Condition="Exists('$(FSharpTargetsPath)')" />
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
Expand Down
2 changes: 1 addition & 1 deletion LocalizationCompiler/Ids.Localization.Parsers/LocalizationTagParser.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ module LocalizationTagParser =

let private parseSource = many1Till (parseVariablePart <|> parseTextPart) (spaces >>? (pchar '|' .>> spaces))

let private buildTag isInAttribute source notes = { source = source; translatorNote = notes; isInAttribute = isInAttribute }
let private buildTag isInAttribute source notes = { LocalizationTag.source = source; translatorNote = notes; isInAttribute = isInAttribute }

let private closeTag str = (manyCharsTill anyChar (spaces >>? pstring str))

Expand Down
200 changes: 103 additions & 97 deletions LocalizationCompiler/Ids.Parsers.XliffGenerator/HtmlCompiler.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,102 +6,108 @@
// Compiles localized versions of html files using xliff
// </summary>
// --------------------------------------------------------------------------------------------------------------------
namespace Ids.Generators

open Ids.Localization.Parsers.LocalizationTag

type LocalizationMatch = {
source : LocalizationSourcePart seq;
sourceText : string;
target : string;
attributeTarget : string;
translatorNote : string;
}

module XmlToLocalizationMatch =
open System
open System.IO
open System.Linq
open System.Xml

open System.Web;

open System.Text.RegularExpressions

let xliffDoc = new XmlDocument()
xliffDoc.PreserveWhitespace <- true

let manager = new XmlNamespaceManager(xliffDoc.NameTable)

let xmlReplacer = new Regex(@"<x id=[""'](.*?)[""'].*?/>", RegexOptions.IgnoreCase);
let xmlReplacerLastTag = new Regex(@"<x id=[""'](.*?)[""'].*?/>$", RegexOptions.IgnoreCase);
let replacer (m : Match) = String.Format("{{{{{0}}}}}", m.Groups.Item(1))
let attributeReplacer (m : Match) = String.Format("' + {0} + '", m.Groups.Item(1))
let attributeReplacerLastTag (m : Match) = String.Format("' + {0}", m.Groups.Item(1))

let sha256Hasher = System.Security.Cryptography.SHA256.Create()

let generate (translatedXliff : string) =
manager.AddNamespace("x", "urn:oasis:names:tc:xliff:document:1.2")

xliffDoc.Load(translatedXliff)

let lang = xliffDoc.SelectNodes("/x:xliff/x:file", manager).Item(0).Attributes.GetNamedItem("target-language").Value

let trim (sourceNode:XmlNode) (nodes:XmlNodeList) =
if nodes.Item(0) <> null && nodes.Item(0).NodeType = XmlNodeType.Whitespace then
sourceNode.RemoveChild(nodes.Item(0)) |> ignore

if nodes.Item(nodes.Count-1) <> null && nodes.Item(nodes.Count-1).NodeType = XmlNodeType.Whitespace then
sourceNode.RemoveChild(nodes.Item(nodes.Count-1)) |> ignore

sourceNode.ChildNodes

let firstNonBlank = Seq.find (not << String.IsNullOrWhiteSpace)

let nodes =
xliffDoc.SelectNodes("/x:xliff/x:file/x:body/x:trans-unit", manager)
|> Seq.cast<XmlNode>
|> Seq.map (fun x ->
let sourceNode = x.SelectNodes("x:source", manager).Item(0)
{ source = sourceNode.ChildNodes
|> trim sourceNode
|> Seq.cast<XmlNode>
|> Seq.map (fun x -> match x with
| x when x.NodeType = XmlNodeType.Text -> Text (FParsec.Text.NormalizeNewlines x.Value)
| x when x.NodeType = XmlNodeType.Whitespace -> Text (FParsec.Text.NormalizeNewlines x.Value)
| _ -> Variable (FParsec.Text.NormalizeNewlines (x.Attributes.GetNamedItem("id").Value)));
sourceText = x.SelectNodes("x:source", manager).Item(0).InnerXml;
target = x.SelectNodes("x:target", manager).Item(0).InnerXml;
attributeTarget = x.SelectNodes("x:target", manager).Item(0).InnerXml;
translatorNote = x.SelectNodes("x:note", manager).Item(0).InnerXml })
|> Seq.map (fun x -> { x with target = firstNonBlank [x.target; x.sourceText] |> HttpUtility.HtmlDecode;
attributeTarget = firstNonBlank [x.attributeTarget; x.sourceText] })
|> Seq.map (fun x ->
{ x with target = xmlReplacerLastTag.Replace(xmlReplacer.Replace(x.target, replacer), replacer);
attributeTarget = "\"'" + xmlReplacerLastTag.Replace(xmlReplacer.Replace(x.attributeTarget, attributeReplacer), attributeReplacerLastTag) + "'\"" })
(lang, nodes)

module HtmlCompiler =

open FParsec
open Ids.Localization.Parsers
namespace Ids

module Generators =
open Ids.Localization.Parsers.LocalizationTag

let private deTag (t : LocalizationSourcePart) : string =
match t with
| Text s -> s
| Variable s -> s

let private getTranslation (matches : LocalizationMatch seq) (tag : LocalizationTag) : string =
let m = (matches |> Seq.find (fun x -> (tag.source |> Seq.map (fun y -> y.GetHashCode()) |> Seq.reduce (+)) = (x.source |> Seq.map (fun y -> y.GetHashCode()) |> Seq.reduce (+))))
if tag.isInAttribute then m.attributeTarget else m.target

let generateNewHtmlFile contents (matches : LocalizationMatch seq) : string =
match run LocalizationTagParser.getChunks (Text.NormalizeNewlines contents) with
| Success (x, _, endPosition) ->
let rest = Text.NormalizeNewlines(contents).Substring(int32(endPosition.Index))
match x with
| [] -> rest
| x -> (x |> Seq.map (fun (a, b) -> a + (getTranslation matches b)) |> Seq.reduce (+)) + rest
| Failure (reasons, state, _) -> reasons
open Ids.Localization.Parsers.XliffGenerator.XliffGenerator

type LocalizationMatch = {
id : string;
source : LocalizationSourcePart seq;
target : LocalizationSourcePart seq;
attributeTarget : string;
}

module XmlToLocalizationMatch =
open System
open System.IO
open System.Linq
open System.Xml

open System.Web;

open System.Text.RegularExpressions

let xliffDoc = new XmlDocument()
xliffDoc.PreserveWhitespace <- true

let manager = new XmlNamespaceManager(xliffDoc.NameTable)

let private xmlToLM(node : XmlNode) =
node.ChildNodes.Cast<XmlNode>()
|> Seq.map(fun n -> match n.NodeType with
| XmlNodeType.Element -> Variable (n.Attributes.GetNamedItem("id").Value)
| _ -> Text n.Value)

let generate (translatedXliff : string) =
manager.AddNamespace("x", "urn:oasis:names:tc:xliff:document:1.2")

xliffDoc.Load(translatedXliff)

let lang = xliffDoc.SelectNodes("/x:xliff/x:file", manager).Item(0).Attributes.GetNamedItem("target-language").Value

let trim (sourceNode:XmlNode) (nodes:XmlNodeList) =
if nodes.Item(0) <> null && nodes.Item(0).NodeType = XmlNodeType.Whitespace then
sourceNode.RemoveChild(nodes.Item(0)) |> ignore

if nodes.Item(nodes.Count-1) <> null && nodes.Item(nodes.Count-1).NodeType = XmlNodeType.Whitespace then
sourceNode.RemoveChild(nodes.Item(nodes.Count-1)) |> ignore

sourceNode.ChildNodes

let nodes =
xliffDoc.SelectNodes("/x:xliff/x:file/x:body/x:trans-unit", manager)
|> Seq.cast<XmlNode>
|> Seq.map (fun x ->
{
id = x.Attributes.GetNamedItem("id").Value;
source = xmlToLM(x.SelectNodes("x:source", manager).Item(0));
target = xmlToLM(x.SelectNodes("x:target", manager).Item(0));
attributeTarget = x.SelectNodes("x:target", manager).Item(0).InnerXml;
})
(lang, nodes)

module HtmlCompiler =
open System
open FParsec
open Ids.Localization.Parsers

let private deTag (t : LocalizationSourcePart) : string =
match t with
| Text s -> s
| Variable s -> "{{" + s + "}}"

let private deTagAttr (t : LocalizationSourcePart) : string =
match t with
| Text s -> "'" + s + "'"
| Variable s -> s

let private lspToTarg(lsp : LocalizationSourcePart seq) =
String.Join("", lsp |> Seq.map deTag)

let private lspToAttr(lsp : LocalizationSourcePart seq) =
"\"" + String.Join(" + ", lsp |> Seq.map deTagAttr ) + "\""


let firstNonEmpty = Seq.find (not << Seq.isEmpty)

let private getTranslation (matches : LocalizationMatch seq) (tag : IdLocalizationTag) : string =
let optionalMatch = matches |> Seq.tryFind (fun x -> tag.id = x.id)
match optionalMatch with
| Some m -> if tag.isInAttribute
then lspToAttr(firstNonEmpty [m.target; m.source])
else lspToTarg(firstNonEmpty [m.target; m.source])
| None -> if tag.isInAttribute
then lspToAttr(tag.source)
else lspToTarg(tag.source)

let generateNewHtmlFile contents (matches : LocalizationMatch seq) : string =
match run LocalizationTagParser.getChunks (Text.NormalizeNewlines contents) with
| Success (x, _, endPosition) ->
let rest = Text.NormalizeNewlines(contents).Substring(int32(endPosition.Index))
match x with
| [] -> rest
| x -> (x |> Seq.map(fun (leader, tag) -> (leader, generateIdTag(tag))) |> Seq.map (fun (leader, tag) -> leader + (getTranslation matches tag)) |> Seq.reduce (+)) + rest
| Failure (reasons, state, _) -> reasons
19 changes: 15 additions & 4 deletions LocalizationCompiler/Ids.Parsers.XliffGenerator/Ids.Parsers.XliffGenerator.fsproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
<RestorePackages>true</RestorePackages>
<TargetFrameworkProfile />
<TargetFSharpCoreVersion>4.3.0.0</TargetFSharpCoreVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
Expand All @@ -37,9 +38,19 @@
<PropertyGroup>
<MinimumVisualStudioVersion Condition="'$(MinimumVisualStudioVersion)' == ''">11</MinimumVisualStudioVersion>
</PropertyGroup>
<Import Project="$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets" Condition="Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')" />
<Import Project="$(MSBuildExtensionsPath32)\..\Microsoft F#\v4.0\Microsoft.FSharp.Targets" Condition="(!Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')) And (Exists('$(MSBuildExtensionsPath32)\..\Microsoft F#\v4.0\Microsoft.FSharp.Targets'))" />
<Import Project="$(MSBuildExtensionsPath32)\FSharp\1.0\Microsoft.FSharp.Targets" Condition="(!Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')) And (!Exists('$(MSBuildExtensionsPath32)\..\Microsoft F#\v4.0\Microsoft.FSharp.Targets')) And (Exists('$(MSBuildExtensionsPath32)\FSharp\1.0\Microsoft.FSharp.Targets'))" />
<Choose>
<When Condition="'$(VisualStudioVersion)' == '11.0'">
<PropertyGroup>
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets</FSharpTargetsPath>
</PropertyGroup>
</When>
<Otherwise>
<PropertyGroup>
<FSharpTargetsPath>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\FSharp\Microsoft.FSharp.Targets</FSharpTargetsPath>
</PropertyGroup>
</Otherwise>
</Choose>
<Import Project="$(FSharpTargetsPath)" Condition="Exists('$(FSharpTargetsPath)')" />
<Import Project="$(SolutionDir)\.nuget\nuget.targets" />
<ItemGroup>
<Compile Include="XliffGenerator.fs" />
Expand All @@ -60,7 +71,7 @@
<HintPath>..\packages\FParsec.0.9.2.0\lib\net40\FParsecCS.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<Reference Include="FSharp.Core, Version=$(TargetFSharpCoreVersion), Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<Private>True</Private>
</Reference>
<Reference Include="mscorlib" />
Expand Down
Loading