diff --git a/README.markdown b/README.markdown index 66545f6..658ba99 100644 --- a/README.markdown +++ b/README.markdown @@ -7,9 +7,19 @@ When people need a list of root certificates, they often turn to Mozilla's. Howe Several people have written quick scripts to try and convert this into PEM format, but they often miss something critical: some certificates are explicitly _distrusted_. These include the DigiNotar certificates and the misissued COMODO certificates. If you don't parse the trust records from the NSS data file, then you end up trusting these! -So this is a tool that I wrote for converting the NSS file to PEM format which is also aware of the trust records. It can be built with Go1. See http://golang.org/doc/install.html, but don't pass "-u release" when fetching the repository. +So this is a tool that was written for converting the NSS file to PEM format which is also aware of the trust records. It can be built with Go 1.3. See http://golang.org/doc/install.html, but don't pass "-u release" when fetching the repository. Once you have Go installed please do the following: % curl https://hg.mozilla.org/mozilla-central/raw-file/tip/security/nss/lib/ckfw/builtins/certdata.txt -o certdata.txt - % go run convert_mozilla_certdata.go > certdata.new + % go run main.go > certdata.new + +To use as a library import it like the following: + + import "github.com/njones/nss/nss" + +Then use: + + output := nss.ParseInput(file) + +This will give you a slice of nss.Blocks that contain the x509 cert along with a UTF-8 encode label. This can then be added to things like a TrustPool http://golang.org/pkg/crypto/x509/#CertPool \ No newline at end of file diff --git a/convert_mozilla_certdata.go b/convert_mozilla_certdata.go deleted file mode 100644 index 9597998..0000000 --- a/convert_mozilla_certdata.go +++ /dev/null @@ -1,512 +0,0 @@ -// Copyright 2012 Google Inc. All Rights Reserved. -// Author: agl@chromium.org (Adam Langley) - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This utility parses Mozilla's certdata.txt and extracts a list of trusted -// certificates in PEM form. -// -// A current version of certdata.txt can be downloaded from: -// https://hg.mozilla.org/mozilla-central/raw-file/tip/security/nss/lib/ckfw/builtins/certdata.txt -package main - -import ( - "bufio" - "bytes" - "crypto" - _ "crypto/md5" - "crypto/sha1" - _ "crypto/sha256" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "flag" - "fmt" - "io" - "log" - "os" - "strconv" - "strings" - "unicode/utf8" -) - -// Object represents a collection of attributes from the certdata.txt file -// which are usually either certificates or trust records. -type Object struct { - attrs map[string]Attribute - startingLine int // the line number that the object started on. -} - -type Attribute struct { - attrType string - value []byte -} - -var ( - // ignoreList maps from CKA_LABEL values (from the upstream roots file) - // to an optional comment which is displayed when skipping matching - // certificates. - ignoreList map[string]string - - includedUntrustedFlag = flag.Bool("include-untrusted", false, "If set, untrusted certificates will also be included in the output") - toFiles = flag.Bool("to-files", false, "If set, individual certificate files will be created in the current directory") - ignoreListFilename = flag.String("ignore-list", "", "File containing a list of certificates to ignore") -) - -func main() { - - flag.Parse() - - inFilename := "certdata.txt" - if len(flag.Args()) == 1 { - inFilename = flag.Arg(0) - } else if len(flag.Args()) > 1 { - fmt.Printf("Usage: %s []\n", os.Args[0]) - os.Exit(1) - } - - ignoreList = make(map[string]string) - if *ignoreListFilename != "" { - ignoreListFile, err := os.Open(*ignoreListFilename) - if err != nil { - log.Fatalf("Failed to open ignore-list file: %s", err) - } - parseIgnoreList(ignoreListFile) - ignoreListFile.Close() - } - - inFile, err := os.Open(inFilename) - if err != nil { - log.Fatalf("Failed to open input file: %s", err) - } - - license, cvsId, objects := parseInput(inFile) - inFile.Close() - - if !*toFiles { - os.Stdout.WriteString(license) - if len(cvsId) > 0 { - os.Stdout.WriteString("CVS_ID " + cvsId + "\n") - } - } - - outputTrustedCerts(os.Stdout, objects) -} - -// parseIgnoreList parses the ignore-list file into ignoreList -func parseIgnoreList(ignoreListFile io.Reader) { - in := bufio.NewReader(ignoreListFile) - var lineNo int - - for line, eof := getLine(in, &lineNo); !eof; line, eof = getLine(in, &lineNo) { - if split := strings.SplitN(line, "#", 2); len(split) == 2 { - // this line has an additional comment - ignoreList[strings.TrimSpace(split[0])] = strings.TrimSpace(split[1]) - } else { - ignoreList[line] = "" - } - } -} - -// parseInput parses a certdata.txt file into it's license blob, the CVS id (if -// included) and a set of Objects. -func parseInput(inFile io.Reader) (license, cvsId string, objects []*Object) { - in := bufio.NewReader(inFile) - var lineNo int - - // Discard anything prior to the license block. - for line, eof := getLine(in, &lineNo); !eof; line, eof = getLine(in, &lineNo) { - if strings.Contains(line, "This Source Code") { - license += line - license += "\n" - break - } - } - if len(license) == 0 { - log.Fatalf("Read whole input and failed to find beginning of license") - } - // Now collect the license block. - // certdata.txt from hg.mozilla.org no longer contains CVS_ID. - for line, eof := getLine(in, &lineNo); !eof; line, eof = getLine(in, &lineNo) { - if strings.Contains(line, "CVS_ID") || len(line) == 0 { - break - } - license += line - license += "\n" - } - - var currentObject *Object - var beginData bool - - for line, eof := getLine(in, &lineNo); !eof; line, eof = getLine(in, &lineNo) { - if len(line) == 0 || line[0] == '#' { - continue - } - - if strings.HasPrefix(line, "CVS_ID ") { - cvsId = line[7:] - continue - } - if line == "BEGINDATA" { - beginData = true - continue - } - - words := strings.Fields(line) - var value []byte - if len(words) == 2 && words[1] == "MULTILINE_OCTAL" { - startingLine := lineNo - var ok bool - value, ok = readMultilineOctal(in, &lineNo) - if !ok { - log.Fatalf("Failed to read octal value starting at line %d", startingLine) - } - } else if len(words) < 3 { - log.Fatalf("Expected three or more values on line %d, but found %d", lineNo, len(words)) - } else { - value = []byte(strings.Join(words[2:], " ")) - } - - if words[0] == "CKA_CLASS" { - // Start of a new object. - if currentObject != nil { - objects = append(objects, currentObject) - } - currentObject = new(Object) - currentObject.attrs = make(map[string]Attribute) - currentObject.startingLine = lineNo - } - if currentObject == nil { - log.Fatalf("Found attribute on line %d which appears to be outside of an object", lineNo) - } - currentObject.attrs[words[0]] = Attribute{ - attrType: words[1], - value: value, - } - } - - if !beginData { - log.Fatalf("Read whole input and failed to find BEGINDATA") - } - - if currentObject != nil { - objects = append(objects, currentObject) - } - - return -} - -// outputTrustedCerts writes a series of PEM encoded certificates to out by -// finding certificates and their trust records in objects. -func outputTrustedCerts(out *os.File, objects []*Object) { - certs := filterObjectsByClass(objects, "CKO_CERTIFICATE") - trusts := filterObjectsByClass(objects, "CKO_NSS_TRUST") - filenames := make(map[string]bool) - - for _, cert := range certs { - derBytes := cert.attrs["CKA_VALUE"].value - hash := sha1.New() - hash.Write(derBytes) - digest := hash.Sum(nil) - - label := string(cert.attrs["CKA_LABEL"].value) - if comment, present := ignoreList[strings.Trim(label, "\"")]; present { - var sep string - if len(comment) > 0 { - sep = ": " - } - log.Printf("Skipping explicitly ignored certificate: %s%s%s", label, sep, comment) - continue - } - - x509, err := x509.ParseCertificate(derBytes) - if err != nil { - // This is known to occur because of a broken certificate in NSS. - // https://bugzilla.mozilla.org/show_bug.cgi?id=707995 - log.Printf("Failed to parse certificate starting on line %d: %s", cert.startingLine, err) - continue - } - - // TODO(agl): wtc tells me that Mozilla might get rid of the - // SHA1 records in the future and use issuer and serial number - // to match trust records to certificates (which is what NSS - // currently uses). This needs some changes to the crypto/x509 - // package to keep the raw names around. - - var trust *Object - for _, possibleTrust := range trusts { - if bytes.Equal(digest, possibleTrust.attrs["CKA_CERT_SHA1_HASH"].value) { - trust = possibleTrust - break - } - } - - if trust == nil { - log.Fatalf("No trust found for certificate object starting on line %d (sha1: %x)", cert.startingLine, digest) - } - - trustType := trust.attrs["CKA_TRUST_SERVER_AUTH"].value - if len(trustType) == 0 { - log.Fatalf("No CKA_TRUST_SERVER_AUTH found in trust starting at line %d", trust.startingLine) - } - - var trusted bool - switch string(trustType) { - case "CKT_NSS_NOT_TRUSTED": - // An explicitly distrusted cert - trusted = false - case "CKT_NSS_TRUSTED_DELEGATOR": - // A cert trusted for issuing SSL server certs. - trusted = true - case "CKT_NSS_TRUST_UNKNOWN", "CKT_NSS_MUST_VERIFY_TRUST": - // A cert not trusted for issuing SSL server certs, but is trusted for other purposes. - trusted = false - default: - log.Fatalf("Unknown trust value '%s' found for trust record starting on line %d", trustType, trust.startingLine) - } - - if !trusted && !*includedUntrustedFlag { - continue - } - - block := &pem.Block{Type: "CERTIFICATE", Bytes: derBytes} - - if *toFiles { - if strings.HasPrefix(label, "\"") { - label = label[1:] - } - if strings.HasSuffix(label, "\"") { - label = label[:len(label)-1] - } - // The label may contain hex-escaped, UTF-8 charactors. - label = unescapeLabel(label) - label = strings.Replace(label, " ", "_", -1) - label = strings.Replace(label, "/", "_", -1) - - filename := label - for i := 2; ; i++ { - if _, ok := filenames[filename]; !ok { - break - } - - filename = label + "-" + strconv.Itoa(i) - } - filenames[filename] = true - - file, err := os.Create(filename + ".pem") - if err != nil { - log.Fatalf("Failed to create output file: %s\n", err) - } - pem.Encode(file, block) - file.Close() - out.WriteString(filename + ".pem\n") - continue - } - - out.WriteString("\n") - if !trusted { - out.WriteString("# NOT TRUSTED FOR SSL\n") - } - - out.WriteString("# Issuer: " + nameToString(x509.Issuer) + "\n") - out.WriteString("# Subject: " + nameToString(x509.Subject) + "\n") - out.WriteString("# Label: " + label + "\n") - out.WriteString("# Serial: " + x509.SerialNumber.String() + "\n") - out.WriteString("# MD5 Fingerprint: " + fingerprintString(crypto.MD5, x509.Raw) + "\n") - out.WriteString("# SHA1 Fingerprint: " + fingerprintString(crypto.SHA1, x509.Raw) + "\n") - out.WriteString("# SHA256 Fingerprint: " + fingerprintString(crypto.SHA256, x509.Raw) + "\n") - pem.Encode(out, block) - } -} - -// nameToString converts name into a string representation containing the -// CommonName, Organization and OrganizationalUnit. -func nameToString(name pkix.Name) string { - ret := "" - if len(name.CommonName) > 0 { - ret += "CN=" + name.CommonName - } - - if org := strings.Join(name.Organization, "/"); len(org) > 0 { - if len(ret) > 0 { - ret += " " - } - ret += "O=" + org - } - - if orgUnit := strings.Join(name.OrganizationalUnit, "/"); len(orgUnit) > 0 { - if len(ret) > 0 { - ret += " " - } - ret += "OU=" + orgUnit - } - - return ret -} - -// filterObjectsByClass returns a subset of in where each element has the given -// class. -func filterObjectsByClass(in []*Object, class string) (out []*Object) { - for _, object := range in { - if string(object.attrs["CKA_CLASS"].value) == class { - out = append(out, object) - } - } - return -} - -// readMultilineOctal converts a series of lines of octal values into a slice -// of bytes. -func readMultilineOctal(in *bufio.Reader, lineNo *int) ([]byte, bool) { - var value []byte - - for line, eof := getLine(in, lineNo); !eof; line, eof = getLine(in, lineNo) { - if line == "END" { - return value, true - } - - for _, octalStr := range strings.Split(line, "\\") { - if len(octalStr) == 0 { - continue - } - v, err := strconv.ParseUint(octalStr, 8, 8) - if err != nil { - log.Printf("error converting octal string '%s' on line %d", octalStr, *lineNo) - return nil, false - } - value = append(value, byte(v)) - } - } - - // Missing "END" - return nil, false -} - -// getLine reads the next line from in, aborting in the event of an error. -func getLine(in *bufio.Reader, lineNo *int) (string, bool) { - *lineNo++ - line, isPrefix, err := in.ReadLine() - if err == io.EOF { - return "", true - } - if err != nil { - log.Fatalf("I/O error while reading input: %s", err) - } - if isPrefix { - log.Fatalf("Line too long while reading line %d", *lineNo) - } - return string(line), false -} - -func fingerprintString(hashFunc crypto.Hash, data []byte) string { - hash := hashFunc.New() - hash.Write(data) - digest := hash.Sum(nil) - - hex := fmt.Sprintf("%x", digest) - ret := "" - for len(hex) > 0 { - if len(ret) > 0 { - ret += ":" - } - todo := 2 - if len(hex) < todo { - todo = len(hex) - } - ret += hex[:todo] - hex = hex[todo:] - } - - return ret -} - -func isHex(c rune) (value byte, ok bool) { - switch { - case c >= '0' && c <= '9': - return byte(c) - '0', true - case c >= 'a' && c <= 'f': - return byte(c) - 'a' + 10, true - case c >= 'A' && c <= 'F': - return byte(c) - 'A' + 10, true - } - - return 0, false -} - -func appendRune(out []byte, r rune) []byte { - if r < 128 { - return append(out, byte(r)) - } - - var buf [utf8.UTFMax]byte - n := utf8.EncodeRune(buf[:], r) - return append(out, buf[:n]...) -} - -// unescapeLabel unescapes "\xab" style hex-escapes. -func unescapeLabel(escaped string) string { - var out []byte - var last rune - var value byte - state := 0 - - for _, r := range escaped { - switch state { - case 0: - if r == '\\' { - state++ - continue - } - case 1: - if r == 'x' { - state++ - continue - } - out = append(out, '\\') - case 2: - if v, ok := isHex(r); ok { - value = v - last = r - state++ - continue - } else { - out = append(out, '\\', 'x') - } - case 3: - if v, ok := isHex(r); ok { - value <<= 4 - value += v - out = append(out, byte(value)) - state = 0 - continue - } else { - out = append(out, '\\', 'x') - out = appendRune(out, last) - } - } - state = 0 - out = appendRune(out, r) - } - - switch state { - case 3: - out = append(out, '\\', 'x') - out = appendRune(out, last) - case 2: - out = append(out, '\\', 'x') - case 1: - out = append(out, '\\') - } - - return string(out) -} diff --git a/main.go b/main.go new file mode 100644 index 0000000..1095416 --- /dev/null +++ b/main.go @@ -0,0 +1,152 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// Author: agl@chromium.org (Adam Langley) + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This utility parses Mozilla's certdata.txt and extracts a list of trusted +// certificates in PEM form. +// +// A current version of certdata.txt can be downloaded from: +// https://hg.mozilla.org/mozilla-central/raw-file/tip/security/nss/lib/ckfw/builtins/certdata.txt +package main + +import ( + "crypto" + "encoding/pem" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "strconv" + "strings" + + "github.com/njones/nss/nss" +) + +// The flags that can be used for the command line options +var ( + quietFlg = flag.Bool("quiet", false, "If set, there will be no output to the display") + toFilesFlg = flag.Bool("to-files", false, "If set, individual certificate files will be created in the current directory") + includeUntrustedFlg = flag.Bool("include-untrusted", false, "If set, untrusted certificates will also be included in the output") + ignoreListFilenameFlg = flag.String("ignore-list", "", "File containing a list of certificates to ignore") +) + +func main() { + var display io.Writer + + flag.Parse() + + // Set up the ignore list + var ignoreList nss.IgnoreList + if *ignoreListFilenameFlg != "" { + ignoreListFile, err := os.Open(*ignoreListFilenameFlg) + if err != nil { + log.Fatalf("Failed to open ignore-list file: %s", err) + } + ignoreList = nss.ParseIgnoreList(ignoreListFile) + ignoreListFile.Close() + } + + // Set up the name of the file we are going to read in + dataFilename := "certdata.txt" + if len(flag.Args()) == 1 { + dataFilename = flag.Arg(0) + } else if len(flag.Args()) > 1 { + fmt.Printf("Usage: %s []\n", os.Args[0]) + os.Exit(1) + } + + file, err := os.Open(dataFilename) + if err != nil { + log.Fatalf("Failed to open input file: %s", err) + } + + license, cvsId, objects := nss.ParseInput(file) + file.Close() + + // Get back the certs from the parsed input + var nssBlocks []nss.Block + if *includeUntrustedFlg { + nssBlocks = nss.AllCertificates(objects, ignoreList) + } else { + nssBlocks = nss.TrustedCertificates(objects, ignoreList) + } + + // Set the display to default to outputting to a screen. But if -quiet is used, then discard + display = os.Stdout + if *quietFlg { + display = ioutil.Discard + } + + if !*toFilesFlg { + fmt.Fprint(display, license) + if len(cvsId) > 0 { + fmt.Fprintln(display, "CVS_ID", cvsId) + } + } + + filenames := make(map[string]bool) + for _, nssBlock := range nssBlocks { + + label := nssBlock.Label + x509 := nssBlock.Cert + block := &pem.Block{Type: "CERTIFICATE", Bytes: x509.Raw} + + // If we are going to output to files then do all of the label stuff + // This is going to be somewhat slower than if the "if/then" block is + // outside of the for loop (because it would be evaluated once) however + // i/o to disk will be a bigger bottle neck, so it's acceptable for + // code clarity. Or if not... feel free to refactor. + if *toFilesFlg { + + // Remove all of the leading and trailing "'s and ' 's + // that's quotes and spaces... + label = strings.Trim(label, " \"") + + // The label may contain hex-escaped, UTF-8 characters. + label = nss.DecodeHexEscapedString(label) + label = strings.Replace(label, " ", "_", -1) + label = strings.Replace(label, "/", "_", -1) + + filename := label + for i := 2; ; i++ { + if _, ok := filenames[filename]; !ok { + break + } + + filename = label + "-" + strconv.Itoa(i) + } + filenames[filename] = true + + file, err := os.Create(filename + ".pem") + if err != nil { + log.Fatalf("Failed to create output file: %s\n", err) + } + + pem.Encode(file, block) + file.Close() + } + + fmt.Fprintln(display) // Just give a line space between output + fmt.Fprintln(display, "# Issuer:", nss.Field(x509.Issuer)) + fmt.Fprintln(display, "# Subject:", nss.Field(x509.Subject)) + fmt.Fprintln(display, "# Label:", label) + fmt.Fprintln(display, "# Serial:", x509.SerialNumber.String()) + fmt.Fprintln(display, "# MD5 Fingerprint:", nss.Fingerprint(crypto.MD5, x509.Raw)) + fmt.Fprintln(display, "# SHA1 Fingerprint:", nss.Fingerprint(crypto.SHA1, x509.Raw)) + fmt.Fprintln(display, "# SHA256 Fingerprint:", nss.Fingerprint(crypto.SHA256, x509.Raw)) + pem.Encode(display, block) + } +} \ No newline at end of file diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..b820aea --- /dev/null +++ b/main_test.go @@ -0,0 +1,11 @@ +package main + +import ( + "testing" +) + +func BenchmarkMain(b *testing.B) { + for i := 0; i < b.N; i++ { + main() + } +} diff --git a/nss/nss.go b/nss/nss.go new file mode 100644 index 0000000..f153549 --- /dev/null +++ b/nss/nss.go @@ -0,0 +1,428 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// Author: agl@chromium.org (Adam Langley) + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This utility parses Mozilla's certdata.txt and extracts a list of trusted +// certificates in PEM form. +// +// A current version of certdata.txt can be downloaded from: +// https://hg.mozilla.org/mozilla-central/raw-file/tip/security/nss/lib/ckfw/builtins/certdata.txt +package nss + +import ( + "bufio" + "bytes" + "crypto" + _ "crypto/md5" + "crypto/sha1" + _ "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "fmt" + "io" + "log" + "strconv" + "strings" +) + +// Block is the exported type from this lib. It has the label and the cert in Binary form +type Block struct { + Label string + Cert *x509.Certificate +} + +// IgnoreList is where all of the strings for certs to ignore a held. +// it maps from CKA_LABEL values (from the upstream roots file) +// to an optional comment which is displayed when skipping matching +// certificates. +type IgnoreList map[string]string + +// object represents a collection of attributes from the certdata.txt file +// which are usually either certificates or trust records. +type object struct { + attrs map[string]attribute + startingLine int // the line number that the object started on. +} + +// attribute are the attributes for a CKA_CLASS object +type attribute struct { + attrType string + value []byte +} + +// filterObjectsByClass returns a subset of in where each element has the given +// class. +func filterObjectsByClass(in []*object, class string) (out []*object) { + for _, o := range in { + if string(o.attrs["CKA_CLASS"].value) == class { + out = append(out, o) + } + } + + return +} + +// parseLicenseBlock parses the license block out of the current scan of text +func parseLicenseBlock(in *bufio.Scanner, ln int) (lineNo int, license, cvsId string) { + license += in.Text() + "\n" // Add this line to the license string + + // Loop through the next lines until we get to an blank line + for in.Scan() { + + // Advance the line count and grab the line + ln += 1 + line := in.Text() + + // Check to see if there is a CVS_ID line within the license + if strings.HasPrefix(line, "CVS_ID ") { + cvsId = line[7:] + continue + } + + // If the line is blank then we can exit out of the license loop + if len(line) == 0 { + break + } + license += line + "\n" // Add this line to the license string. + } + + return ln, license, cvsId +} + +// parseMultiLineOctal parses the octal encoding which can span multiple +// lines out of the blocks as binary data +func parseMultiLineOctal(in *bufio.Scanner, ln int) (lineNo int, value []byte) { + // Loop through the next lines (inner-loop 2) + for in.Scan() { + + // Advance the line count and grab the line + ln += 1 + line := in.Text() + + // If we've hit the end of the block then break out of (inner-loop 2) + // and go back to inner-loop 1 + if line == "END" { + break + } + + // Split all of the octal encodings for the line out. + for _, octalStr := range strings.Split(line, `\`) { + if len(octalStr) == 0 { + continue + } + + // Parse the string value to a int8 (byte) value + v, err := strconv.ParseUint(octalStr, 8, 8) + if err != nil { + log.Fatalf("error converting octal string '%s' on line %d", octalStr, lineNo) + } + + // Append all of the bytes + value = append(value, byte(v)) + } + } + + return ln, value +} + +// parseCkaClassObject parses the CKA_CLASS blocks as an object +func parseCkaClassObject(in *bufio.Scanner, ln int, cka *object) (lineNo int, o *object) { + // Loop through the lines of the CKA_CLASS and add to the object + for in.Scan() { + + ln += 1 + line := in.Text() + + // This signifies the last octal block of an object + if len(line) == 0 || line[0] == '#' { + break + } + + var value []byte + words := strings.Fields(line) + + if len(words) == 2 && words[1] == "MULTILINE_OCTAL" { + ln, value = parseMultiLineOctal(in, ln) + } else if len(words) < 3 { + log.Fatalf("Expected three or more values on line %d, but found %d", lineNo, len(words)) + } else { + lineNo += 1 + value = []byte(strings.Join(words[2:], " ")) + } + + cka.attrs[words[0]] = attribute{words[1], value} + } + + return ln, cka +} + +// ParseIgnoreList parses the ignore-list file into IgnoreList +func ParseIgnoreList(file io.Reader) (ignoreList IgnoreList) { + ignoreList = make(IgnoreList) + in := bufio.NewScanner(file) + + for in.Scan() { + line := in.Text() + if split := strings.SplitN(line, "#", 2); len(split) == 2 { + // this line has an additional comment + ignoreList[strings.TrimSpace(split[0])] = strings.TrimSpace(split[1]) + } else { + ignoreList[line] = "" + } + } + + return +} + +// ParseInput parses a certdata.txt file into it's license blob, the CVS id (if +// included) and a set of Objects. +func ParseInput(file io.Reader) (license, cvsId string, objects []*object) { + in := bufio.NewScanner(file) + + var lineNo int + var hasLicense bool + var hasBeginData bool + + for in.Scan() { + + lineNo += 1 + line := in.Text() + + // Collect the license block + // Loop until we get the line "This Source Code" ... + if strings.Contains(line, "This Source Code") { + hasLicense = true // We have found a license, so set this check to true. + lineNo, license, cvsId = parseLicenseBlock(in, lineNo) + } + + // Loop until we get to the line BEGINDATA + if line == "BEGINDATA" { + hasBeginData = true + + // Now finish the scanning of the document here (inner-loop 1). We shouldn't need to go back to the outer loop + for in.Scan() { + + // Advance the line count and grab the line + lineNo += 1 + line := in.Text() + + // Skip all of the comments + if len(line) == 0 || line[0] == '#' { + continue + } + + // See what words are on this line + words := strings.Fields(line) + + // CKA_CLASS are the magic words to set up an object, so lets start a new object + if words[0] == "CKA_CLASS" { + + ckaClass := new(object) + ckaClass.startingLine = lineNo + ckaClass.attrs = map[string]attribute{ + words[0]: attribute{ + words[1], + []byte(strings.Join(words[2:], " ")), + }, + } + + lineNo, ckaClass = parseCkaClassObject(in, lineNo, ckaClass) + objects = append(objects, ckaClass) + } + } + } + } + + if !hasLicense { + log.Fatalf("Read whole input and failed to find beginning of license") + } + + if !hasBeginData { + log.Fatalf("Read whole input and failed to find BEGINDATA") + } + + return +} + +// TrustedCertificates returns all of the parsed objects that have an +// associated trust certificate. An optional ignoreList can be passed +// along. If more than one is passed, then it is ignored. +func TrustedCertificates(objects []*object, il ...IgnoreList) []Block { + ignoreList := make(map[string]string) + if len(il) > 0 { + ignoreList = il[0] + } + + return certs(objects, ignoreList, false) +} + +// AllCertificates returns all of the parsed objects regardless of whether +// there is an associated trust certificate. An optional ignoreList can +// be passed along. If more than one is passed, then it is ignored. +func AllCertificates(objects []*object, il ...IgnoreList) []Block { + ignoreList := make(map[string]string) + if len(il) > 0 { + ignoreList = il[0] + } + + return certs(objects, ignoreList, true) +} + +// certs writes a series of PEM encoded certificates to out by +// finding certificates and their trust records in objects. +// The output is a slice of Blocks that include the label and x509 cert +func certs(objects []*object, ignoreList IgnoreList, includeUntrusted bool) (blocks []Block) { + certs := filterObjectsByClass(objects, "CKO_CERTIFICATE") + trusts := filterObjectsByClass(objects, "CKO_NSS_TRUST") + + for _, cert := range certs { + derBytes := cert.attrs["CKA_VALUE"].value + hash := sha1.New() + hash.Write(derBytes) + digest := hash.Sum(nil) + + label := string(cert.attrs["CKA_LABEL"].value) + if comment, present := ignoreList[strings.Trim(label, "\"")]; present { + var sep string + if len(comment) > 0 { + sep = ": " + } + log.Printf("Skipping explicitly ignored certificate: %s%s%s", label, sep, comment) + continue + } + + x509, err := x509.ParseCertificate(derBytes) + if err != nil { + // This is known to occur because of a broken certificate in NSS. + // https://bugzilla.mozilla.org/show_bug.cgi?id=707995 + log.Printf("Failed to parse certificate starting on line %d: %s", cert.startingLine, err) + continue + } + + // TODO(agl): wtc tells me that Mozilla might get rid of the + // SHA1 records in the future and use issuer and serial number + // to match trust records to certificates (which is what NSS + // currently uses). This needs some changes to the crypto/x509 + // package to keep the raw names around. + + var trust *object + for _, possibleTrust := range trusts { + if bytes.Equal(digest, possibleTrust.attrs["CKA_CERT_SHA1_HASH"].value) { + trust = possibleTrust + break + } + } + + if trust == nil { + log.Fatalf("No trust found for certificate object starting on line %d (sha1: %x)", cert.startingLine, digest) + } + + trustType := trust.attrs["CKA_TRUST_SERVER_AUTH"].value + if len(trustType) == 0 { + log.Fatalf("No CKA_TRUST_SERVER_AUTH found in trust starting at line %d", trust.startingLine) + } + + var trusted bool + switch string(trustType) { + case "CKT_NSS_NOT_TRUSTED": + // An explicitly distrusted cert + trusted = false + case "CKT_NSS_TRUSTED_DELEGATOR": + // A cert trusted for issuing SSL server certs. + trusted = true + case "CKT_NSS_TRUST_UNKNOWN", "CKT_NSS_MUST_VERIFY_TRUST": + // A cert not trusted for issuing SSL server certs, but is trusted for other purposes. + trusted = false + default: + log.Fatalf("Unknown trust value '%s' found for trust record starting on line %d", trustType, trust.startingLine) + } + + if !trusted && !includeUntrusted { + continue + } + + blocks = append(blocks, Block{label, x509}) + } + + return +} + +// Field converts name into a string representation containing the +// CommonName, Organization and OrganizationalUnit. +func Field(name pkix.Name) string { + ret := "" + if len(name.CommonName) > 0 { + ret += "CN=" + name.CommonName + } + + if org := strings.Join(name.Organization, "/"); len(org) > 0 { + if len(ret) > 0 { + ret += " " + } + ret += "O=" + org + } + + if orgUnit := strings.Join(name.OrganizationalUnit, "/"); len(orgUnit) > 0 { + if len(ret) > 0 { + ret += " " + } + ret += "OU=" + orgUnit + } + + return ret +} + +// Fingerprint returns the passed in hash (MD5, SHA1, SHA256) in a +// fingerprint format (i.e. AA:0B:DC:... ) +func Fingerprint(hashFunc crypto.Hash, data []byte) string { + hash := hashFunc.New() + hash.Write(data) + digest := hash.Sum(nil) + + // Print out Hex numbers with a space, then replace that space with a colon. + return strings.Replace(fmt.Sprintf("% x", digest), " ", ":", -1) +} + +// DecodeHexEscapedString returns unescaped "\xab" style hex-escape strings +func DecodeHexEscapedString(s string) string { + var out []byte + + // Loop through one byte at a time for the length of a string + for i:=0;i < len(s);i++ { + + // Check to see if we are escaping a slash + if i+2 < len(s) && s[i:i+2] == `\\` { + out = append(out, s[i]) + i += 1 + continue + } + + // Check to see if we can have at least 4 bytes to work with, if so check to see if the first two are "\x" + if i+4 < len(s) && s[i:i+2] == `\x` { + r, err := hex.DecodeString(s[i+2:i+4]) + if err == nil { + // No errors, so append the byte, and skip ahead 4 bytes. + out = append(out, r[0]) + i += 3 // the fourth one is added on the loop (i++) + continue + } + } + + // otherwise append the byte to the string and keep moving + out = append(out, s[i]) + } + + return string(out) +} \ No newline at end of file diff --git a/nss/nss_test.go b/nss/nss_test.go new file mode 100644 index 0000000..7085376 --- /dev/null +++ b/nss/nss_test.go @@ -0,0 +1,344 @@ +package nss + +import ( + "crypto" + _ "crypto/md5" + "testing" + // "crypto/sha1" + "bufio" + _ "crypto/sha256" + "encoding/hex" + "strings" +) + +type fgrpnt struct { + hashFn crypto.Hash + data string + expect string +} + +type hexEsc struct { + hexEncde string + expect string +} + +type nssObj struct { + key string + attr string + value string +} + +func TestParseCkaClassObject(t *testing.T) { + var objects []*object // the return data + l := 1 // line + s := bufio.NewScanner(strings.NewReader(nssObject)) // scanner + + for s.Scan() { + l += 1 + line := s.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + words := strings.Fields(line) + + if words[0] == "CKA_CLASS" { + + o := new(object) + o.startingLine = l + o.attrs = map[string]attribute{ + words[0]: attribute{ + words[1], + []byte(strings.Join(words[2:], " ")), + }, + } + + l, o = parseCkaClassObject(s, l, o) + objects = append(objects, o) + + if l != 36 { + t.Error("Invalid line count for the multi-line octal got:", l, "expected: 36") + } + + testTable := []nssObj{ + nssObj{"CKA_TOKEN", "CK_BBOOL", "CK_TRUE"}, + nssObj{"CKA_PRIVATE", "CK_BBOOL", "CK_FALSE"}, + nssObj{"CKA_MODIFIABLE", "CK_BBOOL", "CK_FALSE"}, + nssObj{"CKA_LABEL", "UTF8", "\"WoSign China\""}, + nssObj{"CKA_TRUST_SERVER_AUTH", "CK_TRUST", "CKT_NSS_TRUSTED_DELEGATOR"}, + nssObj{"CKA_TRUST_EMAIL_PROTECTION", "CK_TRUST", "CKT_NSS_TRUSTED_DELEGATOR"}, + nssObj{"CKA_TRUST_CODE_SIGNING", "CK_TRUST", "CKT_NSS_TRUSTED_DELEGATOR"}, + nssObj{"CKA_TRUST_STEP_UP_APPROVED", "CK_BBOOL", "CK_FALSE"}, + } + + for _, x := range testTable { + if aa, ok := o.attrs[x.key]; !ok { + t.Error("Value not found in attribute list expected:", x.key) + } else { + if aa.attrType != x.attr { + t.Error("Invalid attribute type for key got:", aa.attrType, "expected:", x.attr) + } + + if string(aa.value) != x.value { + t.Error("Invalid value type for key got:", string(aa.value), "expected:", x.value) + } + } + } + + testTable2 := []nssObj{ + nssObj{"CKA_CERT_MD5_HASH", "MULTILINE_OCTAL", "78835b521676c4243b8378e8acda9a93"}, + nssObj{"CKA_SERIAL_NUMBER", "MULTILINE_OCTAL", "021050706bcdd813fc1b4e3b3372d211488d"}, + nssObj{"CKA_ISSUER", "MULTILINE_OCTAL", "3046310b300906035504061302434e311a3018060355040a1311576f5369676e204341204c696d69746564311b301906035504030c12434120e6b283e9809ae6a0b9e8af81e4b9a6"}, + } + + for _, x := range testTable2 { + if aa, ok := o.attrs[x.key]; !ok { + t.Error("Value not found in attribute list expected:", x.key) + } else { + if aa.attrType != x.attr { + t.Error("Invalid attribute type for key got:", aa.attrType, "expected:", x.attr) + } + if hex.EncodeToString(aa.value) != x.value { + t.Error("Invalid value type for key got:", hex.EncodeToString(aa.value), "expected:", x.value) + } + } + } + + } + } +} + +func TestParseMultiLineOctal(t *testing.T) { + var b []byte // the return data + l := 1 // line + s := bufio.NewScanner(strings.NewReader(nssMultiLineOctal)) // scanner + + l, b = parseMultiLineOctal(s, l) + if l != 7 { + t.Error("Invalid line count for the multi-line octal got:", l, "expected: 7") + } + + if hex.EncodeToString(b) != nssMultiLineOctalExpect { + t.Error("Invalid octal conversion got:", hex.EncodeToString(b), "expected:", nssMultiLineOctalExpect) + } +} + +func TestParseLicenseBlock(t *testing.T) { + var license, cvsId string // The return data + l := 1 // line + s := bufio.NewScanner(strings.NewReader(nssLicense)) // scanner + + for s.Scan() { + l += 1 + line := s.Text() + + if strings.Contains(line, "This Source Code") { + l, license, cvsId = parseLicenseBlock(s, l) + + if l != 6 { // it adds the extra line + t.Error("Invalid line count for the multi-line octal got:", l, "expected: 6") + } + + if strings.TrimSpace(license) != strings.TrimSpace(nssLicenseExpect) { + t.Error("Invalid parsing for the license got:", license, "expected:", nssLicenseExpect) + } + + if cvsId != "" { + t.Error("Invalid parsing for the license cvsId got:", cvsId, "expected: ") + } + + break + } + } +} + +func TestParseLicenseBlock2(t *testing.T) { + var license, cvsId string // The return data + l := 1 // line + s := bufio.NewScanner(strings.NewReader(nssLicense2)) // scanner + + for s.Scan() { + l += 1 + line := s.Text() + + if strings.Contains(line, "This Source Code") { + l, license, cvsId = parseLicenseBlock(s, l) + + if l != 7 { // it adds the extra line + t.Error("Invalid line count for the multi-line octal got:", l, "expected: 7") + } + + if strings.TrimSpace(license) != strings.TrimSpace(nssLicenseExpect) { + t.Error("Invalid parsing for the license got:", license, "expected:", nssLicenseExpect) + } + + if cvsId != cvsIdExpect { + t.Error("Invalid parsing for the license cvsId got:", cvsId, "expected: ", cvsIdExpect) + } + + break + } + } +} + +func TestFingerprint(t *testing.T) { + testTable := []fgrpnt{ + fgrpnt{ + crypto.MD5, + "The quick brown fox jumped over the lazy dog", + "08:a0:08:a0:1d:49:8c:40:4b:0c:30:85:2b:39:d3:b8", + }, + fgrpnt{ + crypto.SHA1, + "The quick brown fox jumped over the lazy dog", + "f6:51:36:40:f3:04:5e:97:68:b2:39:78:56:25:ca:a6:a2:58:88:42", + }, + fgrpnt{ + crypto.SHA256, + "The quick brown fox jumped over the lazy dog", + "7d:38:b5:cd:25:a2:ba:f8:5a:d3:bb:5b:93:11:38:3e:67:1a:8a:14:2e:b3:02:b3:24:d4:a5:fb:a8:74:8c:69", + }, + } + + for _, x := range testTable { + y := Fingerprint(x.hashFn, []byte(x.data)) + if y != x.expect { + t.Error("Invalid Fingerprint got:", y, "expected:", x.expect) + } + } +} + +func TestDecodeHexEscapedString(t *testing.T) { + testTable := []hexEsc{ + hexEsc{ + "AC Ra\xC3\xADz Certic\xC3\xA1mara S.A.", + "AC Raíz Certicámara S.A.", + }, + hexEsc{ + "T\xc3\x9c\x42\xC4\xB0TAK UEKAE K\xC3\xB6k Sertifika Hizmet Sa\xC4\x9Flay\xc4\xb1\x63\xc4\xb1s\xc4\xb1 - S\xC3\xBCr\xC3\xBCm 3", + "TÜBİTAK UEKAE Kök Sertifika Hizmet Sağlayıcısı - Sürüm 3", + }, + hexEsc{ + `Hizmet Sa\xC4\x9Flay \\xBC`, + `Hizmet Sağlay \xBC`, + }, + hexEsc{ + `\xzk\xbg`, + `\xzk\xbg`, + }, + } + + for _, x := range testTable { + y := DecodeHexEscapedString(x.hexEncde) + if y != x.expect { + t.Error("Invalid HexEscape Decode got:", y, "expected:", x.expect) + } + } +} + +func TestParseIgnoreList(t *testing.T) { + f := strings.NewReader(nssIgnoreList) // scanner + p := ParseIgnoreList(f) + + testTable := map[string]string{ + "DigiCert Trusted Root G4": "", + "E-Guven Kok_Elektronik Sertifika_Hizmet Saglayicisi": "", + "E-Tugra Certification Authority": "", + "EBG_Elektronik Sertifika_Hizmet Sağlayıcısı": "Optional Comment: This has UTF-8 characters", + "EE_Certification Centre Root CA": "", + "Entrust.net_Premium 2048 Secure Server CA": "", + } + + for k, v := range testTable { + if _, ok := p[k]; !ok { + t.Error("Could not find the parsed value for:", k) + } else { + if p[k] != v { + t.Error("Parsing was incorrect got:", p[k], "expected:", v) + } + } + } +} + +func TestField(t *testing.T) { + // I'm not sure how to create a cert that has multiple organizations or common names to test this out with. +} + +var ( + // An ignore list is just a list of label names, with an optional comment after a # + nssIgnoreList = `DigiCert Trusted Root G4 +E-Guven Kok_Elektronik Sertifika_Hizmet Saglayicisi +E-Tugra Certification Authority +EBG_Elektronik Sertifika_Hizmet Sağlayıcısı # Optional Comment: This has UTF-8 characters +EE_Certification Centre Root CA +Entrust.net_Premium 2048 Secure Server CA` + + nssObject = `# Trust for "WoSign China" +# Issuer: CN=CA ...............,O=WoSign CA Limited,C=CN +# Serial Number:50:70:6b:cd:d8:13:fc:1b:4e:3b:33:72:d2:11:48:8d +# Subject: CN=CA ...............,O=WoSign CA Limited,C=CN +# Not Valid Before: Sat Aug 08 01:00:01 2009 +# Not Valid After : Mon Aug 08 01:00:01 2039 +# Fingerprint (SHA-256): D6:F0:34:BD:94:AA:23:3F:02:97:EC:A4:24:5B:28:39:73:E4:47:AA:59:0F:31:0C:77:F4:8F:DF:83:11:22:54 +# Fingerprint (SHA1): 16:32:47:8D:89:F9:21:3A:92:00:85:63:F5:A4:A7:D3:12:40:8A:D6 +CKA_CLASS CK_OBJECT_CLASS CKO_NSS_TRUST +CKA_TOKEN CK_BBOOL CK_TRUE +CKA_PRIVATE CK_BBOOL CK_FALSE +CKA_MODIFIABLE CK_BBOOL CK_FALSE +CKA_LABEL UTF8 "WoSign China" +CKA_CERT_SHA1_HASH MULTILINE_OCTAL +\026\062\107\215\211\371\041\072\222\000\205\143\365\244\247\323 +\022\100\212\326 +END +CKA_CERT_MD5_HASH MULTILINE_OCTAL +\170\203\133\122\026\166\304\044\073\203\170\350\254\332\232\223 +END +CKA_ISSUER MULTILINE_OCTAL +\060\106\061\013\060\011\006\003\125\004\006\023\002\103\116\061 +\032\060\030\006\003\125\004\012\023\021\127\157\123\151\147\156 +\040\103\101\040\114\151\155\151\164\145\144\061\033\060\031\006 +\003\125\004\003\014\022\103\101\040\346\262\203\351\200\232\346 +\240\271\350\257\201\344\271\246 +END +CKA_SERIAL_NUMBER MULTILINE_OCTAL +\002\020\120\160\153\315\330\023\374\033\116\073\063\162\322\021 +\110\215 +END +CKA_TRUST_SERVER_AUTH CK_TRUST CKT_NSS_TRUSTED_DELEGATOR +CKA_TRUST_EMAIL_PROTECTION CK_TRUST CKT_NSS_TRUSTED_DELEGATOR +CKA_TRUST_CODE_SIGNING CK_TRUST CKT_NSS_TRUSTED_DELEGATOR +CKA_TRUST_STEP_UP_APPROVED CK_BBOOL CK_FALSE` + + nssMultiLineOctal = `\060\106\061\013\060\011\006\003\125\004\006\023\002\103\116\061 +\032\060\030\006\003\125\004\012\023\021\127\157\123\151\147\156 +\040\103\101\040\114\151\155\151\164\145\144\061\033\060\031\006 +\003\125\004\003\014\022\103\101\040\346\262\203\351\200\232\346 +\240\271\350\257\201\344\271\246 +END` + + nssMultiLineOctalExpect = "3046310b300906035504061302434e311a3018060355040a1311576f5369676e204341204c696d69746564311b301906035504030c12434120e6b283e9809ae6a0b9e8af81e4b9a6" + + nssLicense = `# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# certdata.txt +# +# This file contains the object definitions for the certs and other +# information "built into" NSS. +#` + nssLicenseExpect = `# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/.` + + nssLicense2 = `# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +CVS_ID "@(#) $RCSfile$ $Revision$ $Date$" + +` + + cvsIdExpect = `"@(#) $RCSfile$ $Revision$ $Date$"` +) \ No newline at end of file