Merge pull request #127 from Shopify/collapse-newlines

Author: burke

dev.yml

@@ -5,7 +5,7 @@ up:
     - gnu-tar
   - ruby: 3.0.3
   - go:
-      version: 1.18
+      version: '1.20'
   - bundler
 
 commands:

ejson.go

@@ -43,6 +43,11 @@ func Encrypt(in io.Reader, out io.Writer) (int, error) {
 		return -1, err
 	}
 
+	data, err = json.CollapseMultilineStringLiterals(data)
+	if err != nil {
+		return -1, err
+	}
+
 	pubkey, err := json.ExtractPublicKey(data)
 	if err != nil {
 		return -1, err

ejson_test.go

@@ -69,7 +69,7 @@ func TestEncryptFileInPlace(t *testing.T) {
 			_, err := EncryptFileInPlace(tempFileName)
 			Convey("should fail", func() {
 				So(err, ShouldNotBeNil)
-				So(err.Error(), ShouldContainSubstring, "invalid character")
+				So(err.Error(), ShouldContainSubstring, "invalid json")
 			})
 		})
 
@@ -95,6 +95,19 @@ func TestEncryptFileInPlace(t *testing.T) {
 			})
 		})
 
+		Convey("called with a valid keypair and multiline string", func() {
+			setData(tempFileName, []byte(`{"_public_key": "`+validPubKey+"\", \"a\": \"b\nc\"\n}"))
+
+			_, err := EncryptFileInPlace(tempFileName)
+			output, err := ioutil.ReadFile(tempFileName)
+			So(err, ShouldBeNil)
+			Convey("should encrypt the file", func() {
+				So(err, ShouldBeNil)
+				match := regexp.MustCompile(`{"_public_key": "8d8.*", "a": "EJ.*"`+"\n}")
+				So(match.Find(output), ShouldNotBeNil)
+			})
+		})
+
 	})
 }
 

json/walker.go

@@ -25,16 +25,70 @@ import (
 // underscore-to-disable-encryption syntax does not propagate down the hierarchy
 // to children.
 // That is:
-//   * In {"_a": "b"}, Action will not be run at all.
-//   * In {"a": "b"}, Action will be run with "b", and the return value will
-//      replace "b".
-//   * In {"k": {"a": ["b"]}, Action will run on "b".
-//   * In {"_k": {"a": ["b"]}, Action run on "b".
-//   * In {"k": {"_a": ["b"]}, Action will not run.
+//   - In {"_a": "b"}, Action will not be run at all.
+//   - In {"a": "b"}, Action will be run with "b", and the return value will
+//     replace "b".
+//   - In {"k": {"a": ["b"]}, Action will run on "b".
+//   - In {"_k": {"a": ["b"]}, Action run on "b".
+//   - In {"k": {"_a": ["b"]}, Action will not run.
 type Walker struct {
 	Action func([]byte) ([]byte, error)
 }
 
+// It's common to want to paste multiline secrets into an EJSON file, and JSON
+// doesn't handle multiline literals, so we cheat here. Our first pass over the
+// file is to replace embedded newlines in string literals with escaped newlines.
+func CollapseMultilineStringLiterals(data []byte) ([]byte, error) {
+	var (
+		inString bool
+		esc      bool
+		scanner  json.Scanner
+		buf      = make([]byte, 0, len(data))
+	)
+
+	scanner.Reset()
+	for _, c := range data {
+		if inString && c == '\n' {
+			buf = append(buf, []byte{'\\', 'n'}...)
+			continue
+    } else if inString && c == '\r' {
+			buf = append(buf, []byte{'\\', 'r'}...)
+			continue
+    }
+		buf = append(buf, c)
+		switch v := scanner.Step(&scanner, int(c)); v {
+		case json.ScanContinue:
+			switch c {
+			case '\\':
+				esc = !esc
+			case '"':
+				if esc {
+					esc = false
+				} else {
+					inString = false
+				}
+			default:
+				esc = false
+			}
+		case json.ScanBeginLiteral:
+			esc = false
+			inString = (c == '"')
+		case json.ScanError:
+			return nil, fmt.Errorf("invalid json")
+		case json.ScanEnd:
+			return buf, nil
+		default:
+			inString = false
+			esc = false
+		}
+	}
+	if scanner.EOF() == json.ScanError {
+		// Unexpected EOF => malformed JSON
+		return nil, fmt.Errorf("invalid json")
+	}
+	return buf, nil
+}
+
 // Walk walks an entire JSON structure, running the ejsonWalker.Action on each
 // actionable node. A node is actionable if it's a string *value*, and its
 // referencing key doesn't begin with an underscore. For each actionable node,

json/walker_test.go

@@ -12,21 +12,29 @@ func TestWalker(t *testing.T) {
 	}
 
 	Convey("Walker passes the provided test-cases", t, func() {
-		for _, tc := range testCases {
+		for _, tc := range walkTestCases {
 			walker := Walker{Action: action}
 			act, err := walker.Walk([]byte(tc.in))
 			So(err, ShouldBeNil)
 			So(string(act), ShouldEqual, tc.out)
 		}
 	})
+
+	Convey("CollapseMultilineStringLiterals passes the provided test-cases", t, func() {
+		for _, tc := range collapseTestCases {
+			act, err := CollapseMultilineStringLiterals([]byte(tc.in))
+			So(err, ShouldBeNil)
+			So(string(act), ShouldEqual, tc.out)
+		}
+	})
 }
 
 type testCase struct {
 	in, out string
 }
 
 // "E" means encrypted.
-var testCases = []testCase{
+var walkTestCases = []testCase{
 	{`{"a": "b"}`, `{"a": "E"}`},                     // encryption
 	{`{"a" : "b"}`, `{"a" : "E"}`},                   // weird spacing
 	{` {  "a"  :"b" } `, ` {  "a"  :"E" } `},         // trailing spaces
@@ -41,3 +49,10 @@ var testCases = []testCase{
 	{`{"a": {"_b": "c"}}`, `{"a": {"_b": "c"}}`},     // nested comment
 	{`{"_a": {"b": "c"}}`, `{"_a": {"b": "E"}}`},     // comments don't inherit
 }
+
+var collapseTestCases = []testCase{
+	{
+		"{\"a\": \"b\r\nc\nd\"\r\n}", 
+		"{\"a\": \"b\\r\\nc\\nd\"\r\n}",
+	},
+}

man/man5/ejson.5.ronn

@@ -50,6 +50,10 @@ example, the password in this excerpt `will` be encrypted.
 }
 ```
 
+For convenience, JSON strings to be encrypted are allowed to contain embedded
+newline characters. These will be translated to escaped newlines. This is often
+useful when encrypting multi-line secrets such as PEM-encoded certificates.
+
 ## SECRET SCHEMA
 
 When a value is encrypted, it will be replaced by a relatively long string of