1) EDI doc 2) add ignore_crlf to edi schema 3) add default to edi schema 4) make edi err msg more useful 5) fix cli.sh (#129)

1) EDI doc 2) add `ignore_crlf` to edi schema 3) add `default` to edi schema 4) make edi err msg more useful 5) fix cli.sh

Author: jf-tech

cli.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 CUR_DIR=$(pwd)
-SCRIPT_DIR=$(dirname "$0")
-cd $SCRIPT_DIR && \
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+cd "$SCRIPT_DIR" && \
   go build \
-    -o $CUR_DIR/op \
+    -o "$CUR_DIR/op" \
     -ldflags "-X main.gitCommit=$(git rev-parse HEAD) -X main.buildEpochSec=$(date +%s)" \
-    $SCRIPT_DIR/cli/op.go
-cd $CUR_DIR && ./op "$@"
+    "$SCRIPT_DIR/cli/op.go"
+cd "$CUR_DIR" && ./op "$@"

cli/cmd/transformCmd.go

@@ -22,7 +22,11 @@ var (
 		Short: "Transforms input to desired output based on a schema.",
 		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			return doTransform()
+			if err := doTransform(); err != nil {
+				fmt.Println() // to sure cobra cli always write out "Error: ..." on a new line.
+				return err
+			}
+			return nil
 		},
 	}
 	schema string

doc/edi_in_depth.md

@@ -1,4 +1,4 @@
 {
 	"Records": null,
-	"FinalErr": "input 'test' between character [1,6]: missing segment name"
+	"FinalErr": "input 'test' at segment no.1 (char[1,6]): missing segment name"
 }

doc/resources/edi_comp_delim.png

@@ -1,4 +1,4 @@
 {
 	"Records": null,
-	"FinalErr": "input 'test' between character [1,7]: unable to find element 'e2' on segment ''"
+	"FinalErr": "input 'test' at segment no.1 (char[1,7]): unable to find element 'e2' on segment ''"
 }

doc/resources/edi_seg_structure.png

@@ -2,5 +2,5 @@
 	"Records": [
 		"{'e1':'0','e2':'1','e3':'2'}"
 	],
-	"FinalErr": "input 'test' between character [11,11]: segment 'IEA' needs min occur 1, but only got 0"
+	"FinalErr": "input 'test' at segment no.3 (char[11,11]): segment 'IEA' needs min occur 1, but only got 0"
 }

extensions/omniv21/fileformat/edi/.snapshots/TestRead-missing_raw_seg_name,_failure

@@ -1,4 +1,4 @@
 {
 	"Records": null,
-	"FinalErr": "input 'test' between character [7,7]: segment 'ISA' needs min occur 1, but only got 0"
+	"FinalErr": "input 'test' at segment no.1 (char[7,7]): segment 'ISA' needs min occur 1, but only got 0"
 }

extensions/omniv21/fileformat/edi/.snapshots/TestRead-raw_seg_processing_wrong,_failure

@@ -1,7 +1,7 @@
 {
 	"Records": [
 		"{'e1':'0','e2':'1','e3':'2'}",
-		"{'e1':'3','e2':'','e3':''}"
+		"{'e1':'3','e2':'','e3':'x'}"
 	],
 	"FinalErr": "EOF"
 }

extensions/omniv21/fileformat/edi/.snapshots/TestRead-seg_min_not_satisfied_before_EOF,_failure

@@ -5,5 +5,6 @@ type fileDecl struct {
 	ElemDelim   string     `json:"element_delimiter,omitempty"`
 	CompDelim   *string    `json:"component_delimiter,omitempty"`
 	ReleaseChar *string    `json:"release_character,omitempty"`
+	IgnoreCRLF  bool       `json:"ignore_crlf,omitempty"`
 	SegDecls    []*segDecl `json:"segment_declarations,omitempty"`
 }

extensions/omniv21/fileformat/edi/.snapshots/TestRead-seg_min_not_satisfied_before_next_seg_appearance,_failure

@@ -101,6 +101,7 @@ type ediReader struct {
 	target             *idr.Node
 	targetXPath        *xpath.Expr
 	runeBegin, runeEnd int
+	segCount           int
 	unprocessedRawSeg  rawSeg
 }
 
@@ -169,6 +170,7 @@ func runeCountAndHasOnlyCRLF(b []byte) (int, bool) {
 
 var (
 	crBytes = []byte("\r")
+	lfBytes = []byte("\n")
 )
 
 func (r *ediReader) getUnprocessedRawSeg() (rawSeg, error) {
@@ -189,6 +191,7 @@ func (r *ediReader) getUnprocessedRawSeg() (rawSeg, error) {
 		token = b
 		break
 	}
+	r.segCount++
 	// We are here because:
 	// 1. we find next token (i.e. segment), great, let's process it, OR
 	// 2. r.scanner.Scan() returns false and it's EOF (note scanner never returns EOF, it just returns false
@@ -259,14 +262,16 @@ func (r *ediReader) rawSegToNode(segDecl *segDecl) (*idr.Node, error) {
 				break
 			}
 		}
-		if rawElemIndex < len(rawElems) || elemDecl.EmptyIfMissing {
+		if rawElemIndex < len(rawElems) || elemDecl.EmptyIfMissing || elemDecl.Default != nil {
 			elemN := idr.CreateNode(idr.ElementNode, elemDecl.Name)
 			idr.AddChild(n, elemN)
 			data := ""
 			if rawElemIndex < len(rawElems) {
 				data = string(strs.ByteUnescape(rawElems[rawElemIndex].data, r.releaseChar.b, true))
 				rawElems[rawElemIndex].dateUnescaped = true
 				rawElemIndex++
+			} else if elemDecl.Default != nil {
+				data = *elemDecl.Default
 			}
 			elemV := idr.CreateNode(idr.TextNode, data)
 			idr.AddChild(elemN, elemV)
@@ -325,7 +330,7 @@ func (r *ediReader) segNext() error {
 		// 'end' to 'begin' to make the error msg less confusing.
 		r.runeBegin = r.runeEnd
 		return ErrInvalidEDI(r.fmtErrStr("segment '%s' needs min occur %d, but only got %d",
-			cur.segDecl.Name, cur.segDecl.minOccurs(), cur.occurred))
+			strs.FirstNonBlank(cur.segDecl.fqdn, cur.segDecl.Name), cur.segDecl.minOccurs(), cur.occurred))
 	}
 	if len(r.stack) <= 1 {
 		return nil
@@ -420,8 +425,8 @@ func (r *ediReader) FmtErr(format string, args ...interface{}) error {
 }
 
 func (r *ediReader) fmtErrStr(format string, args ...interface{}) string {
-	return fmt.Sprintf("input '%s' between character [%d,%d]: %s",
-		r.inputName, r.runeBegin, r.runeEnd, fmt.Sprintf(format, args...))
+	return fmt.Sprintf("input '%s' at segment no.%d (char[%d,%d]): %s",
+		r.inputName, r.segCount, r.runeBegin, r.runeEnd, fmt.Sprintf(format, args...))
 }
 
 const (
@@ -442,6 +447,10 @@ func NewReader(inputName string, r io.Reader, decl *fileDecl, targetXPath string
 	elemDelim := newStrPtrByte(&decl.ElemDelim)
 	compDelim := newStrPtrByte(decl.CompDelim)
 	releaseChar := newStrPtrByte(decl.ReleaseChar)
+	if decl.IgnoreCRLF {
+		r = ios.NewBytesReplacingReader(r, crBytes, nil)
+		r = ios.NewBytesReplacingReader(r, lfBytes, nil)
+	}
 	scanner := ios.NewScannerByDelim3(r, segDelim.b, releaseChar.b, scannerFlags, make([]byte, ReaderBufSize))
 	targetXPathExpr, err := func() (*xpath.Expr, error) {
 		if targetXPath == "" || targetXPath == "." {
@@ -463,6 +472,7 @@ func NewReader(inputName string, r io.Reader, decl *fileDecl, targetXPath string
 		targetXPath:       targetXPathExpr,
 		runeBegin:         1,
 		runeEnd:           1,
+		segCount:          0,
 		unprocessedRawSeg: newRawSeg(),
 	}
 	reader.growStack(stackEntry{

extensions/omniv21/fileformat/edi/.snapshots/TestRead-single_seg_decl,_multiple_seg_instances,_success

@@ -189,7 +189,7 @@ func TestGetUnprocessedRawSeg(t *testing.T) {
 				ElemDelim: ":",
 			},
 			expected: []result{
-				{rawSeg: rawSeg{}, err: `input 'test' between character [1,1]: cannot read segment, err: read failure`},
+				{rawSeg: rawSeg{}, err: `input 'test' at segment no.1 (char[1,1]): cannot read segment, err: read failure`},
 			},
 		},
 		{
@@ -244,7 +244,7 @@ func TestGetUnprocessedRawSeg(t *testing.T) {
 				ElemDelim: "*",
 			},
 			expected: []result{
-				{rawSeg: rawSeg{}, err: `input 'test' between character [1,2]: missing segment name`},
+				{rawSeg: rawSeg{}, err: `input 'test' at segment no.1 (char[1,2]): missing segment name`},
 			},
 		},
 		{
@@ -344,7 +344,7 @@ func TestRawSegToNode(t *testing.T) {
 				},
 				fqdn: "ISA",
 			},
-			err:      `input 'test' between character [10,20]: unable to find element 'e3' on segment 'ISA'`,
+			err:      `input 'test' at segment no.0 (char[10,20]): unable to find element 'e3' on segment 'ISA'`,
 			expected: "",
 		},
 		{
@@ -516,7 +516,7 @@ func TestSegDoneSegNext(t *testing.T) {
 			target:      nil,
 			callSegDone: false,
 			panicStr:    "",
-			err:         `input 'test' between character [20,20]: segment 'C' needs min occur 1, but only got 0`,
+			err:         `input 'test' at segment no.0 (char[20,20]): segment 'C' needs min occur 1, but only got 0`,
 		},
 		{
 			name: "root-A-C, C segDone, C over max, A becomes target, but r.target not nil",
@@ -632,7 +632,7 @@ func TestRead(t *testing.T) {
 							"elements": [
 								{ "name": "e1", "index": 1 },
 								{ "name": "e2", "index": 2, "empty_if_missing": true },
-								{ "name": "e3", "index": 3, "empty_if_missing": true }
+								{ "name": "e3", "index": 3, "default": "x" }
 							]
 						}
 					]
@@ -671,11 +671,12 @@ func TestRead(t *testing.T) {
 		},
 		{
 			name:  "2 seg groups, filtered target, success",
-			input: "ISA*0*1*2\nISA*3*4*5\nISA*6*7*8\nIEA*6\n",
+			input: "ISA*0*1*2|\nISA*3*4*5|\nISA*6*7*8|\r\nIEA*6|\n",
 			declJSON: `
 				{
-					"segment_delimiter": "\n",
+					"segment_delimiter": "|",
 					"element_delimiter": "*",
+					"ignore_crlf": true,
 					"segment_declarations": [
 						{
 							"name": "isa_group",

extensions/omniv21/fileformat/edi/decl.go

@@ -30,10 +30,11 @@ const (
 )
 
 type elem struct {
-	Name           string `json:"name,omitempty"`
-	Index          int    `json:"index,omitempty"`
-	CompIndex      *int   `json:"component_index,omitempty"`
-	EmptyIfMissing bool   `json:"empty_if_missing,omitempty"`
+	Name           string  `json:"name,omitempty"`
+	Index          int     `json:"index,omitempty"`
+	CompIndex      *int    `json:"component_index,omitempty"`
+	EmptyIfMissing bool    `json:"empty_if_missing,omitempty"` // Deprecated, use Default
+	Default        *string `json:"default,omitempty"`
 }
 
 func (e elem) compIndex() int {