Skip to content

Commit eb3d0e6

Browse files
mikefarahmikefarah
committed
Fixing handling of CRLF #2352 
1 parent 2072808 commit eb3d0e6

File tree

3 files changed

+101
-37
lines changed

3 files changed

+101
-37
lines changed

‎pkg/yqlib/decoder_yaml.go‎

Lines changed: 58 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,13 @@ import (
1111
yaml "go.yaml.in/yaml/v4"
1212
)
1313

14+
var (
15+
commentLineRe = regexp.MustCompile(`^\s*#`)
16+
yamlDirectiveLineRe = regexp.MustCompile(`^\s*%YAML`)
17+
separatorLineRe = regexp.MustCompile(`^\s*---\s*$`)
18+
separatorPrefixRe = regexp.MustCompile(`^\s*---\s+`)
19+
)
20+
1421
type yamlDecoder struct {
1522
decoder yaml.Decoder
1623

@@ -33,51 +40,72 @@ func NewYamlDecoder(prefs YamlPreferences) Decoder {
3340
}
3441

3542
func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
36-
var commentLineRegEx = regexp.MustCompile(`^\s*#`)
37-
var yamlDirectiveLineRegEx = regexp.MustCompile(`^\s*%YA`)
3843
var sb strings.Builder
44+
3945
for {
40-
peekBytes, err := reader.Peek(4)
41-
if errors.Is(err, io.EOF) {
42-
// EOF are handled else where..
46+
line, err := reader.ReadString('\n')
47+
if errors.Is(err, io.EOF) && line == "" {
48+
// no more data
4349
return reader, sb.String(), nil
44-
} else if err != nil {
50+
}
51+
if err != nil && !errors.Is(err, io.EOF) {
4552
return reader, sb.String(), err
46-
} else if string(peekBytes[0]) == "\n" {
47-
_, err := reader.ReadString('\n')
48-
sb.WriteString("\n")
53+
}
54+
55+
// Determine newline style and strip it for inspection
56+
newline := ""
57+
if strings.HasSuffix(line, "\r\n") {
58+
newline = "\r\n"
59+
line = strings.TrimSuffix(line, "\r\n")
60+
} else if strings.HasSuffix(line, "\n") {
61+
newline = "\n"
62+
line = strings.TrimSuffix(line, "\n")
63+
}
64+
65+
trimmed := strings.TrimSpace(line)
66+
67+
// Document separator: exact line '---' or a '--- ' prefix followed by content
68+
if separatorLineRe.MatchString(trimmed) {
69+
sb.WriteString("$yqDocSeparator$")
70+
sb.WriteString(newline)
4971
if errors.Is(err, io.EOF) {
5072
return reader, sb.String(), nil
51-
} else if err != nil {
52-
return reader, sb.String(), err
5373
}
54-
} else if string(peekBytes) == "--- " {
55-
_, err := reader.ReadString(' ')
56-
sb.WriteString("$yqDocSeparator$\n")
57-
if errors.Is(err, io.EOF) {
58-
return reader, sb.String(), nil
59-
} else if err != nil {
60-
return reader, sb.String(), err
74+
continue
75+
}
76+
77+
// Handle lines that start with '--- ' followed by more content (e.g. '--- cat')
78+
if separatorPrefixRe.MatchString(line) {
79+
match := separatorPrefixRe.FindString(line)
80+
remainder := line[len(match):]
81+
// normalize separator newline: if original had none, default to LF
82+
sepNewline := newline
83+
if sepNewline == "" {
84+
sepNewline = "\n"
6185
}
62-
} else if string(peekBytes) == "---\n" {
63-
_, err := reader.ReadString('\n')
64-
sb.WriteString("$yqDocSeparator$\n")
65-
if errors.Is(err, io.EOF) {
86+
sb.WriteString("$yqDocSeparator$")
87+
sb.WriteString(sepNewline)
88+
// push the remainder back onto the reader and continue processing
89+
reader = bufio.NewReader(io.MultiReader(strings.NewReader(remainder), reader))
90+
if errors.Is(err, io.EOF) && remainder == "" {
6691
return reader, sb.String(), nil
67-
} else if err != nil {
68-
return reader, sb.String(), err
6992
}
70-
} else if commentLineRegEx.MatchString(string(peekBytes)) || yamlDirectiveLineRegEx.MatchString(string(peekBytes)) {
71-
line, err := reader.ReadString('\n')
93+
continue
94+
}
95+
96+
// Comments, YAML directives, and blank lines are leading content
97+
if commentLineRe.MatchString(line) || yamlDirectiveLineRe.MatchString(line) || trimmed == "" {
7298
sb.WriteString(line)
99+
sb.WriteString(newline)
73100
if errors.Is(err, io.EOF) {
74101
return reader, sb.String(), nil
75-
} else if err != nil {
76-
return reader, sb.String(), err
77102
}
78-
} else {
79-
return reader, sb.String(), nil
103+
continue
80104
}
105+
106+
// First non-leading line: push it back onto a reader and return
107+
originalLine := line + newline
108+
return io.MultiReader(strings.NewReader(originalLine), reader), sb.String(), nil
81109
}
82110
}
83111

‎pkg/yqlib/encoder_yaml.go‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"bytes"
66
"errors"
77
"io"
8-
"regexp"
98
"strings"
109

1110
"github.com/fatih/color"
@@ -37,7 +36,8 @@ func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error {
3736
func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
3837
reader := bufio.NewReader(strings.NewReader(content))
3938

40-
var commentLineRegEx = regexp.MustCompile(`^\s*#`)
39+
// reuse precompiled package-level regex
40+
// (declared in decoder_yaml.go)
4141

4242
for {
4343

@@ -46,13 +46,19 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
4646
return errReading
4747
}
4848
if strings.Contains(readline, "$yqDocSeparator$") {
49-
50-
if err := ye.PrintDocumentSeparator(writer); err != nil {
51-
return err
49+
// Preserve the original line ending (CRLF or LF)
50+
lineEnding := "\n"
51+
if strings.HasSuffix(readline, "\r\n") {
52+
lineEnding = "\r\n"
53+
}
54+
if ye.prefs.PrintDocSeparators {
55+
if err := writeString(writer, "---"+lineEnding); err != nil {
56+
return err
57+
}
5258
}
5359

5460
} else {
55-
if len(readline) > 0 && readline != "\n" && readline[0] != '%' && !commentLineRegEx.MatchString(readline) {
61+
if len(readline) > 0 && readline != "\n" && readline[0] != '%' && !commentLineRe.MatchString(readline) {
5662
readline = "# " + readline
5763
}
5864
if ye.prefs.ColorsEnabled && strings.TrimSpace(readline) != "" {
@@ -79,10 +85,15 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
7985

8086
func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
8187
log.Debug("encoderYaml - going to print %v", NodeToString(node))
88+
// Detect line ending style from LeadingContent
89+
lineEnding := "\n"
90+
if strings.Contains(node.LeadingContent, "\r\n") {
91+
lineEnding = "\r\n"
92+
}
8293
if node.Kind == ScalarNode && ye.prefs.UnwrapScalar {
8394
valueToPrint := node.Value
8495
if node.LeadingContent == "" || valueToPrint != "" {
85-
valueToPrint = valueToPrint + "\n"
96+
valueToPrint = valueToPrint + lineEnding
8697
}
8798
return writeString(writer, valueToPrint)
8899
}

‎pkg/yqlib/yaml_test.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,31 @@ var yamlFormatScenarios = []formatScenario{
1313
input: "--- cat",
1414
expected: "---\ncat\n",
1515
},
16+
{
17+
description: "CRLF doc separator",
18+
skipDoc: true,
19+
input: "---\r\ncat\r\n",
20+
expected: "---\r\ncat\r\n",
21+
},
22+
{
23+
description: "yaml directive preserved (LF)",
24+
skipDoc: true,
25+
input: "%YAML 1.1\n---\ncat\n",
26+
expected: "%YAML 1.1\n---\ncat\n",
27+
},
28+
{
29+
description: "yaml directive preserved (CRLF)",
30+
skipDoc: true,
31+
input: "%YAML 1.1\r\n---\r\ncat\r\n",
32+
expected: "%YAML 1.1\r\n---\r\ncat\r\n",
33+
},
34+
{
35+
description: "comment only no trailing newline",
36+
skipDoc: true,
37+
input: "# hello",
38+
expected: "# hello\n",
39+
},
40+
1641
{
1742
description: "scalar with doc separator",
1843
skipDoc: true,

0 commit comments

Comments
 (0)