diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go index 612fb75f4..a1a1b5de6 100644 --- a/parser/pageparser/pagelexer.go +++ b/parser/pageparser/pagelexer.go @@ -248,7 +248,7 @@ func (l *pageLexer) consumeCRLF() bool { func (l *pageLexer) consumeToSpace() { for { r := l.next() - if r == eof || unicode.IsSpace(r) { + if r == eof || isASCIISpace(r) { l.backup() return } @@ -258,7 +258,7 @@ func (l *pageLexer) consumeToSpace() { func (l *pageLexer) consumeSpace() { for { r := l.next() - if r == eof || !unicode.IsSpace(r) { + if r == eof || !isASCIISpace(r) { l.backup() return } @@ -497,7 +497,7 @@ func minIndex(indices ...int) int { func indexNonWhiteSpace(s []byte, in rune) int { idx := bytes.IndexFunc(s, func(r rune) bool { - return !unicode.IsSpace(r) + return !isASCIISpace(r) }) if idx == -1 { @@ -511,8 +511,12 @@ func indexNonWhiteSpace(s []byte, in rune) int { return -1 } -func isSpace(r rune) bool { - return r == ' ' || r == '\t' +func isASCIISpace(r rune) bool { + switch r { + case '\t', '\n', '\v', '\f', '\r', ' ': + return true + } + return false } func isAlphaNumericOrHyphen(r rune) bool { diff --git a/parser/pageparser/pagelexer_intro.go b/parser/pageparser/pagelexer_intro.go index a68a9e03a..03bd4c887 100644 --- a/parser/pageparser/pagelexer_intro.go +++ b/parser/pageparser/pagelexer_intro.go @@ -32,7 +32,7 @@ LOOP: return lexFrontMatterOrgMode case r == byteOrderMark: l.emit(TypeIgnore) - case !isSpace(r) && !isEndOfLine(r): + case !isASCIISpace(r) && !isEndOfLine(r): break LOOP } } diff --git a/parser/pageparser/pagelexer_shortcode.go b/parser/pageparser/pagelexer_shortcode.go index eb1b0bf73..15e9ead80 100644 --- a/parser/pageparser/pagelexer_shortcode.go +++ b/parser/pageparser/pagelexer_shortcode.go @@ -290,7 +290,7 @@ func lexEndOfShortcode(l *pageLexer) stateFunc { return lexShortcodeRightDelim } switch r := l.next(); { - case isSpace(r): + case isASCIISpace(r): l.ignore() default: return l.errorf("unclosed shortcode") @@ -307,7 +307,7 @@ func lexInsideShortcode(l *pageLexer) stateFunc { case r == eof: // eol is allowed inside shortcodes; this may go to end of document before it fails return l.errorf("unclosed shortcode action") - case isSpace(r), isEndOfLine(r): + case isASCIISpace(r): l.ignore() case r == '=': l.consumeSpace() diff --git a/parser/pageparser/pageparser_test.go b/parser/pageparser/pageparser_test.go index 561ad7ea2..3407a3704 100644 --- a/parser/pageparser/pageparser_test.go +++ b/parser/pageparser/pageparser_test.go @@ -138,3 +138,13 @@ func TestSummaryDividerStartingFromMain(t *testing.T) { c.Assert(items, qt.HasLen, 4) c.Assert(items[1].Type, qt.Equals, TypeLeadSummaryDivider) } + +func TestIdeographicAfterSummaryDivider(t *testing.T) { + c := qt.New(t) + + input := []byte(`aaa  bbb`) + items, err := collectStringMain(string(input)) + c.Assert(err, qt.IsNil) + c.Assert(items, qt.HasLen, 4) + c.Assert(items[2].ValStr(input), qt.Equals, "\u3000bbb") +}