From 70a9068aa67c67a9eb2ab5fe062faf5a99e6b650 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?=
 <bjorn.erik.pedersen@gmail.com>
Date: Tue, 16 Jun 2026 16:27:24 +0200
Subject: [PATCH] parser/pageparser: Preserve non-ASCII whitespace after e.g.
 summary divider

Make it insted consume just ASCII whitespace, which preserves e.g. ideographic space (U+3000) after the summary divider, which is important for e.g. Chinese and Japanese content, and possibly other Unicode whitespace characters with meaning.

Doing this is possibly breaking, but not likely, and obviously the correct thing to do.
---
 parser/pageparser/pagelexer.go           | 14 +++++++++-----
 parser/pageparser/pagelexer_intro.go     |  2 +-
 parser/pageparser/pagelexer_shortcode.go |  4 ++--
 parser/pageparser/pageparser_test.go     | 10 ++++++++++
 4 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go
index 612fb75f4..a1a1b5de6 100644
--- a/parser/pageparser/pagelexer.go
+++ b/parser/pageparser/pagelexer.go
@@ -248,7 +248,7 @@ func (l *pageLexer) consumeCRLF() bool {
 func (l *pageLexer) consumeToSpace() {
 	for {
 		r := l.next()
-		if r == eof || unicode.IsSpace(r) {
+		if r == eof || isASCIISpace(r) {
 			l.backup()
 			return
 		}
@@ -258,7 +258,7 @@ func (l *pageLexer) consumeToSpace() {
 func (l *pageLexer) consumeSpace() {
 	for {
 		r := l.next()
-		if r == eof || !unicode.IsSpace(r) {
+		if r == eof || !isASCIISpace(r) {
 			l.backup()
 			return
 		}
@@ -497,7 +497,7 @@ func minIndex(indices ...int) int {
 
 func indexNonWhiteSpace(s []byte, in rune) int {
 	idx := bytes.IndexFunc(s, func(r rune) bool {
-		return !unicode.IsSpace(r)
+		return !isASCIISpace(r)
 	})
 
 	if idx == -1 {
@@ -511,8 +511,12 @@ func indexNonWhiteSpace(s []byte, in rune) int {
 	return -1
 }
 
-func isSpace(r rune) bool {
-	return r == ' ' || r == '\t'
+func isASCIISpace(r rune) bool {
+	switch r {
+	case '\t', '\n', '\v', '\f', '\r', ' ':
+		return true
+	}
+	return false
 }
 
 func isAlphaNumericOrHyphen(r rune) bool {
diff --git a/parser/pageparser/pagelexer_intro.go b/parser/pageparser/pagelexer_intro.go
index a68a9e03a..03bd4c887 100644
--- a/parser/pageparser/pagelexer_intro.go
+++ b/parser/pageparser/pagelexer_intro.go
@@ -32,7 +32,7 @@ LOOP:
 			return lexFrontMatterOrgMode
 		case r == byteOrderMark:
 			l.emit(TypeIgnore)
-		case !isSpace(r) && !isEndOfLine(r):
+		case !isASCIISpace(r) && !isEndOfLine(r):
 			break LOOP
 		}
 	}
diff --git a/parser/pageparser/pagelexer_shortcode.go b/parser/pageparser/pagelexer_shortcode.go
index eb1b0bf73..15e9ead80 100644
--- a/parser/pageparser/pagelexer_shortcode.go
+++ b/parser/pageparser/pagelexer_shortcode.go
@@ -290,7 +290,7 @@ func lexEndOfShortcode(l *pageLexer) stateFunc {
 		return lexShortcodeRightDelim
 	}
 	switch r := l.next(); {
-	case isSpace(r):
+	case isASCIISpace(r):
 		l.ignore()
 	default:
 		return l.errorf("unclosed shortcode")
@@ -307,7 +307,7 @@ func lexInsideShortcode(l *pageLexer) stateFunc {
 	case r == eof:
 		// eol is allowed inside shortcodes; this may go to end of document before it fails
 		return l.errorf("unclosed shortcode action")
-	case isSpace(r), isEndOfLine(r):
+	case isASCIISpace(r):
 		l.ignore()
 	case r == '=':
 		l.consumeSpace()
diff --git a/parser/pageparser/pageparser_test.go b/parser/pageparser/pageparser_test.go
index 561ad7ea2..3407a3704 100644
--- a/parser/pageparser/pageparser_test.go
+++ b/parser/pageparser/pageparser_test.go
@@ -138,3 +138,13 @@ func TestSummaryDividerStartingFromMain(t *testing.T) {
 	c.Assert(items, qt.HasLen, 4)
 	c.Assert(items[1].Type, qt.Equals, TypeLeadSummaryDivider)
 }
+
+func TestIdeographicAfterSummaryDivider(t *testing.T) {
+	c := qt.New(t)
+
+	input := []byte(`aaa <!--more-->   　bbb`)
+	items, err := collectStringMain(string(input))
+	c.Assert(err, qt.IsNil)
+	c.Assert(items, qt.HasLen, 4)
+	c.Assert(items[2].ValStr(input), qt.Equals, "\u3000bbb")
+}