fix(markup): correct character escaping at read buffer boundaries (#40209)

When a file, README or wiki page is rendered as markup, the HTML is read
through a 32 KiB buffer to escape invisible and ambiguous characters. A
multi-byte character cut by that buffer right after a tag was shown as
broken characters, e.g. `<p>보기</p>` became `<p>���기</p>`. Such a part is
now kept for the next read. Two related cases are fixed as well:

- Invalid bytes right before a tag are marked as broken instead of
passing through raw as part of the tag.
- A BOM split across reads is kept instead of being escaped as
invisible.

To reproduce, view a Markdown file with 32734 `A` characters, a blank
line and `보기`, which puts `보` across the first 32 KiB of the rendered
HTML:
[split-rune-after-tag.md](https://github.com/user-attachments/files/34361104/split-rune-after-tag.md)

https://github.com/go-gitea/gitea/pull/39937 replaces this reader with a
single in-memory pass. This small fix can be backported to the release
branches.

Assisted-by: Claude Code:claude-opus-5-5

----

| Before (2ed6a6f61b) | After
(2a1d09755f) |
| --- | --- |
|
![image](https://github.com/user-attachments/assets/d4f79a8a-17bd-4b7f-b853-1d1a7241dc9b)
|
![image](https://github.com/user-attachments/assets/d4b8b3dd-e28e-44dd-99de-d250ee4c282c)
|

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/charset/escape_stream.go

@@ -55,11 +55,9 @@ func escapeStream(locale translation.Locale, in io.Reader, out io.Writer, opts .
 		es.allowed = opts[0].Allowed
 	}
 
-	readCount := 0
-	lastIsTag := false
+	isContentBegin := true
 	for {
 		parts, partInTag, err := es.readRunes()
-		readCount++
 		if err == io.EOF {
 			return es.escaped, nil
 		} else if err != nil {
@@ -67,17 +65,14 @@ func escapeStream(locale translation.Locale, in io.Reader, out io.Writer, opts .
 		}
 		for i, part := range parts {
 			if partInTag[i] {
-				lastIsTag = true
+				isContentBegin = true
 				es.trackHtmlTag(part)
 				if _, err := out.Write(part); err != nil {
 					return nil, err
 				}
 			} else {
-				// if last part is tag, then this part is content begin
-				// if the content is the first part of the first read, then it's also content begin
-				isContentBegin := lastIsTag || (readCount == 1 && i == 0)
-				lastIsTag = false
 				if isContentBegin {
+					isContentBegin = false
 					if part, err = es.trimAndWriteBom(part); err != nil {
 						return nil, err
 					}
@@ -236,18 +231,23 @@ func (e *htmlChunkReader) readRunes() (parts [][]byte, partInTag []bool, _ error
 		copy(curPart, e.readBuf[pos:curPartEnd])
 		// now we get the curPart bytes, but we can't directly use it, the last rune in it might have been cut
 		// try to decode the last rune, if it's invalid, then we cut the last byte and try again until we get a valid rune or no byte left
-		for i := curPartLen - 1; i >= 0; i-- {
-			last, lastSize := utf8.DecodeRune(curPart[i:])
-			if last == utf8.RuneError && lastSize == 1 {
-				curPartLen--
-			} else {
-				curPartLen += lastSize - 1
-				break
+		if e.readErr == nil && curPartEnd == len(e.readBuf) {
+			for i := curPartLen - 1; i >= 0; i-- {
+				last, lastSize := utf8.DecodeRune(curPart[i:])
+				if last == utf8.RuneError && lastSize == 1 {
+					curPartLen--
+				} else {
+					curPartLen += lastSize - 1
+					break
+				}
 			}
 		}
 		if curPartLen == 0 {
+			if !utf8.FullRune(curPart) && len(curPart) <= cap(e.readBuf)*3/4 {
+				break // keep a rune cut by the read boundary for the next read
+			}
 			// actually it's impossible that the part doesn't contain any valid rune,
-			// the only case is that the cap(readBuf) is too small, or the origin contain indeed doesn't contain any valid rune
+			// the only case is that the cap(readBuf) is too small, or the origin content indeed doesn't contain any valid rune
 			// * try to leave the last 4 bytes (possible longest utf-8 encoding) to next round
 			// * at least consume 1 byte to avoid infinite loop
 			curPartLen = max(len(curPart)-utf8.UTFMax, 1)

modules/charset/escape_test.go

@@ -6,6 +6,7 @@ package charset
 import (
 	"strings"
 	"testing"
+	"testing/iotest"
 
 	"gitea.dev/modules/htmlutil"
 	"gitea.dev/modules/setting"
@@ -160,6 +161,11 @@ func TestEscapeControlReader(t *testing.T) {
 			assert.Equal(t, tt.result, outStr)
 		})
 	}
+
+	output := &htmlutil.HTMLBuilder{}
+	_, err := EscapeControlReader(iotest.OneByteReader(strings.NewReader("\xef\xbb\xbftest")), output, &translation.MockLocale{})
+	assert.NoError(t, err)
+	assert.Equal(t, "\xef\xbb\xbftest", output.String())
 }
 
 func TestTrackHtmlTag(t *testing.T) {
@@ -234,4 +240,11 @@ func TestHTMLChunkReader(t *testing.T) {
 	testReadChunks(t, 5, input, []textPart{{"<Aé", true}, {"啊", true}, {"🌞>", true}, {"Aé", false}, {"啊", false}, {"🌞", false}})
 	testReadChunks(t, 6, input, []textPart{{"<Aé", true}, {"啊", true}, {"🌞>", true}, {"A", false}, {"é啊", false}, {"🌞", false}})
 	testReadChunks(t, 7, input, []textPart{{"<Aé啊", true}, {"🌞>", true}, {"A", false}, {"é啊", false}, {"🌞", false}})
+
+	for chunkSize := 6; chunkSize < 5+len(rune4); chunkSize++ {
+		testReadChunks(t, chunkSize, "<abc>"+rune4, []textPart{{"<abc>", true}, {rune4, false}})
+	}
+	testReadChunks(t, 6, "<abc>"+rune3[:2], []textPart{{"<abc>", true}, {rune3[:2], false}})
+	testReadChunks(t, 7, "<abc>"+rune3[:2]+"<b>", []textPart{{"<abc>", true}, {rune3[:2], false}, {"<b>", true}})
+	testReadChunks(t, 1, rune2, []textPart{{rune2[:1], false}, {rune2[1:], false}})
 }