From 24bceb9f6ec34045bd6e54b6d8963f196a59ea56 Mon Sep 17 00:00:00 2001 From: Hyeonmin Park Date: Mon, 9 Nov 2026 16:36:34 +0000 Subject: [PATCH] fix(markup): correct character escaping at read buffer boundaries (#40209) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a file, README or wiki page is rendered as markup, the HTML is read through a 32 KiB buffer to escape invisible and ambiguous characters. A multi-byte character cut by that buffer right after a tag was shown as broken characters, e.g. `

보기

` became `

���기

`. Such a part is now kept for the next read. Two related cases are fixed as well: - Invalid bytes right before a tag are marked as broken instead of passing through raw as part of the tag. - A BOM split across reads is kept instead of being escaped as invisible. To reproduce, view a Markdown file with 32734 `A` characters, a blank line and `보기`, which puts `보` across the first 32 KiB of the rendered HTML: [split-rune-after-tag.md](https://github.com/user-attachments/files/34361104/split-rune-after-tag.md) https://github.com/go-gitea/gitea/pull/39937 replaces this reader with a single in-memory pass. This small fix can be backported to the release branches. Assisted-by: Claude Code:claude-opus-5-5 ---- | Before (2ed6a6f61b6ec7f99b791bdc84c083a8667fcb7e) | After (2a1d09755fecab764336ce9069e864c1a3b20c7c) | | --- | --- | | ![image](https://github.com/user-attachments/assets/d4f79a8a-17bd-4b7f-b853-1d1a7241dc9b) | ![image](https://github.com/user-attachments/assets/d4b8b3dd-e28e-44dd-99de-d250ee4c282c) | Co-authored-by: wxiaoguang --- modules/charset/escape_stream.go | 32 ++++++++++++++++---------------- modules/charset/escape_test.go | 13 +++++++++++++ 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/modules/charset/escape_stream.go b/modules/charset/escape_stream.go index df31c3b16cf..8ab69622c8e 100644 --- a/modules/charset/escape_stream.go +++ b/modules/charset/escape_stream.go @@ -55,11 +55,9 @@ func escapeStream(locale translation.Locale, in io.Reader, out io.Writer, opts . es.allowed = opts[0].Allowed } - readCount := 0 - lastIsTag := false + isContentBegin := true for { parts, partInTag, err := es.readRunes() - readCount++ if err == io.EOF { return es.escaped, nil } else if err != nil { @@ -67,17 +65,14 @@ func escapeStream(locale translation.Locale, in io.Reader, out io.Writer, opts . } for i, part := range parts { if partInTag[i] { - lastIsTag = true + isContentBegin = true es.trackHtmlTag(part) if _, err := out.Write(part); err != nil { return nil, err } } else { - // if last part is tag, then this part is content begin - // if the content is the first part of the first read, then it's also content begin - isContentBegin := lastIsTag || (readCount == 1 && i == 0) - lastIsTag = false if isContentBegin { + isContentBegin = false if part, err = es.trimAndWriteBom(part); err != nil { return nil, err } @@ -236,18 +231,23 @@ func (e *htmlChunkReader) readRunes() (parts [][]byte, partInTag []bool, _ error copy(curPart, e.readBuf[pos:curPartEnd]) // now we get the curPart bytes, but we can't directly use it, the last rune in it might have been cut // try to decode the last rune, if it's invalid, then we cut the last byte and try again until we get a valid rune or no byte left - for i := curPartLen - 1; i >= 0; i-- { - last, lastSize := utf8.DecodeRune(curPart[i:]) - if last == utf8.RuneError && lastSize == 1 { - curPartLen-- - } else { - curPartLen += lastSize - 1 - break + if e.readErr == nil && curPartEnd == len(e.readBuf) { + for i := curPartLen - 1; i >= 0; i-- { + last, lastSize := utf8.DecodeRune(curPart[i:]) + if last == utf8.RuneError && lastSize == 1 { + curPartLen-- + } else { + curPartLen += lastSize - 1 + break + } } } if curPartLen == 0 { + if !utf8.FullRune(curPart) && len(curPart) <= cap(e.readBuf)*3/4 { + break // keep a rune cut by the read boundary for the next read + } // actually it's impossible that the part doesn't contain any valid rune, - // the only case is that the cap(readBuf) is too small, or the origin contain indeed doesn't contain any valid rune + // the only case is that the cap(readBuf) is too small, or the origin content indeed doesn't contain any valid rune // * try to leave the last 4 bytes (possible longest utf-8 encoding) to next round // * at least consume 1 byte to avoid infinite loop curPartLen = max(len(curPart)-utf8.UTFMax, 1) diff --git a/modules/charset/escape_test.go b/modules/charset/escape_test.go index 9961d55bc65..dd57f539556 100644 --- a/modules/charset/escape_test.go +++ b/modules/charset/escape_test.go @@ -6,6 +6,7 @@ package charset import ( "strings" "testing" + "testing/iotest" "gitea.dev/modules/htmlutil" "gitea.dev/modules/setting" @@ -160,6 +161,11 @@ func TestEscapeControlReader(t *testing.T) { assert.Equal(t, tt.result, outStr) }) } + + output := &htmlutil.HTMLBuilder{} + _, err := EscapeControlReader(iotest.OneByteReader(strings.NewReader("\xef\xbb\xbftest")), output, &translation.MockLocale{}) + assert.NoError(t, err) + assert.Equal(t, "\xef\xbb\xbftest", output.String()) } func TestTrackHtmlTag(t *testing.T) { @@ -234,4 +240,11 @@ func TestHTMLChunkReader(t *testing.T) { testReadChunks(t, 5, input, []textPart{{"", true}, {"Aé", false}, {"啊", false}, {"🌞", false}}) testReadChunks(t, 6, input, []textPart{{"", true}, {"A", false}, {"é啊", false}, {"🌞", false}}) testReadChunks(t, 7, input, []textPart{{"", true}, {"A", false}, {"é啊", false}, {"🌞", false}}) + + for chunkSize := 6; chunkSize < 5+len(rune4); chunkSize++ { + testReadChunks(t, chunkSize, ""+rune4, []textPart{{"", true}, {rune4, false}}) + } + testReadChunks(t, 6, ""+rune3[:2], []textPart{{"", true}, {rune3[:2], false}}) + testReadChunks(t, 7, ""+rune3[:2]+"", []textPart{{"", true}, {rune3[:2], false}, {"", true}}) + testReadChunks(t, 1, rune2, []textPart{{rune2[:1], false}, {rune2[1:], false}}) }