diff --git a/modules/charset/escape_stream.go b/modules/charset/escape_stream.go
index df31c3b16cf..8ab69622c8e 100644
--- a/modules/charset/escape_stream.go
+++ b/modules/charset/escape_stream.go
@@ -55,11 +55,9 @@ func escapeStream(locale translation.Locale, in io.Reader, out io.Writer, opts .
es.allowed = opts[0].Allowed
}
- readCount := 0
- lastIsTag := false
+ isContentBegin := true
for {
parts, partInTag, err := es.readRunes()
- readCount++
if err == io.EOF {
return es.escaped, nil
} else if err != nil {
@@ -67,17 +65,14 @@ func escapeStream(locale translation.Locale, in io.Reader, out io.Writer, opts .
}
for i, part := range parts {
if partInTag[i] {
- lastIsTag = true
+ isContentBegin = true
es.trackHtmlTag(part)
if _, err := out.Write(part); err != nil {
return nil, err
}
} else {
- // if last part is tag, then this part is content begin
- // if the content is the first part of the first read, then it's also content begin
- isContentBegin := lastIsTag || (readCount == 1 && i == 0)
- lastIsTag = false
if isContentBegin {
+ isContentBegin = false
if part, err = es.trimAndWriteBom(part); err != nil {
return nil, err
}
@@ -236,18 +231,23 @@ func (e *htmlChunkReader) readRunes() (parts [][]byte, partInTag []bool, _ error
copy(curPart, e.readBuf[pos:curPartEnd])
// now we get the curPart bytes, but we can't directly use it, the last rune in it might have been cut
// try to decode the last rune, if it's invalid, then we cut the last byte and try again until we get a valid rune or no byte left
- for i := curPartLen - 1; i >= 0; i-- {
- last, lastSize := utf8.DecodeRune(curPart[i:])
- if last == utf8.RuneError && lastSize == 1 {
- curPartLen--
- } else {
- curPartLen += lastSize - 1
- break
+ if e.readErr == nil && curPartEnd == len(e.readBuf) {
+ for i := curPartLen - 1; i >= 0; i-- {
+ last, lastSize := utf8.DecodeRune(curPart[i:])
+ if last == utf8.RuneError && lastSize == 1 {
+ curPartLen--
+ } else {
+ curPartLen += lastSize - 1
+ break
+ }
}
}
if curPartLen == 0 {
+ if !utf8.FullRune(curPart) && len(curPart) <= cap(e.readBuf)*3/4 {
+ break // keep a rune cut by the read boundary for the next read
+ }
// actually it's impossible that the part doesn't contain any valid rune,
- // the only case is that the cap(readBuf) is too small, or the origin contain indeed doesn't contain any valid rune
+ // the only case is that the cap(readBuf) is too small, or the origin content indeed doesn't contain any valid rune
// * try to leave the last 4 bytes (possible longest utf-8 encoding) to next round
// * at least consume 1 byte to avoid infinite loop
curPartLen = max(len(curPart)-utf8.UTFMax, 1)
diff --git a/modules/charset/escape_test.go b/modules/charset/escape_test.go
index 9961d55bc65..dd57f539556 100644
--- a/modules/charset/escape_test.go
+++ b/modules/charset/escape_test.go
@@ -6,6 +6,7 @@ package charset
import (
"strings"
"testing"
+ "testing/iotest"
"gitea.dev/modules/htmlutil"
"gitea.dev/modules/setting"
@@ -160,6 +161,11 @@ func TestEscapeControlReader(t *testing.T) {
assert.Equal(t, tt.result, outStr)
})
}
+
+ output := &htmlutil.HTMLBuilder{}
+ _, err := EscapeControlReader(iotest.OneByteReader(strings.NewReader("\xef\xbb\xbftest")), output, &translation.MockLocale{})
+ assert.NoError(t, err)
+ assert.Equal(t, "\xef\xbb\xbftest", output.String())
}
func TestTrackHtmlTag(t *testing.T) {
@@ -234,4 +240,11 @@ func TestHTMLChunkReader(t *testing.T) {
testReadChunks(t, 5, input, []textPart{{"", true}, {"AĆ©", false}, {"å", false}, {"š", false}})
testReadChunks(t, 6, input, []textPart{{"", true}, {"A", false}, {"Ć©å", false}, {"š", false}})
testReadChunks(t, 7, input, []textPart{{"", true}, {"A", false}, {"Ć©å", false}, {"š", false}})
+
+ for chunkSize := 6; chunkSize < 5+len(rune4); chunkSize++ {
+ testReadChunks(t, chunkSize, ""+rune4, []textPart{{"", true}, {rune4, false}})
+ }
+ testReadChunks(t, 6, ""+rune3[:2], []textPart{{"", true}, {rune3[:2], false}})
+ testReadChunks(t, 7, ""+rune3[:2]+"", []textPart{{"", true}, {rune3[:2], false}, {"", true}})
+ testReadChunks(t, 1, rune2, []textPart{{rune2[:1], false}, {rune2[1:], false}})
}