diff --git a/modules/charset/escape_stream.go b/modules/charset/escape_stream.go index df31c3b16cf..8ab69622c8e 100644 --- a/modules/charset/escape_stream.go +++ b/modules/charset/escape_stream.go @@ -55,11 +55,9 @@ func escapeStream(locale translation.Locale, in io.Reader, out io.Writer, opts . es.allowed = opts[0].Allowed } - readCount := 0 - lastIsTag := false + isContentBegin := true for { parts, partInTag, err := es.readRunes() - readCount++ if err == io.EOF { return es.escaped, nil } else if err != nil { @@ -67,17 +65,14 @@ func escapeStream(locale translation.Locale, in io.Reader, out io.Writer, opts . } for i, part := range parts { if partInTag[i] { - lastIsTag = true + isContentBegin = true es.trackHtmlTag(part) if _, err := out.Write(part); err != nil { return nil, err } } else { - // if last part is tag, then this part is content begin - // if the content is the first part of the first read, then it's also content begin - isContentBegin := lastIsTag || (readCount == 1 && i == 0) - lastIsTag = false if isContentBegin { + isContentBegin = false if part, err = es.trimAndWriteBom(part); err != nil { return nil, err } @@ -236,18 +231,23 @@ func (e *htmlChunkReader) readRunes() (parts [][]byte, partInTag []bool, _ error copy(curPart, e.readBuf[pos:curPartEnd]) // now we get the curPart bytes, but we can't directly use it, the last rune in it might have been cut // try to decode the last rune, if it's invalid, then we cut the last byte and try again until we get a valid rune or no byte left - for i := curPartLen - 1; i >= 0; i-- { - last, lastSize := utf8.DecodeRune(curPart[i:]) - if last == utf8.RuneError && lastSize == 1 { - curPartLen-- - } else { - curPartLen += lastSize - 1 - break + if e.readErr == nil && curPartEnd == len(e.readBuf) { + for i := curPartLen - 1; i >= 0; i-- { + last, lastSize := utf8.DecodeRune(curPart[i:]) + if last == utf8.RuneError && lastSize == 1 { + curPartLen-- + } else { + curPartLen += lastSize - 1 + break + } } } if curPartLen == 0 { + if !utf8.FullRune(curPart) && len(curPart) <= cap(e.readBuf)*3/4 { + break // keep a rune cut by the read boundary for the next read + } // actually it's impossible that the part doesn't contain any valid rune, - // the only case is that the cap(readBuf) is too small, or the origin contain indeed doesn't contain any valid rune + // the only case is that the cap(readBuf) is too small, or the origin content indeed doesn't contain any valid rune // * try to leave the last 4 bytes (possible longest utf-8 encoding) to next round // * at least consume 1 byte to avoid infinite loop curPartLen = max(len(curPart)-utf8.UTFMax, 1) diff --git a/modules/charset/escape_test.go b/modules/charset/escape_test.go index 9961d55bc65..dd57f539556 100644 --- a/modules/charset/escape_test.go +++ b/modules/charset/escape_test.go @@ -6,6 +6,7 @@ package charset import ( "strings" "testing" + "testing/iotest" "gitea.dev/modules/htmlutil" "gitea.dev/modules/setting" @@ -160,6 +161,11 @@ func TestEscapeControlReader(t *testing.T) { assert.Equal(t, tt.result, outStr) }) } + + output := &htmlutil.HTMLBuilder{} + _, err := EscapeControlReader(iotest.OneByteReader(strings.NewReader("\xef\xbb\xbftest")), output, &translation.MockLocale{}) + assert.NoError(t, err) + assert.Equal(t, "\xef\xbb\xbftest", output.String()) } func TestTrackHtmlTag(t *testing.T) { @@ -234,4 +240,11 @@ func TestHTMLChunkReader(t *testing.T) { testReadChunks(t, 5, input, []textPart{{"", true}, {"AĆ©", false}, {"啊", false}, {"šŸŒž", false}}) testReadChunks(t, 6, input, []textPart{{"", true}, {"A", false}, {"Ć©å•Š", false}, {"šŸŒž", false}}) testReadChunks(t, 7, input, []textPart{{"", true}, {"A", false}, {"Ć©å•Š", false}, {"šŸŒž", false}}) + + for chunkSize := 6; chunkSize < 5+len(rune4); chunkSize++ { + testReadChunks(t, chunkSize, ""+rune4, []textPart{{"", true}, {rune4, false}}) + } + testReadChunks(t, 6, ""+rune3[:2], []textPart{{"", true}, {rune3[:2], false}}) + testReadChunks(t, 7, ""+rune3[:2]+"", []textPart{{"", true}, {rune3[:2], false}, {"", true}}) + testReadChunks(t, 1, rune2, []textPart{{rune2[:1], false}, {rune2[1:], false}}) }