From 7efb5c055fc6480567571a4868b6dc8ec9653088 Mon Sep 17 00:00:00 2001 From: Wei-Lun Li Date: Sat, 7 Nov 2026 10:14:47 +0000 Subject: [PATCH] fix(markup): recognize issue references next to CJK punctuation (#40097) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue references next to non-ASCII characters, common in Chinese and Japanese text, are ignored: `修复#1`, `（#2）` or `Closes #3，` create no link or cross reference. Like on GitHub, any non-ASCII character may now precede `#123`, `!123` or `owner/repo#123`, and anything but a word character may follow it. Dead trailing-space handling in the reference and mention search is removed. I used Claude Code to write this change. --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: silverwind Co-authored-by: wxiaoguang --- modules/markup/common/footnote.go | 6 +- modules/markup/html_internal_test.go | 5 ++ modules/references/references.go | 88 +++++++++++++-------------- modules/references/references_test.go | 9 +++ 4 files changed, 61 insertions(+), 47 deletions(-) diff --git a/modules/markup/common/footnote.go b/modules/markup/common/footnote.go index 4f1c0ed9363..cf2a57445f1 100644 --- a/modules/markup/common/footnote.go +++ b/modules/markup/common/footnote.go @@ -21,6 +21,10 @@ import ( "github.com/yuin/goldmark/util" ) +func canKeepRune(r rune) bool { + return r == '-' || unicode.In(r, unicode.L, unicode.M, unicode.Nd, unicode.Nl, unicode.Pc, unicode.Other_Alphabetic) +} + // CleanValue will clean a value to make it safe to be an id // This function is quite different from the original goldmark function // and more closely matches the output from the shurcooL sanitizer @@ -30,7 +34,7 @@ func CleanValue(value []byte) []byte { switch { case r == ' ': return '-' - case r == '-' || unicode.In(r, unicode.L, unicode.M, unicode.Nd, unicode.Nl, unicode.Pc, unicode.Other_Alphabetic): + case canKeepRune(r): return unicode.ToLower(r) } return -1 diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go index 6c0f52c879a..248f1066ba2 100644 --- a/modules/markup/html_internal_test.go +++ b/modules/markup/html_internal_test.go @@ -157,6 +157,11 @@ func TestRender_IssueIndexPattern2(t *testing.T) { // should render with : test("#1234: test", "%s: test", "#", 1234) test("wow (#54321: test)", "wow (%s: test)", "#", 54321) + + // should render next to non-ASCII characters + test("修复#1。见（#2）", "修复%s。见（%s）", "#", 1, 2) + test("「#3」と“#4”、#5～#6🎉", "「%s」と“%s”、%s～%s🎉", "#", 3, 4, 5, 6) + test("见 #7修复，#8", "见 #7修复，%s", "#", 8) } func TestRender_IssueIndexPattern3(t *testing.T) { diff --git a/modules/references/references.go b/modules/references/references.go index 88530f509ef..46e89ab16c0 100644 --- a/modules/references/references.go +++ b/modules/references/references.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "sync" + "unicode" "unicode/utf8" "gitea.dev/modules/log" @@ -32,17 +33,17 @@ var ( // mentionPattern matches all mentions in the form of "@user" or "@org/team" mentionPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(@[-\w][-.\w]*?|@[-\w][-.\w]*?/[-\w][-.\w]*?)(?:\s|$|[:,;.?!](\s|$)|'|\)|\])`) // issueNumericPattern matches string that references to a numeric issue, e.g. #1287 - issueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|\'|\")([#!][0-9]+)(?:\s|$|\)|\]|\'|\"|[:;,.?!]\s|[:;,.?!]$)`) + // TODO: REFERENCE-REGEXP-PATTERN: findValidReferenceIndex checks a non-ASCII trailing character + issueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|\'|\"|[^\x00-\x7f])([#!][0-9]+)(?:\s|$|\)|\]|\'|\"|[:;,.?!]\s|[:;,.?!]$|[^\x00-\x7f])`) // issueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234 issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|\"|\')([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|:|\.(\s|$)|\"|\'|,)`) // crossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository // e.g. org/repo#12345 - crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+[#!][0-9]+)(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`) + // TODO: REFERENCE-REGEXP-PATTERN: findValidReferenceIndex checks a non-ASCII trailing character + crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|[^\x00-\x7f])([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+[#!][0-9]+)(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$|[^\x00-\x7f])`) // crossReferenceCommitPattern matches a string that references a commit in a different repository // e.g. go-gitea/gitea@d8a994ef, go-gitea/gitea@d8a994ef243349f321568f9e36d5c3f444b99cae (7-40 characters) crossReferenceCommitPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+)/([0-9a-zA-Z-_\.]+)@([0-9a-f]{7,64})(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`) - // spaceTrimmedPattern let's find the trailing space - spaceTrimmedPattern = regexp.MustCompile(`(?:.*[0-9a-zA-Z-_])\s`) // timeLogPattern matches string for time tracking timeLogPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(@([0-9]+([\.,][0-9]+)?(w|d|m|h))+)(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`) @@ -229,12 +230,7 @@ func FindAllMentionsBytes(content []byte) []RefSpan { break } ret = append(ret, RefSpan{Start: match[2] + pos, End: match[3] + pos}) - notrail := spaceTrimmedPattern.FindSubmatchIndex(content[match[2]+pos : match[3]+pos]) - if notrail == nil { - pos = match[3] + pos - } else { - pos = match[3] + pos + notrail[1] - notrail[3] - } + pos += match[3] } return ret } @@ -338,18 +334,45 @@ func FindAllIssueReferences(content string) []IssueReference { return rawToIssueReferenceList(findAllIssueReferencesBytes(contentBytes, []string{}, nil)) } +func isReferencePostBoundary(r rune) bool { + isWordChar := unicode.In(r, unicode.L, unicode.M, unicode.Nd, unicode.Nl, unicode.Pc, unicode.Other_Alphabetic, unicode.Join_Control) + return !isWordChar +} + +// findValidReferenceIndex returns the first match from pos that isn't followed by a word character +func findValidReferenceIndex(pattern *regexp.Regexp, content []byte, pos int) []int { + for pos < len(content) { + match := pattern.FindSubmatchIndex(content[pos:]) + if match == nil { + return nil + } + for i := range match { + match[i] += pos + } + // TODO: REFERENCE-REGEXP-PATTERN: it depends on the regex pattern: the target is in the second capture group. + // It is very fragile because the pattern is passed in freely. + remaining := content[match[3]:] + if r, _ := utf8.DecodeRune(remaining); isReferencePostBoundary(r) { + return match + } + pos = match[3] + } + return nil +} + // FindRenderizableReferenceNumeric returns the first unvalidated reference found in a string. func FindRenderizableReferenceNumeric(content string, prOnly, crossLinkOnly bool) *RenderizableReference { + contentBytes := util.UnsafeStringToBytes(content) var match []int if !crossLinkOnly { - match = issueNumericPattern.FindStringSubmatchIndex(content) + match = findValidReferenceIndex(issueNumericPattern, contentBytes, 0) } if match == nil { - if match = crossReferenceIssueNumericPattern.FindStringSubmatchIndex(content); match == nil { + if match = findValidReferenceIndex(crossReferenceIssueNumericPattern, contentBytes, 0); match == nil { return nil } } - r := getCrossReference(util.UnsafeStringToBytes(content), match[2], match[3], false, prOnly) + r := getCrossReference(contentBytes, match[2], match[3], false, prOnly) if r == nil { return nil } @@ -432,43 +455,16 @@ func FindRenderizableReferenceAlphanumeric(content string) *RenderizableReferenc // originalContent is optional and used to detect closing/reopening keywords for URL references. func findAllIssueReferencesBytes(content []byte, links []string, originalContent []byte) []*rawReference { ret := make([]*rawReference, 0, 10) - pos := 0 // Sadly we can't use FindAllSubmatchIndex because our pattern checks for starting and // trailing spaces (\s#ref,\s), so if we get two consecutive references, the space // from the second reference will be "eaten" by the first one: // ...\s#ref1\s#ref2\s... --> ...`\s#ref1\s`, (not) `#ref2,\s...` - for { - match := issueNumericPattern.FindSubmatchIndex(content[pos:]) - if match == nil { - break - } - if ref := getCrossReference(content, match[2]+pos, match[3]+pos, false, false); ref != nil { - ret = append(ret, ref) - } - notrail := spaceTrimmedPattern.FindSubmatchIndex(content[match[2]+pos : match[3]+pos]) - if notrail == nil { - pos = match[3] + pos - } else { - pos = match[3] + pos + notrail[1] - notrail[3] - } - } - - pos = 0 - - for { - match := crossReferenceIssueNumericPattern.FindSubmatchIndex(content[pos:]) - if match == nil { - break - } - if ref := getCrossReference(content, match[2]+pos, match[3]+pos, false, false); ref != nil { - ret = append(ret, ref) - } - notrail := spaceTrimmedPattern.FindSubmatchIndex(content[match[2]+pos : match[3]+pos]) - if notrail == nil { - pos = match[3] + pos - } else { - pos = match[3] + pos + notrail[1] - notrail[3] + for _, pattern := range []*regexp.Regexp{issueNumericPattern, crossReferenceIssueNumericPattern} { + for match := findValidReferenceIndex(pattern, content, 0); match != nil; match = findValidReferenceIndex(pattern, content, match[3]) { + if ref := getCrossReference(content, match[2], match[3], false, false); ref != nil { + ret = append(ret, ref) + } } } @@ -524,7 +520,7 @@ func findAllIssueReferencesBytes(content []byte, links []string, originalContent return ret } - pos = 0 + pos := 0 for { match := timeLogPattern.FindSubmatchIndex(content[pos:]) diff --git a/modules/references/references_test.go b/modules/references/references_test.go index 53cd98a931c..bfa5220a0e7 100644 --- a/modules/references/references_test.go +++ b/modules/references/references_test.go @@ -89,6 +89,15 @@ func TestFindAllIssueReferences(t *testing.T) { {200, "org3", "repo4", "200", true, XRefActionNone, &RefSpan{Start: 5, End: 19}, nil, ""}, }, }, + { + "Closes #3，see（#4）and org3/repo4#200。修复#5 #6修", + []testResult{ + {3, "", "", "3", false, XRefActionCloses, &RefSpan{Start: 7, End: 9}, &RefSpan{Start: 0, End: 6}, ""}, + {4, "", "", "4", false, XRefActionNone, &RefSpan{Start: 18, End: 20}, nil, ""}, + {5, "", "", "5", false, XRefActionNone, &RefSpan{Start: 50, End: 52}, nil, ""}, + {200, "org3", "repo4", "200", false, XRefActionNone, &RefSpan{Start: 27, End: 41}, nil, ""}, + }, + }, { "This [one](#919) no, this is a URL fragment.", []testResult{},