fix(markup): recognize issue references next to CJK punctuation (#40097)

Issue references next to non-ASCII characters, common in Chinese and
Japanese text, are ignored: `修复#1`, `（#2）` or `Closes #3，` create no
link or cross reference.

Like on GitHub, any non-ASCII character may now precede `#123`, `!123`
or `owner/repo#123`, and anything but a word character may follow it.
Dead trailing-space handling in the reference and mention search is
removed.

I used Claude Code to write this change.

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/markup/common/footnote.go

@@ -21,6 +21,10 @@ import (
 	"github.com/yuin/goldmark/util"
 )
 
+func canKeepRune(r rune) bool {
+	return r == '-' || unicode.In(r, unicode.L, unicode.M, unicode.Nd, unicode.Nl, unicode.Pc, unicode.Other_Alphabetic)
+}
+
 // CleanValue will clean a value to make it safe to be an id
 // This function is quite different from the original goldmark function
 // and more closely matches the output from the shurcooL sanitizer
@@ -30,7 +34,7 @@ func CleanValue(value []byte) []byte {
 		switch {
 		case r == ' ':
 			return '-'
-		case r == '-' || unicode.In(r, unicode.L, unicode.M, unicode.Nd, unicode.Nl, unicode.Pc, unicode.Other_Alphabetic):
+		case canKeepRune(r):
 			return unicode.ToLower(r)
 		}
 		return -1

modules/markup/html_internal_test.go

@@ -157,6 +157,11 @@ func TestRender_IssueIndexPattern2(t *testing.T) {
 	// should render with :
 	test("#1234: test", "%s: test", "#", 1234)
 	test("wow (#54321: test)", "wow (%s: test)", "#", 54321)
+
+	// should render next to non-ASCII characters
+	test("修复#1。见（#2）", "修复%s。见（%s）", "#", 1, 2)
+	test("「#3」と“#4”、#5～#6🎉", "「%s」と“%s”、%s～%s🎉", "#", 3, 4, 5, 6)
+	test("见 #7修复，#8", "见 #7修复，%s", "#", 8)
 }
 
 func TestRender_IssueIndexPattern3(t *testing.T) {

modules/references/references.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"unicode"
 	"unicode/utf8"
 
 	"gitea.dev/modules/log"
@@ -32,17 +33,17 @@ var (
 	// mentionPattern matches all mentions in the form of "@user" or "@org/team"
 	mentionPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(@[-\w][-.\w]*?|@[-\w][-.\w]*?/[-\w][-.\w]*?)(?:\s|$|[:,;.?!](\s|$)|'|\)|\])`)
 	// issueNumericPattern matches string that references to a numeric issue, e.g. #1287
-	issueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|\'|\")([#!][0-9]+)(?:\s|$|\)|\]|\'|\"|[:;,.?!]\s|[:;,.?!]$)`)
+	// TODO: REFERENCE-REGEXP-PATTERN: findValidReferenceIndex checks a non-ASCII trailing character
+	issueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|\'|\"|[^\x00-\x7f])([#!][0-9]+)(?:\s|$|\)|\]|\'|\"|[:;,.?!]\s|[:;,.?!]$|[^\x00-\x7f])`)
 	// issueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234
 	issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|\"|\')([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|:|\.(\s|$)|\"|\'|,)`)
 	// crossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository
 	// e.g. org/repo#12345
-	crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+[#!][0-9]+)(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`)
+	// TODO: REFERENCE-REGEXP-PATTERN: findValidReferenceIndex checks a non-ASCII trailing character
+	crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[|[^\x00-\x7f])([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+[#!][0-9]+)(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$|[^\x00-\x7f])`)
 	// crossReferenceCommitPattern matches a string that references a commit in a different repository
 	// e.g. go-gitea/gitea@d8a994ef, go-gitea/gitea@d8a994ef243349f321568f9e36d5c3f444b99cae (7-40 characters)
 	crossReferenceCommitPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+)/([0-9a-zA-Z-_\.]+)@([0-9a-f]{7,64})(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`)
-	// spaceTrimmedPattern let's find the trailing space
-	spaceTrimmedPattern = regexp.MustCompile(`(?:.*[0-9a-zA-Z-_])\s`)
 	// timeLogPattern matches string for time tracking
 	timeLogPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(@([0-9]+([\.,][0-9]+)?(w|d|m|h))+)(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`)
 
@@ -229,12 +230,7 @@ func FindAllMentionsBytes(content []byte) []RefSpan {
 			break
 		}
 		ret = append(ret, RefSpan{Start: match[2] + pos, End: match[3] + pos})
-		notrail := spaceTrimmedPattern.FindSubmatchIndex(content[match[2]+pos : match[3]+pos])
-		if notrail == nil {
-			pos = match[3] + pos
-		} else {
-			pos = match[3] + pos + notrail[1] - notrail[3]
-		}
+		pos += match[3]
 	}
 	return ret
 }
@@ -338,18 +334,45 @@ func FindAllIssueReferences(content string) []IssueReference {
 	return rawToIssueReferenceList(findAllIssueReferencesBytes(contentBytes, []string{}, nil))
 }
 
+func isReferencePostBoundary(r rune) bool {
+	isWordChar := unicode.In(r, unicode.L, unicode.M, unicode.Nd, unicode.Nl, unicode.Pc, unicode.Other_Alphabetic, unicode.Join_Control)
+	return !isWordChar
+}
+
+// findValidReferenceIndex returns the first match from pos that isn't followed by a word character
+func findValidReferenceIndex(pattern *regexp.Regexp, content []byte, pos int) []int {
+	for pos < len(content) {
+		match := pattern.FindSubmatchIndex(content[pos:])
+		if match == nil {
+			return nil
+		}
+		for i := range match {
+			match[i] += pos
+		}
+		// TODO: REFERENCE-REGEXP-PATTERN: it depends on the regex pattern: the target is in the second capture group.
+		// It is very fragile because the pattern is passed in freely.
+		remaining := content[match[3]:]
+		if r, _ := utf8.DecodeRune(remaining); isReferencePostBoundary(r) {
+			return match
+		}
+		pos = match[3]
+	}
+	return nil
+}
+
 // FindRenderizableReferenceNumeric returns the first unvalidated reference found in a string.
 func FindRenderizableReferenceNumeric(content string, prOnly, crossLinkOnly bool) *RenderizableReference {
+	contentBytes := util.UnsafeStringToBytes(content)
 	var match []int
 	if !crossLinkOnly {
-		match = issueNumericPattern.FindStringSubmatchIndex(content)
+		match = findValidReferenceIndex(issueNumericPattern, contentBytes, 0)
 	}
 	if match == nil {
-		if match = crossReferenceIssueNumericPattern.FindStringSubmatchIndex(content); match == nil {
+		if match = findValidReferenceIndex(crossReferenceIssueNumericPattern, contentBytes, 0); match == nil {
 			return nil
 		}
 	}
-	r := getCrossReference(util.UnsafeStringToBytes(content), match[2], match[3], false, prOnly)
+	r := getCrossReference(contentBytes, match[2], match[3], false, prOnly)
 	if r == nil {
 		return nil
 	}
@@ -432,43 +455,16 @@ func FindRenderizableReferenceAlphanumeric(content string) *RenderizableReferenc
 // originalContent is optional and used to detect closing/reopening keywords for URL references.
 func findAllIssueReferencesBytes(content []byte, links []string, originalContent []byte) []*rawReference {
 	ret := make([]*rawReference, 0, 10)
-	pos := 0
 
 	// Sadly we can't use FindAllSubmatchIndex because our pattern checks for starting and
 	// trailing spaces (\s#ref,\s), so if we get two consecutive references, the space
 	// from the second reference will be "eaten" by the first one:
 	// ...\s#ref1\s#ref2\s...	--> ...`\s#ref1\s`, (not) `#ref2,\s...`
-	for {
-		match := issueNumericPattern.FindSubmatchIndex(content[pos:])
-		if match == nil {
-			break
-		}
-		if ref := getCrossReference(content, match[2]+pos, match[3]+pos, false, false); ref != nil {
-			ret = append(ret, ref)
-		}
-		notrail := spaceTrimmedPattern.FindSubmatchIndex(content[match[2]+pos : match[3]+pos])
-		if notrail == nil {
-			pos = match[3] + pos
-		} else {
-			pos = match[3] + pos + notrail[1] - notrail[3]
-		}
-	}
-
-	pos = 0
-
-	for {
-		match := crossReferenceIssueNumericPattern.FindSubmatchIndex(content[pos:])
-		if match == nil {
-			break
-		}
-		if ref := getCrossReference(content, match[2]+pos, match[3]+pos, false, false); ref != nil {
-			ret = append(ret, ref)
-		}
-		notrail := spaceTrimmedPattern.FindSubmatchIndex(content[match[2]+pos : match[3]+pos])
-		if notrail == nil {
-			pos = match[3] + pos
-		} else {
-			pos = match[3] + pos + notrail[1] - notrail[3]
+	for _, pattern := range []*regexp.Regexp{issueNumericPattern, crossReferenceIssueNumericPattern} {
+		for match := findValidReferenceIndex(pattern, content, 0); match != nil; match = findValidReferenceIndex(pattern, content, match[3]) {
+			if ref := getCrossReference(content, match[2], match[3], false, false); ref != nil {
+				ret = append(ret, ref)
+			}
 		}
 	}
 
@@ -524,7 +520,7 @@ func findAllIssueReferencesBytes(content []byte, links []string, originalContent
 		return ret
 	}
 
-	pos = 0
+	pos := 0
 
 	for {
 		match := timeLogPattern.FindSubmatchIndex(content[pos:])

modules/references/references_test.go

@@ -89,6 +89,15 @@ func TestFindAllIssueReferences(t *testing.T) {
 				{200, "org3", "repo4", "200", true, XRefActionNone, &RefSpan{Start: 5, End: 19}, nil, ""},
 			},
 		},
+		{
+			"Closes #3，see（#4）and org3/repo4#200。修复#5 #6修",
+			[]testResult{
+				{3, "", "", "3", false, XRefActionCloses, &RefSpan{Start: 7, End: 9}, &RefSpan{Start: 0, End: 6}, ""},
+				{4, "", "", "4", false, XRefActionNone, &RefSpan{Start: 18, End: 20}, nil, ""},
+				{5, "", "", "5", false, XRefActionNone, &RefSpan{Start: 50, End: 52}, nil, ""},
+				{200, "org3", "repo4", "200", false, XRefActionNone, &RefSpan{Start: 27, End: 41}, nil, ""},
+			},
+		},
 		{
 			"This [one](#919) no, this is a URL fragment.",
 			[]testResult{},