mirror of
https://github.com/sipeed/sipeed_wiki.git
synced 2026-11-09 17:44:31 -06:00
ci(links): check notebook markdown cells and video/audio sources
The image link check only looked at markdown and <img> tags, so two kinds of references were never verified: - MaixPy3 docs are Jupyter notebooks (.ipynb). The GIF->WebP conversion updated the markdown pages but not the notebooks, and the check did not notice. - <video>/<source>/<audio> src and poster attributes. Both kinds of breakage stay invisible on the live site because the COS bucket still serves files that were removed from the repo. Notebooks that are not valid JSON are reported on stderr instead of being skipped silently. Also drop the two baseline entries fixed by #1048.
This commit is contained in:
@@ -6,8 +6,10 @@ file without touching the pages that use it) and hard to notice, because the
|
|||||||
published site keeps serving stale copies that were uploaded by earlier builds.
|
published site keeps serving stale copies that were uploaded by earlier builds.
|
||||||
This script catches them at CI time instead.
|
This script catches them at CI time instead.
|
||||||
|
|
||||||
Only real image references are checked: markdown ```` and HTML
|
Only real media references are checked: markdown ````, HTML
|
||||||
``<img src=...>``. Fenced and inline code is stripped first, so example commands
|
``<img src=...>``, and ``<video>``/``<source>``/``<audio>`` ``src``/``poster``.
|
||||||
|
Markdown cells of Jupyter notebooks (``.ipynb``, used by the MaixPy3 docs) are
|
||||||
|
checked too. Fenced and inline code is stripped first, so example commands
|
||||||
like ``convert in.jpg out.jpg`` are not mistaken for links.
|
like ``convert in.jpg out.jpg`` are not mistaken for links.
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
@@ -27,8 +29,9 @@ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|||||||
BASELINE = os.path.join(ROOT, "tools", "known_broken_images.txt")
|
BASELINE = os.path.join(ROOT, "tools", "known_broken_images.txt")
|
||||||
SCAN_DIRS = ["docs", "news", "pages", "layout"]
|
SCAN_DIRS = ["docs", "news", "pages", "layout"]
|
||||||
SKIP_DIRS = {".git", "out", ".venv", "node_modules", "__pycache__"}
|
SKIP_DIRS = {".git", "out", ".venv", "node_modules", "__pycache__"}
|
||||||
PAGE_EXTS = {".md", ".html"}
|
PAGE_EXTS = {".md", ".html", ".ipynb"}
|
||||||
IMG_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp", ".ico"}
|
IMG_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp", ".ico",
|
||||||
|
".mp4", ".webm", ".mov", ".mp3", ".ogg", ".wav"}
|
||||||
|
|
||||||
FENCED = re.compile(r"```.*?```|~~~.*?~~~", re.S)
|
FENCED = re.compile(r"```.*?```|~~~.*?~~~", re.S)
|
||||||
# HTML 注释里的引用不会渲染,不算断链
|
# HTML 注释里的引用不会渲染,不算断链
|
||||||
@@ -36,6 +39,20 @@ HTML_COMMENT = re.compile(r"<!--.*?-->", re.S)
|
|||||||
INLINE_CODE = re.compile(r"`[^`\n]*`")
|
INLINE_CODE = re.compile(r"`[^`\n]*`")
|
||||||
MD_IMG = re.compile(r"!\[[^\]]*\]\(\s*<?([^)\s>]+)")
|
MD_IMG = re.compile(r"!\[[^\]]*\]\(\s*<?([^)\s>]+)")
|
||||||
HTML_IMG = re.compile(r"<img\b[^>]*?\bsrc\s*=\s*[\"']([^\"']+)", re.I)
|
HTML_IMG = re.compile(r"<img\b[^>]*?\bsrc\s*=\s*[\"']([^\"']+)", re.I)
|
||||||
|
MEDIA_TAG = re.compile(r"<(?:video|source|audio)\b[^>]*>", re.I)
|
||||||
|
MEDIA_ATTR = re.compile(r"(?<![\w-])(?:src|poster)\s*=\s*[\"']([^\"']+)", re.I)
|
||||||
|
|
||||||
|
|
||||||
|
def read_page(path):
|
||||||
|
"""Return the text of a page; for notebooks, only the markdown cells."""
|
||||||
|
with open(path, encoding="utf-8") as f:
|
||||||
|
if not path.endswith(".ipynb"):
|
||||||
|
return f.read()
|
||||||
|
nb = json.load(f)
|
||||||
|
cells = [c for c in nb.get("cells", []) if c.get("cell_type") == "markdown"]
|
||||||
|
return "\n\n".join(
|
||||||
|
"".join(c["source"]) if isinstance(c.get("source"), list) else c.get("source", "")
|
||||||
|
for c in cells)
|
||||||
|
|
||||||
|
|
||||||
def load_routes():
|
def load_routes():
|
||||||
@@ -91,13 +108,19 @@ def scan():
|
|||||||
continue
|
continue
|
||||||
page = os.path.join(dirpath, name)
|
page = os.path.join(dirpath, name)
|
||||||
try:
|
try:
|
||||||
text = open(page, encoding="utf-8").read()
|
text = read_page(page)
|
||||||
|
except ValueError as e: # malformed notebook JSON
|
||||||
|
print(f"warning: cannot parse {os.path.relpath(page, ROOT)}: {e}",
|
||||||
|
file=sys.stderr)
|
||||||
|
continue
|
||||||
except (OSError, UnicodeDecodeError):
|
except (OSError, UnicodeDecodeError):
|
||||||
continue
|
continue
|
||||||
text = INLINE_CODE.sub(" ", FENCED.sub(" ", HTML_COMMENT.sub(" ", text)))
|
text = INLINE_CODE.sub(" ", FENCED.sub(" ", HTML_COMMENT.sub(" ", text)))
|
||||||
rel_page = os.path.relpath(page, ROOT)
|
rel_page = os.path.relpath(page, ROOT)
|
||||||
for ref in set(MD_IMG.findall(text) + HTML_IMG.findall(text)):
|
refs = MD_IMG.findall(text) + HTML_IMG.findall(text)
|
||||||
if "{{" in ref or "{%" in ref: # jinja template expression
|
refs += [r for tag in MEDIA_TAG.findall(text) for r in MEDIA_ATTR.findall(tag)]
|
||||||
|
for ref in set(refs):
|
||||||
|
if "{{" in ref or "{%" in ref or ref.startswith("attachment:"):
|
||||||
continue
|
continue
|
||||||
target = resolve(ref, page, routes)
|
target = resolve(ref, page, routes)
|
||||||
if target and not os.path.exists(target):
|
if target and not os.path.exists(target):
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
# CI fails on anything NOT listed here, so this file may shrink
|
# CI fails on anything NOT listed here, so this file may shrink
|
||||||
# but must never grow. Regenerate with:
|
# but must never grow. Regenerate with:
|
||||||
# python3 tools/check_image_links.py --update-baseline
|
# python3 tools/check_image_links.py --update-baseline
|
||||||
docs/hardware/en/kvm/NanoKVM/user_guide.md ./../../../assets/NanoKVM/guide/atx_2024.png
|
|
||||||
docs/hardware/en/kvm/NanoKVM_Pro/ledstrip.md ../../../assets/NanoKVM/pro/ledstrip/ledstrip_show.png
|
|
||||||
docs/hardware/en/logic_analyzer/slogic16u3/Software_User_Guide.md ./assets/Screenshots/SLogic_Connect_1.png
|
docs/hardware/en/logic_analyzer/slogic16u3/Software_User_Guide.md ./assets/Screenshots/SLogic_Connect_1.png
|
||||||
docs/hardware/zh/logic_analyzer/slogic16u3/Software_User_Guide.md ../../../en/logic_analyzer/slogic16u3/assets/Screenshots/SLogic_Connect_1.png
|
docs/hardware/zh/logic_analyzer/slogic16u3/Software_User_Guide.md ../../../en/logic_analyzer/slogic16u3/assets/Screenshots/SLogic_Connect_1.png
|
||||||
pages/index/zh/threshold.html demo_color.jpg
|
pages/index/zh/threshold.html demo_color.jpg
|
||||||
|
|||||||
Reference in New Issue
Block a user