From 3f35eacad2827e450d62bfcb0f9c4f38c7a49ada Mon Sep 17 00:00:00 2001 From: kuxiaowo <1127944431@qq.com> Date: Fri, 23 Oct 2026 02:31:11 +0800 Subject: [PATCH] fix(website): handle UTF-8 output and slash refs in link audits (#4123) --- scripts/audit-website-links.py | 5 +- scripts/test-website.py | 110 +++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/scripts/audit-website-links.py b/scripts/audit-website-links.py index cff49da0b5..389887090c 100644 --- a/scripts/audit-website-links.py +++ b/scripts/audit-website-links.py @@ -19,7 +19,7 @@ from website import authored_markdown @lru_cache(maxsize=None) def github_api(endpoint): """Read GitHub evidence through the caller's gh installation and authentication.""" - result = subprocess.run(["gh", "api", endpoint], capture_output=True, text=True, timeout=20) + result = subprocess.run(["gh", "api", endpoint], capture_output=True, encoding="utf-8", timeout=20) if result.returncode: raise ValueError(result.stderr.strip()) return json.loads(result.stdout) @@ -32,7 +32,8 @@ def github_target(url): return None repo = f"repos/{parts[0]}/{parts[1]}" if url.hostname == "raw.githubusercontent.com" and len(parts) >= 4: - return f"{repo}/contents/{'/'.join(parts[3:])}?ref={quote(parts[2], safe='')}" + # Raw URLs omit the blob segment; share ref/path resolution below. + parts = [*parts[:2], "blob", *parts[2:]] if len(parts) == 2: return repo kind = parts[2] diff --git a/scripts/test-website.py b/scripts/test-website.py index 0b60cff7e4..c6f091cde8 100644 --- a/scripts/test-website.py +++ b/scripts/test-website.py @@ -7,7 +7,10 @@ import io import json from pathlib import Path import shutil +import subprocess +import sys import tempfile +from textwrap import dedent from threading import Thread import unittest from unittest.mock import patch @@ -35,6 +38,113 @@ def load_script(name): PREPARE = load_script("prepare-website") VERIFY = load_script("verify-website") WEBSITE = load_script("website") +AUDIT = load_script("audit-website-links") + + +class WebsiteLinkAuditChecks(unittest.TestCase): + """Check GitHub transport and URL classification without live network access.""" + + def test_github_json_is_decoded_as_utf8_under_a_legacy_locale(self): + # Exercise real subprocess pipe decoding with UTF-8 mode disabled. Only + # the external gh executable and the default locale are substituted. + program = dedent('''\ + import importlib.util + import json + from pathlib import Path + import subprocess + import sys + from unittest.mock import patch + + scripts = Path(sys.argv[1]) + sys.path.insert(0, str(scripts)) + spec = importlib.util.spec_from_file_location("audit", scripts / "audit-website-links.py") + audit = importlib.util.module_from_spec(spec) + spec.loader.exec_module(audit) + payload = json.dumps({"title": "Release \\u2603"}, ensure_ascii=False).encode("utf-8") + run = subprocess.run + + def gh(command, **options): + assert command == ["gh", "api", "repos/example/project"] + return run([sys.executable, "-c", "import sys; sys.stdout.buffer.write(" + repr(payload) + ")"], **options) + + with patch("locale.getencoding", return_value="cp936"), patch.object(audit.subprocess, "run", side_effect=gh): + print(json.dumps(audit.github_api("repos/example/project"))) + ''') + result = subprocess.run( + [sys.executable, "-X", "utf8=0", "-c", program, str(ROOT / "scripts")], + capture_output=True, encoding="utf-8", timeout=20, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout), {"title": "Release \u2603"}) + + def test_raw_and_browser_links_resolve_the_same_ref_and_encoded_path(self): + # Prefix lookalikes must not win, and tags use the same ref/path split. + references = { + "repos/example/project/git/matching-refs/heads/topic": [ + {"ref": "refs/heads/topic"}, + {"ref": "refs/heads/topic/nested"}, + {"ref": "refs/heads/topic/nestedness"}, + ], + "repos/example/project/git/matching-refs/tags/topic": [ + {"ref": "refs/tags/topic/nested/v1"}, + ], + "repos/example/project/git/matching-refs/heads/topic%2Fnested": [ + {"ref": "refs/heads/topic/nested"}, + {"ref": "refs/heads/topic/nestedness"}, + ], + "repos/example/project/git/matching-refs/tags/topic%2Fnested": [ + {"ref": "refs/tags/topic/nested/v1"}, + ], + } + cases = ( + ("topic/nested/guide%20%23%3F.md", "guide%20%23%3F.md", "topic%2Fnested"), + ("topic/nested/v1/README.md", "README.md", "topic%2Fnested%2Fv1"), + ("topic%2Fnested/README.md", "README.md", "topic%2Fnested"), + ) + for suffix, path, ref in cases: + endpoint = f"repos/example/project/contents/{path}?ref={ref}" + responses = {**references, endpoint: {"name": "README.md"}} + + def github_api(target): + if target not in responses: + raise ValueError("gh: Not Found (HTTP 404)") + return responses[target] + + for base in ( + "https://raw.githubusercontent.com/example/project/", + "https://github.com/example/project/blob/", + "https://github.com/example/project/tree/", + ): + with self.subTest(url=base + suffix), patch.object(AUDIT, "github_api", side_effect=github_api): + report = AUDIT.audit_url((base + suffix, ["fixture.html"])) + self.assertEqual(report["status"], "ok") + self.assertEqual(report["endpoint"], endpoint) + + def test_plain_refs_and_pinned_commits_keep_their_contents_target(self): + for ref in ("main", "a" * 40): + responses = { + "repos/example/project/git/matching-refs/heads/main": [{"ref": "refs/heads/main"}], + "repos/example/project/git/matching-refs/tags/main": [], + f"repos/example/project/contents/README.md?ref={ref}": {"name": "README.md"}, + } + for base in ("https://raw.githubusercontent.com/example/project/", "https://github.com/example/project/blob/"): + with self.subTest(ref=ref, base=base), patch.object(AUDIT, "github_api", side_effect=responses.__getitem__): + report = AUDIT.audit_url((base + ref + "/README.md", ["fixture.html"])) + self.assertEqual(report["status"], "ok") + self.assertEqual(report["endpoint"], f"repos/example/project/contents/README.md?ref={ref}") + + def test_missing_contents_and_unavailable_github_probes_remain_distinct(self): + cases = ( + (ValueError("gh: Not Found (HTTP 404)"), "broken"), + (ValueError("gh: Forbidden (HTTP 403)"), "unknown"), + (subprocess.TimeoutExpired(["gh", "api"], 20), "unknown"), + ) + for failure, status in cases: + for base in ("https://raw.githubusercontent.com/example/project/", "https://github.com/example/project/blob/"): + with self.subTest(failure=str(failure), base=base), patch.object(AUDIT, "github_api", side_effect=failure): + report = AUDIT.audit_url((base + "a" * 40 + "/missing.md", ["fixture.html"])) + self.assertEqual(report["status"], status) + self.assertEqual(report["reason"], str(failure)) class WebsiteSearchChecks(unittest.TestCase):