fix(website): handle UTF-8 output and slash refs in link audits (#4123)

scripts/audit-website-links.py

@@ -19,7 +19,7 @@ from website import authored_markdown
 @lru_cache(maxsize=None)
 def github_api(endpoint):
     """Read GitHub evidence through the caller's gh installation and authentication."""
-    result = subprocess.run(["gh", "api", endpoint], capture_output=True, text=True, timeout=20)
+    result = subprocess.run(["gh", "api", endpoint], capture_output=True, encoding="utf-8", timeout=20)
     if result.returncode:
         raise ValueError(result.stderr.strip())
     return json.loads(result.stdout)
@@ -32,7 +32,8 @@ def github_target(url):
         return None
     repo = f"repos/{parts[0]}/{parts[1]}"
     if url.hostname == "raw.githubusercontent.com" and len(parts) >= 4:
-        return f"{repo}/contents/{'/'.join(parts[3:])}?ref={quote(parts[2], safe='')}"
+        # Raw URLs omit the blob segment; share ref/path resolution below.
+        parts = [*parts[:2], "blob", *parts[2:]]
     if len(parts) == 2:
         return repo
     kind = parts[2]

scripts/test-website.py

@@ -7,7 +7,10 @@ import io
 import json
 from pathlib import Path
 import shutil
+import subprocess
+import sys
 import tempfile
+from textwrap import dedent
 from threading import Thread
 import unittest
 from unittest.mock import patch
@@ -35,6 +38,113 @@ def load_script(name):
 PREPARE = load_script("prepare-website")
 VERIFY = load_script("verify-website")
 WEBSITE = load_script("website")
+AUDIT = load_script("audit-website-links")
+
+
+class WebsiteLinkAuditChecks(unittest.TestCase):
+    """Check GitHub transport and URL classification without live network access."""
+
+    def test_github_json_is_decoded_as_utf8_under_a_legacy_locale(self):
+        # Exercise real subprocess pipe decoding with UTF-8 mode disabled. Only
+        # the external gh executable and the default locale are substituted.
+        program = dedent('''\
+            import importlib.util
+            import json
+            from pathlib import Path
+            import subprocess
+            import sys
+            from unittest.mock import patch
+
+            scripts = Path(sys.argv[1])
+            sys.path.insert(0, str(scripts))
+            spec = importlib.util.spec_from_file_location("audit", scripts / "audit-website-links.py")
+            audit = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(audit)
+            payload = json.dumps({"title": "Release \\u2603"}, ensure_ascii=False).encode("utf-8")
+            run = subprocess.run
+
+            def gh(command, **options):
+                assert command == ["gh", "api", "repos/example/project"]
+                return run([sys.executable, "-c", "import sys; sys.stdout.buffer.write(" + repr(payload) + ")"], **options)
+
+            with patch("locale.getencoding", return_value="cp936"), patch.object(audit.subprocess, "run", side_effect=gh):
+                print(json.dumps(audit.github_api("repos/example/project")))
+        ''')
+        result = subprocess.run(
+            [sys.executable, "-X", "utf8=0", "-c", program, str(ROOT / "scripts")],
+            capture_output=True, encoding="utf-8", timeout=20,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(json.loads(result.stdout), {"title": "Release \u2603"})
+
+    def test_raw_and_browser_links_resolve_the_same_ref_and_encoded_path(self):
+        # Prefix lookalikes must not win, and tags use the same ref/path split.
+        references = {
+            "repos/example/project/git/matching-refs/heads/topic": [
+                {"ref": "refs/heads/topic"},
+                {"ref": "refs/heads/topic/nested"},
+                {"ref": "refs/heads/topic/nestedness"},
+            ],
+            "repos/example/project/git/matching-refs/tags/topic": [
+                {"ref": "refs/tags/topic/nested/v1"},
+            ],
+            "repos/example/project/git/matching-refs/heads/topic%2Fnested": [
+                {"ref": "refs/heads/topic/nested"},
+                {"ref": "refs/heads/topic/nestedness"},
+            ],
+            "repos/example/project/git/matching-refs/tags/topic%2Fnested": [
+                {"ref": "refs/tags/topic/nested/v1"},
+            ],
+        }
+        cases = (
+            ("topic/nested/guide%20%23%3F.md", "guide%20%23%3F.md", "topic%2Fnested"),
+            ("topic/nested/v1/README.md", "README.md", "topic%2Fnested%2Fv1"),
+            ("topic%2Fnested/README.md", "README.md", "topic%2Fnested"),
+        )
+        for suffix, path, ref in cases:
+            endpoint = f"repos/example/project/contents/{path}?ref={ref}"
+            responses = {**references, endpoint: {"name": "README.md"}}
+
+            def github_api(target):
+                if target not in responses:
+                    raise ValueError("gh: Not Found (HTTP 404)")
+                return responses[target]
+
+            for base in (
+                "https://raw.githubusercontent.com/example/project/",
+                "https://github.com/example/project/blob/",
+                "https://github.com/example/project/tree/",
+            ):
+                with self.subTest(url=base + suffix), patch.object(AUDIT, "github_api", side_effect=github_api):
+                    report = AUDIT.audit_url((base + suffix, ["fixture.html"]))
+                    self.assertEqual(report["status"], "ok")
+                    self.assertEqual(report["endpoint"], endpoint)
+
+    def test_plain_refs_and_pinned_commits_keep_their_contents_target(self):
+        for ref in ("main", "a" * 40):
+            responses = {
+                "repos/example/project/git/matching-refs/heads/main": [{"ref": "refs/heads/main"}],
+                "repos/example/project/git/matching-refs/tags/main": [],
+                f"repos/example/project/contents/README.md?ref={ref}": {"name": "README.md"},
+            }
+            for base in ("https://raw.githubusercontent.com/example/project/", "https://github.com/example/project/blob/"):
+                with self.subTest(ref=ref, base=base), patch.object(AUDIT, "github_api", side_effect=responses.__getitem__):
+                    report = AUDIT.audit_url((base + ref + "/README.md", ["fixture.html"]))
+                    self.assertEqual(report["status"], "ok")
+                    self.assertEqual(report["endpoint"], f"repos/example/project/contents/README.md?ref={ref}")
+
+    def test_missing_contents_and_unavailable_github_probes_remain_distinct(self):
+        cases = (
+            (ValueError("gh: Not Found (HTTP 404)"), "broken"),
+            (ValueError("gh: Forbidden (HTTP 403)"), "unknown"),
+            (subprocess.TimeoutExpired(["gh", "api"], 20), "unknown"),
+        )
+        for failure, status in cases:
+            for base in ("https://raw.githubusercontent.com/example/project/", "https://github.com/example/project/blob/"):
+                with self.subTest(failure=str(failure), base=base), patch.object(AUDIT, "github_api", side_effect=failure):
+                    report = AUDIT.audit_url((base + "a" * 40 + "/missing.md", ["fixture.html"]))
+                    self.assertEqual(report["status"], status)
+                    self.assertEqual(report["reason"], str(failure))
 
 
 class WebsiteSearchChecks(unittest.TestCase):