Coverage for src/ai_jury/redaction.py: 96%
49 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-26 23:18 +0000
1"""Secret redaction for prompt text sent to external agents (issue #6).
3Deterministic: the same input always yields the same redacted output and count.
4Each match is replaced with ``[REDACTED:<kind>]``.
5"""
7from __future__ import annotations
9import re
10from urllib.parse import urlsplit, urlunsplit
12# An already-emitted redaction marker (`[REDACTED:<kind>]`). Used to avoid
13# re-redacting a value an earlier, more-specific pattern already replaced
14# (issue v1.5.0/L-3): without this guard a secret inside a basic-auth URL —
15# `https://user:AKIA…@host` — is redacted by `aws_access_key`, then the
16# resulting `[REDACTED:aws_access_key]` token is re-redacted by `basic_auth`,
17# losing the informative kind and double-counting.
18_REDACTED_MARKER = re.compile(r"^\[REDACTED:[a-z_]+\]$")
20# Ordered list of (kind, compiled pattern). Order matters: more specific
21# patterns run before the generic key=value catch-all so secrets are labeled
22# with the most informative kind.
23_PATTERNS: list[tuple[str, re.Pattern]] = [
24 (
25 "pem_private_key",
26 re.compile(
27 r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----"
28 r".*?-----END (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----",
29 re.DOTALL,
30 ),
31 ),
32 ("aws_access_key", re.compile(r"AKIA[0-9A-Z]{16}")),
33 ("github_token", re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}")),
34 # Classic `sk-…` AND modern project/service keys `sk-proj-…` /
35 # `sk-svcacct-…` / `sk-admin-…`, which embed hyphens the old `[A-Za-z0-9]`
36 # class stopped at (issue #122). First char after `sk-` is alphanumeric, then
37 # 18+ of alphanumeric / hyphen / underscore.
38 ("openai_key", re.compile(r"sk-[A-Za-z0-9][A-Za-z0-9_-]{18,}")),
39 ("bearer_token", re.compile(r"Bearer\s+[A-Za-z0-9._\-]+")),
40 # Common provider token formats (issue #290). Each runs BEFORE the generic
41 # `secret_assignment` catch-all so the value is replaced with its most
42 # informative kind and cannot be double-counted by the assignment pattern.
43 # Stripe keys use an underscore form (`sk_live_…`) distinct from the OpenAI
44 # `sk-…` hyphen pattern above; GitHub fine-grained PATs (`github_pat_…`) are
45 # not covered by the `gh[pousr]_` class.
46 ("slack_token", re.compile(r"xox[baprs]-[0-9A-Za-z-]{10,}")),
47 ("google_api_key", re.compile(r"AIza[0-9A-Za-z_\-]{35}")),
48 ("stripe_key", re.compile(r"(?:sk|rk|pk)_(?:live|test)_[0-9A-Za-z]{16,}")),
49 ("github_pat", re.compile(r"github_pat_[0-9A-Za-z_]{20,}")),
50 ("jwt", re.compile(r"eyJ[0-9A-Za-z_\-]+\.[0-9A-Za-z_\-]+\.[0-9A-Za-z_\-]+")),
51 # More provider formats (issue #316/L-2), each anchored on a distinctive
52 # prefix to avoid false positives. (Twilio Account SIDs and bare hex are
53 # deliberately NOT added — a SID is an identifier, and a broad hex rule would
54 # over-match commit SHAs.)
55 ("sendgrid_key", re.compile(r"SG\.[A-Za-z0-9_\-]{22}\.[A-Za-z0-9_\-]{43}")),
56 ("pypi_token", re.compile(r"pypi-[A-Za-z0-9_\-]{16,}")),
57 ("npm_token", re.compile(r"npm_[A-Za-z0-9]{36}")),
58 ("slack_webhook", re.compile(r"https://hooks\.slack\.com/services/[A-Za-z0-9/]+")),
59 # Basic-auth credentials in a URL (issue #302): `scheme://user:password@host`
60 # (e.g. `redis://default:s3cr3t…@cache:6379`). Only the password (group 2) is
61 # redacted; the `://user:` prefix (group 1) and `@` suffix (group 3) are kept
62 # so the URL stays readable. The trailing `@` requirement means a normal
63 # `host:port` (no `@`) never matches. Handled group-wise in `redact`.
64 #
65 # The username run is `*`, not `+` (review of #302): a token-as-password URL
66 # with an EMPTY username — `https://:SECRET@host` — is common, and `+` made
67 # the match fail there and leak the secret.
68 #
69 # The password run is `{1,}`, not `{6,}` (issue v1.5.0/L-1): a short password
70 # (`http://user:pass@host`, 4 chars) is still a leaked credential.
71 ("basic_auth", re.compile(r"(://[^/:@\s]*:)([^@\s]+)(@)")),
72 # Colon-less userinfo — a bare token in the userinfo position with no
73 # password colon (`http://apitoken12345@host/v1`) (issue v1.5.0/L-1). The
74 # colon form above never matches this (it requires a `:`), so the token would
75 # otherwise reach an external agent in cleartext. Group 1 (`://`) and group 3
76 # (`@`) are kept; group 2 (the token) is redacted. Disjoint from the colon
77 # form: `[^/:@\s]+` stops at a `:`, so a `user:pass@` URL is handled above.
78 #
79 # Note: this also redacts a bare, non-secret username (`https://user@host` ->
80 # `https://[REDACTED:basic_auth]@host`). That is deliberate safe-by-default
81 # over-redaction — userinfo in a reviewed diff is rare and a credential more
82 # often than not, and over-masking a username never leaks anything.
83 ("basic_auth", re.compile(r"(://)([^/:@\s]+)(@)")),
84 # Capture the surrounding quotes (groups 3 and 4) so they are PRESERVED in
85 # the replacement (issue #102): redacting only the value keeps a quoted
86 # assignment a valid string literal instead of producing a broken,
87 # unterminated string that misleads reviewers into phantom syntax findings.
88 #
89 # The key side (group 1) allows surrounding identifier chars so a keyword
90 # embedded mid-name is still recognized (issue #289): `aws_secret_access_key`
91 # matches via `secret`, where the old anchored `(secret)` required the `=`
92 # to follow `secret` directly and so leaked the canonical AWS variable name.
93 # `password`/`passwd` and a few more key names are included for the same
94 # reason. The surrounding identifier runs are BOUNDED (`{0,40}`), not `*`:
95 # a real secret variable name is short, and an unbounded run on both sides of
96 # the keyword makes the scan quadratic on a long word-char input with no
97 # separator (a ReDoS vector). The bound keeps it linear.
98 #
99 # `account[_-]?key` is included for Azure `AccountKey=…` / connection strings
100 # (issue #302). The separator group tolerates an optional closing quote
101 # (`["']?`) BEFORE the `=`/`:`, so a JSON-quoted key — `"private_key_id": "…"`
102 # in a GCP service-account blob — is matched too (its value is redacted while
103 # the structure stays valid). A non-secret like `client_email` is not caught:
104 # its value contains `@`/`.` outside the value char class.
105 (
106 "secret_assignment",
107 re.compile(
108 r"([A-Za-z0-9_]{0,40}(?:api[_-]?key|secret|token|password|passwd|"
109 r"access[_-]?key|account[_-]?key|private[_-]?key|client[_-]?secret|"
110 r"credential)"
111 r"[A-Za-z0-9_]{0,40})"
112 r"([\"']?\s*[=:]\s*)([\"']?)[A-Za-z0-9_\-+/=]{16,}([\"']?)",
113 re.IGNORECASE,
114 ),
115 ),
116]
119def redact(text: str) -> tuple[str, int]:
120 """Replace recognized secrets with ``[REDACTED:<kind>]``.
122 Returns ``(redacted_text, count)`` where count is the number of replacements.
123 """
124 if not text:
125 return text, 0
126 count = 0
127 result = text
128 for kind, pattern in _PATTERNS:
129 if kind == "secret_assignment":
131 def _sub_assign(m, _kind=kind):
132 nonlocal count
133 count += 1
134 # Preserve the key, separator, AND surrounding quotes; redact
135 # only the value so a quoted assignment stays syntactically valid.
136 return f"{m.group(1)}{m.group(2)}{m.group(3)}[REDACTED:{_kind}]{m.group(4)}"
138 result = pattern.sub(_sub_assign, result)
139 elif kind == "basic_auth":
141 def _sub_basic_auth(m, _kind=kind):
142 nonlocal count
143 # Don't re-redact a value an earlier pattern already replaced
144 # (issue v1.5.0/L-3): keep the more-informative kind and the
145 # accurate count.
146 if _REDACTED_MARKER.match(m.group(2)):
147 return m.group(0)
148 count += 1
149 # Keep the prefix (group 1: `://user:` or `://`) and `@` suffix
150 # (group 3); redact only the credential so the URL stays readable.
151 return f"{m.group(1)}[REDACTED:{_kind}]{m.group(3)}"
153 result = pattern.sub(_sub_basic_auth, result)
154 else:
156 def _sub(_m, _kind=kind):
157 nonlocal count
158 count += 1
159 return f"[REDACTED:{_kind}]"
161 result = pattern.sub(_sub, result)
162 return result, count
165def redact_url_userinfo(url: str) -> str:
166 """Strip any userinfo (``user[:password]``) from a URL before display.
168 Structural counterpart to :func:`redact` for the one place a base endpoint
169 URL can carry a credential (issue v1.5.0/L-1). Rather than relying on the
170 `basic_auth` regex — which can miss short or colon-less userinfo — this
171 parses the URL and replaces the whole ``userinfo@`` run with
172 ``[REDACTED]@``, keeping the scheme, host, port, path, and query verbatim
173 (so an IPv6 literal or non-default port is preserved exactly). A URL that
174 `urlsplit` cannot parse falls back to the regex-based :func:`redact`.
175 """
176 if not url:
177 return url
178 try:
179 parts = urlsplit(url)
180 except ValueError:
181 return redact(url)[0]
182 netloc = parts.netloc
183 if "@" in netloc:
184 # The host part has no unencoded '@'; split on the last one so a userinfo
185 # that contains a percent-encoded '@' is still handled correctly.
186 hostport = netloc[netloc.rfind("@") + 1 :]
187 return urlunsplit(parts._replace(netloc=f"[REDACTED]@{hostport}"))
188 # A scheme-less URL ("user:pass@host/p") leaves netloc empty — urlsplit dumps
189 # the authority into the scheme/path, so the userinfo would slip through
190 # unredacted (security audit 2026-06-13; continues v1.5.0/L-1). If the URL
191 # has no "://" authority but still carries an "@", re-parse with a synthetic
192 # "//" authority so netloc is populated, redact, then strip it back off.
193 if "@" in url and "://" not in url:
194 try:
195 reparsed = urlsplit("//" + url)
196 except ValueError:
197 return redact(url)[0]
198 if "@" in reparsed.netloc: 198 ↛ 202line 198 didn't jump to line 202 because the condition on line 198 was always true
199 hostport = reparsed.netloc[reparsed.netloc.rfind("@") + 1 :]
200 redacted = urlunsplit(reparsed._replace(netloc=f"[REDACTED]@{hostport}"))
201 return redacted[2:] if redacted.startswith("//") else redacted
202 return url