fix(cookie-inventory): fuzzy prefix-match + BMW-GT-File
BMW-Mail zeigte 738 deklariert / 31 Browser / **0 OK** — alle
Browser-Cookies landeten als UNDOC, alle deklarierten als ORPH.
Ursache: exact-string-match scheitert bei Suffix-Cookies.
_norm_for_match() + _matches() Helper:
- Strippt Wildcards (`*`, `.*`, `<id>`, `{var}`) + Lower-Case
- Erhält führende Underscores (`__cf_bm`, `_ga` sind meaningful)
- Prefix-Match in BEIDE Richtungen, min 3 Chars (kein "_"-Garbage)
build_cookie_inventory():
- Für jeden Browser-Cookie: längster Prefix-Match in declared wählen
- browser-to-decl Index + decl-match-Index für O(N×M) → O(N+M)
- matched browser-keys werden aus all_keys entfernt → kein
Double-Count (vorher: ORPH + UNDOC parallel)
Realistischer BMW-Match-Test:
declared=[_ga, _gid, __cf_bm, AMP_TOKEN, _fbp, intercom-session,
_pk_id.*, OptanonConsent]
browser= [_ga_K8YL3M9T, _gid_xyz, __cf_bm_actual_hash,
AMP_TOKEN_runtime, _fbp_123, intercom-session-2026,
_pk_id.5.7d8, OptanonConsent]
→ 8 OK (vorher 0)
BMW-GT-File (zeroclaw/docs/ground-truth/bmw_de_2026-06-07.json):
- OneTrust CMP + 14 erwartete Vendoren
- Cookie-Count-Ranges (browser 80-250, deklariert 300-800)
- 7 expected findings inkl. neuem COOKIE-INVENTORY-MATCH-001 als
Benchmark gegen den Fuzzy-Match-Bug
Tests: 14/14 grün (4 _norm_for_match + 5 _matches + 5
build_cookie_inventory inkl. realistic_bmw_pattern).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -40,6 +40,54 @@ def _norm(s: str | None) -> str:
|
||||
return (s or "").strip().lower()
|
||||
|
||||
|
||||
def _norm_for_match(s: str) -> str:
|
||||
"""Normalised name for fuzzy matching.
|
||||
|
||||
Common patterns in DSE-tables: wildcards (`_ga*`, `_ga.*`, `_pk_id.*`,
|
||||
`<random>`), trailing dots, brackets. Browser cookies often have
|
||||
a runtime suffix (`_ga_K8YL3M9T`, `__cf_bm_session_hash`). We strip
|
||||
trailing wildcards / suffix-noise so the prefix-match below works.
|
||||
|
||||
IMPORTANT: leading `_`/`__` are MEANINGFUL (`__cf_bm`, `_ga`) and
|
||||
must NOT be stripped.
|
||||
"""
|
||||
out = _norm(s)
|
||||
out = out.replace("*", "").replace("…", "")
|
||||
out = re.sub(r"\.\*$", "", out)
|
||||
out = re.sub(r"\.\$?$", "", out)
|
||||
out = re.sub(r"<[^>]+>", "", out)
|
||||
out = re.sub(r"\{[^}]+\}", "", out)
|
||||
return out.strip()
|
||||
|
||||
|
||||
def _matches(decl_key: str, browser_key: str) -> bool:
|
||||
"""Fuzzy match between a declared cookie name and a browser cookie.
|
||||
|
||||
Rules (in priority order):
|
||||
1. exact match after normalisation
|
||||
2. declared is a PREFIX of browser (declared "_ga" matches
|
||||
browser "_ga_k8yl3m9t")
|
||||
3. browser is a PREFIX of declared (rare: declared has a
|
||||
specific variant, browser only generic — e.g. declared
|
||||
"__cf_bm_session" with browser "__cf_bm")
|
||||
"""
|
||||
if not decl_key or not browser_key:
|
||||
return False
|
||||
if decl_key == browser_key:
|
||||
return True
|
||||
# Only allow prefix-match for prefixes ≥ 3 chars to avoid garbage
|
||||
# (e.g. declared "_" matching everything).
|
||||
if len(decl_key) >= 3 and browser_key.startswith(decl_key):
|
||||
return True
|
||||
if len(browser_key) >= 3 and decl_key.startswith(browser_key):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# Need re-import for the regex use above
|
||||
import re # noqa: E402
|
||||
|
||||
|
||||
def _missing(value: str | None) -> bool:
|
||||
if value is None:
|
||||
return True
|
||||
@@ -190,7 +238,30 @@ def build_cookie_inventory(state: dict) -> tuple[list[dict], dict]:
|
||||
for c in (cookie_audit.get("compliant") or [])
|
||||
}
|
||||
|
||||
all_keys = set(declared.keys()) | set(browser.keys())
|
||||
# Build fuzzy-match-Index: declared-key (normalised) → list of
|
||||
# browser-keys that match. Browser-key only matches ONE declared
|
||||
# entry (the longest prefix match wins) so we don't double-count.
|
||||
decl_match_index: dict[str, list[str]] = {k: [] for k in declared}
|
||||
browser_to_decl: dict[str, str] = {}
|
||||
for bkey in browser:
|
||||
bnorm = _norm_for_match(bkey)
|
||||
best = ""
|
||||
best_len = -1
|
||||
for dkey in declared:
|
||||
dnorm = _norm_for_match(dkey)
|
||||
if _matches(dnorm, bnorm) and len(dnorm) > best_len:
|
||||
best = dkey
|
||||
best_len = len(dnorm)
|
||||
if best:
|
||||
decl_match_index[best].append(bkey)
|
||||
browser_to_decl[bkey] = best
|
||||
|
||||
# all_keys = declared + browser, but browser-keys that fuzzy-match
|
||||
# an existing declared entry are FOLDED into the declared row
|
||||
# (avoid double-counting them as both ORPH and UNDOC).
|
||||
matched_browser_keys = set(browser_to_decl.keys())
|
||||
all_keys = (set(declared.keys())
|
||||
| (set(browser.keys()) - matched_browser_keys))
|
||||
rows: list[dict] = []
|
||||
for key in sorted(all_keys):
|
||||
d = declared.get(key) or {}
|
||||
@@ -200,7 +271,7 @@ def build_cookie_inventory(state: dict) -> tuple[list[dict], dict]:
|
||||
or b.get("domain") or "").strip() or ""
|
||||
country = d.get("country", "")
|
||||
country_display, is_third, adq = _country_third(country)
|
||||
in_browser = key in browser
|
||||
in_browser = (key in browser) or bool(decl_match_index.get(key))
|
||||
is_declared = key in declared
|
||||
status, sev = _build_status(
|
||||
is_declared, in_browser, undeclared_set, compliant_set, key,
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Tests for the Cookie-Inventory fuzzy-matcher.
|
||||
|
||||
Regression: BMW-Mail zeigte 0 OK obwohl 31 Browser-Cookies + 738
|
||||
deklarierte vorhanden waren. Ursache: exact-string-match scheitert
|
||||
bei `_ga` (declared) vs `_ga_K8YL3M9T` (browser).
|
||||
"""
|
||||
|
||||
from compliance.services.mail_render_v2._cookie_inventory import (
|
||||
_matches,
|
||||
_norm_for_match,
|
||||
build_cookie_inventory,
|
||||
)
|
||||
|
||||
|
||||
class TestNormForMatch:
|
||||
def test_strip_wildcard(self):
|
||||
assert _norm_for_match("_ga*") == "_ga"
|
||||
|
||||
def test_strip_regex_wildcard(self):
|
||||
assert _norm_for_match("_pk_id.*") == "_pk_id"
|
||||
|
||||
def test_strip_placeholder(self):
|
||||
assert _norm_for_match("session_<id>") == "session_"
|
||||
|
||||
def test_lowercase(self):
|
||||
assert _norm_for_match("__CF_BM") == "__cf_bm"
|
||||
|
||||
|
||||
class TestMatches:
|
||||
def test_exact(self):
|
||||
assert _matches("_ga", "_ga")
|
||||
|
||||
def test_declared_prefix_of_browser(self):
|
||||
# declared "_ga" matches browser "_ga_k8yl3m9t"
|
||||
assert _matches("_ga", "_ga_k8yl3m9t")
|
||||
|
||||
def test_browser_prefix_of_declared(self):
|
||||
# browser "__cf_bm" matches declared "__cf_bm_session"
|
||||
assert _matches("__cf_bm_session", "__cf_bm")
|
||||
|
||||
def test_short_prefix_rejected(self):
|
||||
# 2-char prefix would match too much
|
||||
assert not _matches("_g", "_ga_k8yl3m9t")
|
||||
|
||||
def test_unrelated(self):
|
||||
assert not _matches("_ga", "intercom-session")
|
||||
|
||||
|
||||
class TestBuildInventory:
|
||||
def _make_state(self, declared_cookies, browser_cookies):
|
||||
return {
|
||||
"cmp_vendors": [{
|
||||
"name": "Test", "country": "DE", "source": "dse",
|
||||
"cookies": [{"name": n} for n in declared_cookies],
|
||||
}],
|
||||
"banner_result": {
|
||||
"cookies_detailed": [{"name": n} for n in browser_cookies],
|
||||
},
|
||||
"cookie_audit": {},
|
||||
}
|
||||
|
||||
def test_no_match_no_ok(self):
|
||||
rows, summary = build_cookie_inventory(
|
||||
self._make_state(["foo"], ["bar"]),
|
||||
)
|
||||
assert summary["ok"] == 0
|
||||
assert summary["orph"] == 1
|
||||
assert summary["undoc"] == 1
|
||||
|
||||
def test_exact_match_yields_ok(self):
|
||||
rows, summary = build_cookie_inventory(
|
||||
self._make_state(["_ga"], ["_ga"]),
|
||||
)
|
||||
assert summary["ok"] == 1
|
||||
assert summary["orph"] == 0
|
||||
assert summary["undoc"] == 0
|
||||
|
||||
def test_prefix_match_yields_ok_no_double_count(self):
|
||||
# Realistic BMW case: declared "_ga", browser "_ga_K8YL3M9T"
|
||||
rows, summary = build_cookie_inventory(
|
||||
self._make_state(["_ga"], ["_ga_K8YL3M9T"]),
|
||||
)
|
||||
assert summary["ok"] == 1, "fuzzy prefix-match should land in OK"
|
||||
assert summary["orph"] == 0, "declared must not double-count as ORPH"
|
||||
assert summary["undoc"] == 0, (
|
||||
"browser cookie must fold into declared row, not appear separately"
|
||||
)
|
||||
|
||||
def test_wildcard_match(self):
|
||||
rows, summary = build_cookie_inventory(
|
||||
self._make_state(["_pk_id.*"], ["_pk_id.5"]),
|
||||
)
|
||||
assert summary["ok"] == 1
|
||||
|
||||
def test_realistic_bmw_pattern(self):
|
||||
# Declared: long list with common cookies
|
||||
decl = ["_ga", "_gid", "__cf_bm", "AMP_TOKEN", "_fbp",
|
||||
"intercom-session", "_pk_id.*", "OptanonConsent"]
|
||||
# Browser: actual cookies with runtime suffixes
|
||||
bro = ["_ga_K8YL3M9T", "_gid_xyz", "__cf_bm_actual_hash",
|
||||
"AMP_TOKEN_runtime", "_fbp_123",
|
||||
"intercom-session-2026", "_pk_id.5.7d8", "OptanonConsent"]
|
||||
rows, summary = build_cookie_inventory(
|
||||
self._make_state(decl, bro),
|
||||
)
|
||||
# All 8 browser cookies should fold into the 8 declared rows.
|
||||
assert summary["ok"] == 8, f"expected 8 OK, got {summary}"
|
||||
assert summary["orph"] == 0
|
||||
assert summary["undoc"] == 0
|
||||
Reference in New Issue
Block a user