/
/
/
1"""Tests for scoping pip-audit findings to the vulnerabilities a pull request introduces."""
2
3from __future__ import annotations
4
5import json
6from pathlib import Path
7
8import pytest
9
10from scripts.audit_changed_packages import (
11 audit_status,
12 changed_packages,
13 introduced_packages,
14 main,
15 normalize,
16 requirement_versions,
17 vulnerable_packages,
18)
19
20# Resolved dependency set of the target branch, as the audit workflow writes it
21BASE_CLOSURE = """
22aiohttp==3.14.1
23clean-package==1.0.0
24"""
25
26
27def audit_report(*vulnerable: str, skipped: str | None = None) -> str:
28 """
29 Return a pip-audit JSON report where the given packages carry a vulnerability.
30
31 :param vulnerable: Packages to report a vulnerability for, as `name` or `name==version`.
32 :param skipped: Package name pip-audit could not audit, if any.
33 """
34 dependencies = []
35 for entry in vulnerable:
36 name, _, version = entry.partition("==")
37 dependencies.append(
38 {
39 "name": name,
40 "version": version or "1.0.0",
41 "vulns": [{"id": "GHSA-test", "fix_versions": []}],
42 }
43 )
44 dependencies.append({"name": "clean-package", "version": "1.0.0", "vulns": []})
45 if skipped:
46 dependencies.append({"name": skipped, "skip_reason": "not found on PyPI"})
47 return json.dumps({"dependencies": dependencies, "fixes": []})
48
49
50@pytest.mark.parametrize(
51 ("name", "expected"),
52 [
53 ("aiohttp", "aiohttp"),
54 ("Zeroconf", "zeroconf"),
55 ("aiohttp_fast_zlib", "aiohttp-fast-zlib"),
56 ("zope.interface", "zope-interface"),
57 ("a__b", "a-b"),
58 ],
59)
60def test_normalize(name: str, expected: str) -> None:
61 """Package names are compared in their PEP 503 normalized form."""
62 assert normalize(name) == expected
63
64
65def test_changed_packages_reads_requirement_lines() -> None:
66 """Package names are extracted regardless of the version specifier used."""
67 requirements = """
68 aiohttp==3.14.1
69 music-assistant-models>=1.2
70 some_pkg[extra]~=2.0 ; python_version >= '3.14'
71 aiolibdatachannel @ git+https://github.com/example/aiolibdatachannel@v1
72
73 # a comment
74 --index-url https://example.org/simple
75 """
76 assert changed_packages(requirements) == {
77 "aiohttp",
78 "music-assistant-models",
79 "some-pkg",
80 "aiolibdatachannel",
81 }
82
83
84def test_requirement_versions_reads_pinned_versions() -> None:
85 """A resolved dependency set pins one version per package, except for URL requirements."""
86 closure = """
87 # This file was autogenerated by uv
88 --extra-index-url https://example.org/simple
89 aiohttp==3.14.1
90 some_pkg[extra]==2.0
91 torch==2.13.0+cpu ; sys_platform == 'linux'
92 aiolibdatachannel @ git+https://github.com/example/aiolibdatachannel@v1
93 """
94 assert requirement_versions(closure) == {
95 "aiohttp": {"3.14.1"},
96 "some-pkg": {"2.0"},
97 "torch": {"2.13.0+cpu"},
98 "aiolibdatachannel": set(),
99 }
100
101
102def test_requirement_versions_collects_every_pinned_version() -> None:
103 """A package can appear more than once, as the marker-split torch requirements do."""
104 closure = """
105 torch==2.13.0+cpu ; sys_platform == 'linux' and platform_machine == 'x86_64'
106 torch==2.13.0 ; sys_platform != 'linux' or platform_machine != 'x86_64'
107 """
108 assert requirement_versions(closure) == {"torch": {"2.13.0+cpu", "2.13.0"}}
109
110
111def test_vulnerable_packages_ignores_clean_and_skipped() -> None:
112 """Only packages with findings count; skipped ones carry no vulns key."""
113 report = audit_report("aiohttp==3.14.3", skipped="torch")
114 assert vulnerable_packages(report) == {("aiohttp", "3.14.3")}
115
116
117def test_introduced_packages_compares_name_and_version() -> None:
118 """A finding is introduced unless the target branch resolves to that exact version."""
119 resolved = {"aiohttp": {"3.14.1"}, "aiolibdatachannel": set()}
120 findings = {
121 ("aiohttp", "3.14.1"),
122 ("aiohttp", "3.14.3"),
123 ("transformers", "5.14.1"),
124 ("aiolibdatachannel", "0.1.0"),
125 }
126 assert introduced_packages(findings, resolved) == {
127 ("aiohttp", "3.14.3"),
128 ("transformers", "5.14.1"),
129 }
130
131
132def test_introduced_packages_reads_a_url_pin_from_the_changed_requirements() -> None:
133 """A package the target branch pins by URL is only introduced when its requirement changes."""
134 findings = {("aiolibdatachannel", "0.1.0")}
135 resolved: dict[str, set[str]] = {"aiolibdatachannel": set()}
136
137 assert introduced_packages(findings, resolved) == set()
138 assert introduced_packages(findings, resolved, {"aiolibdatachannel"}) == findings
139
140
141def test_no_findings_passes() -> None:
142 """A clean audit passes even when the pull request changes dependencies."""
143 assert audit_status(audit_report(), "aiohttp==3.14.1", BASE_CLOSURE) == "pass"
144
145
146def test_findings_the_target_branch_has_too_are_preexisting() -> None:
147 """Vulnerabilities the target branch already installs describe it, not the pull request."""
148 assert audit_status(audit_report("aiohttp==3.14.1"), "aiohttp==3.14.1", BASE_CLOSURE) == (
149 "preexisting"
150 )
151
152
153def test_findings_in_a_bumped_package_fail() -> None:
154 """A version the target branch does not install is one this pull request brings in."""
155 assert audit_status(audit_report("aiohttp==3.14.3"), "aiohttp==3.14.3", BASE_CLOSURE) == "fail"
156
157
158def test_findings_in_a_new_transitive_package_fail() -> None:
159 """A vulnerability pulled in indirectly gates too, even with no direct requirement for it."""
160 assert audit_status(audit_report("transformers==5.14.1"), "beat-this==1.1.0", BASE_CLOSURE) == (
161 "fail"
162 )
163
164
165def test_swapping_a_url_pin_for_a_vulnerable_release_fails() -> None:
166 """Replacing a URL requirement with a published version brings that version in."""
167 base_closure = "aiolibdatachannel @ git+https://github.com/example/aiolibdatachannel@v1"
168 report = audit_report("aiolibdatachannel==0.1.0")
169
170 assert audit_status(report, "aiolibdatachannel==0.1.0", base_closure) == "fail"
171 assert audit_status(report, "", base_closure) == "preexisting"
172
173
174def test_an_unreadable_target_branch_set_raises() -> None:
175 """A resolved set no package can be read from must fail loudly rather than gate everything."""
176 with pytest.raises(ValueError, match="No packages"):
177 audit_status(audit_report("aiohttp"), "", '{"packages": [{"name": "aiohttp"}]}')
178
179
180def test_findings_without_a_target_branch_set_compare_changed_requirements() -> None:
181 """Resolving the target branch can fail; the changed requirements remain a usable basis."""
182 assert audit_status(audit_report("aiohttp"), "some-pkg==1.0") == "preexisting"
183 assert audit_status(audit_report("aiohttp"), "aiohttp==3.14.1") == "fail"
184
185
186def test_findings_without_dependency_changes_are_preexisting() -> None:
187 """A pull request that changes no dependency can never introduce a vulnerability."""
188 assert audit_status(audit_report("aiohttp"), "") == "preexisting"
189
190
191def test_findings_match_across_name_spellings() -> None:
192 """The comparison survives the underscore/dash spellings both sides use."""
193 assert audit_status(audit_report("aiohttp_fast_zlib"), "aiohttp-fast-zlib==0.3.0") == "fail"
194
195
196def test_main_prints_status(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
197 """The status is printed for the reporting workflow to pick up."""
198 audit = tmp_path / "audit.json"
199 audit.write_text(audit_report("aiohttp==3.14.3"))
200 requirements = tmp_path / "new_deps.txt"
201 requirements.write_text("aiohttp==3.14.3\n")
202 base_closure = tmp_path / "base_closure.txt"
203 base_closure.write_text(BASE_CLOSURE)
204
205 assert main([str(audit), str(requirements), str(base_closure)]) == 0
206 assert capsys.readouterr().out.strip() == "fail"
207
208
209def test_main_without_a_base_closure_file(
210 tmp_path: Path, capsys: pytest.CaptureFixture[str]
211) -> None:
212 """The resolved set is absent when the target branch could not be resolved."""
213 audit = tmp_path / "audit.json"
214 audit.write_text(audit_report("transformers==5.14.1"))
215 requirements = tmp_path / "new_deps.txt"
216 requirements.write_text("beat-this==1.1.0\n")
217
218 assert main([str(audit), str(requirements), str(tmp_path / "missing.txt")]) == 0
219 assert capsys.readouterr().out.strip() == "preexisting"
220
221
222def test_main_with_an_empty_base_closure_file(
223 tmp_path: Path, capsys: pytest.CaptureFixture[str]
224) -> None:
225 """An empty resolved set carries no comparison and must not raise."""
226 audit = tmp_path / "audit.json"
227 audit.write_text(audit_report("transformers==5.14.1"))
228 base_closure = tmp_path / "base_closure.txt"
229 base_closure.write_text("")
230
231 assert main([str(audit), str(tmp_path / "missing.txt"), str(base_closure)]) == 0
232 assert capsys.readouterr().out.strip() == "preexisting"
233
234
235def test_main_without_a_requirements_file(
236 tmp_path: Path, capsys: pytest.CaptureFixture[str]
237) -> None:
238 """The requirements file is absent when the analysis found no dependency changes."""
239 audit = tmp_path / "audit.json"
240 audit.write_text(audit_report("aiohttp"))
241
242 assert main([str(audit), str(tmp_path / "missing.txt")]) == 0
243 assert capsys.readouterr().out.strip() == "preexisting"
244
245
246@pytest.mark.parametrize(
247 ("report", "expected"),
248 [
249 ("not json at all", json.JSONDecodeError),
250 # A pip-audit release that renames the key must not read as "no findings"
251 ('{"deps": [], "fixes": []}', KeyError),
252 ],
253)
254def test_an_unusable_report_raises(tmp_path: Path, report: str, expected: type[Exception]) -> None:
255 """An unreadable audit must fail loudly; the workflow gates on a non-zero exit."""
256 audit = tmp_path / "audit.json"
257 audit.write_text(report)
258
259 with pytest.raises(expected):
260 main([str(audit), str(tmp_path / "missing.txt")])
261
262
263def test_a_missing_report_raises(tmp_path: Path) -> None:
264 """pip-audit writing no report at all must fail loudly rather than pass."""
265 with pytest.raises(FileNotFoundError):
266 main([str(tmp_path / "missing.json"), str(tmp_path / "missing.txt")])
267