/
/
/
1"""Tests for scoping pip-audit findings to the vulnerabilities a pull request introduces."""
2
3from __future__ import annotations
4
5import json
6from pathlib import Path
7
8import pytest
9
10from scripts.audit_changed_packages import (
11 audit_status,
12 changed_packages,
13 introduced_packages,
14 main,
15 normalize,
16 requirement_versions,
17 resolved_findings,
18 vulnerable_packages,
19)
20
21# Resolved dependency set of the target branch, as the audit workflow writes it
22BASE_CLOSURE = """
23aiohttp==3.14.1
24clean-package==1.0.0
25"""
26
27
28def audit_report(*vulnerable: str, skipped: str | None = None) -> str:
29 """
30 Return a pip-audit JSON report where the given packages carry a vulnerability.
31
32 :param vulnerable: Packages to report a vulnerability for, as `name` or `name==version`.
33 :param skipped: Package name pip-audit could not audit, if any.
34 """
35 dependencies = []
36 for entry in vulnerable:
37 name, _, version = entry.partition("==")
38 dependencies.append(
39 {
40 "name": name,
41 "version": version or "1.0.0",
42 "vulns": [{"id": "GHSA-test", "fix_versions": []}],
43 }
44 )
45 dependencies.append({"name": "clean-package", "version": "1.0.0", "vulns": []})
46 if skipped:
47 dependencies.append({"name": skipped, "skip_reason": "not found on PyPI"})
48 return json.dumps({"dependencies": dependencies, "fixes": []})
49
50
51@pytest.mark.parametrize(
52 ("name", "expected"),
53 [
54 ("aiohttp", "aiohttp"),
55 ("Zeroconf", "zeroconf"),
56 ("aiohttp_fast_zlib", "aiohttp-fast-zlib"),
57 ("zope.interface", "zope-interface"),
58 ("a__b", "a-b"),
59 ],
60)
61def test_normalize(name: str, expected: str) -> None:
62 """Package names are compared in their PEP 503 normalized form."""
63 assert normalize(name) == expected
64
65
66def test_changed_packages_reads_requirement_lines() -> None:
67 """Package names are extracted regardless of the version specifier used."""
68 requirements = """
69 aiohttp==3.14.1
70 music-assistant-models>=1.2
71 some_pkg[extra]~=2.0 ; python_version >= '3.14'
72 aiolibdatachannel @ git+https://github.com/example/aiolibdatachannel@v1
73
74 # a comment
75 --index-url https://example.org/simple
76 """
77 assert changed_packages(requirements) == {
78 "aiohttp",
79 "music-assistant-models",
80 "some-pkg",
81 "aiolibdatachannel",
82 }
83
84
85def test_requirement_versions_reads_pinned_versions() -> None:
86 """A resolved dependency set pins one version per package, except for URL requirements."""
87 closure = """
88 # This file was autogenerated by uv
89 --extra-index-url https://example.org/simple
90 aiohttp==3.14.1
91 some_pkg[extra]==2.0
92 torch==2.13.0+cpu ; sys_platform == 'linux'
93 aiolibdatachannel @ git+https://github.com/example/aiolibdatachannel@v1
94 """
95 assert requirement_versions(closure) == {
96 "aiohttp": {"3.14.1"},
97 "some-pkg": {"2.0"},
98 "torch": {"2.13.0+cpu"},
99 "aiolibdatachannel": set(),
100 }
101
102
103def test_requirement_versions_collects_every_pinned_version() -> None:
104 """A package can appear more than once, as the marker-split torch requirements do."""
105 closure = """
106 torch==2.13.0+cpu ; sys_platform == 'linux' and platform_machine == 'x86_64'
107 torch==2.13.0 ; sys_platform != 'linux' or platform_machine != 'x86_64'
108 """
109 assert requirement_versions(closure) == {"torch": {"2.13.0+cpu", "2.13.0"}}
110
111
112def test_vulnerable_packages_ignores_clean_and_skipped() -> None:
113 """Only packages with findings count; skipped ones carry no vulns key."""
114 report = audit_report("aiohttp==3.14.3", skipped="torch")
115 assert vulnerable_packages(report) == {("aiohttp", "3.14.3")}
116
117
118def test_introduced_packages_compares_name_and_version() -> None:
119 """A finding is introduced unless the target branch resolves to that exact version."""
120 resolved = {"aiohttp": {"3.14.1"}, "aiolibdatachannel": set()}
121 findings = {
122 ("aiohttp", "3.14.1"),
123 ("aiohttp", "3.14.3"),
124 ("transformers", "5.14.1"),
125 ("aiolibdatachannel", "0.1.0"),
126 }
127 assert introduced_packages(findings, resolved) == {
128 ("aiohttp", "3.14.3"),
129 ("transformers", "5.14.1"),
130 }
131
132
133def test_introduced_packages_reads_a_url_pin_from_the_changed_requirements() -> None:
134 """A package the target branch pins by URL is only introduced when its requirement changes."""
135 findings = {("aiolibdatachannel", "0.1.0")}
136 resolved: dict[str, set[str]] = {"aiolibdatachannel": set()}
137
138 assert introduced_packages(findings, resolved) == set()
139 assert introduced_packages(findings, resolved, {"aiolibdatachannel"}) == findings
140
141
142def test_resolved_findings_keeps_what_the_set_installs() -> None:
143 """Only findings the resolved set carries count; a URL requirement matches on name alone."""
144 resolved = {"aiohttp": {"3.14.1"}, "aiolibdatachannel": set()}
145 findings = {
146 ("aiohttp", "3.14.1"),
147 ("aiohttp", "3.14.3"),
148 ("transformers", "5.14.1"),
149 ("aiolibdatachannel", "0.1.0"),
150 }
151 assert resolved_findings(findings, resolved) == {
152 ("aiohttp", "3.14.1"),
153 ("aiolibdatachannel", "0.1.0"),
154 }
155
156
157def test_no_findings_passes() -> None:
158 """A clean audit passes even when the pull request changes dependencies."""
159 assert audit_status(audit_report(), "aiohttp==3.14.1", BASE_CLOSURE) == "pass"
160
161
162def test_findings_the_target_branch_has_too_are_preexisting() -> None:
163 """Vulnerabilities the target branch already installs describe it, not the pull request."""
164 report = audit_report("aiohttp==3.14.1")
165
166 assert audit_status(report, "aiohttp==3.14.1", BASE_CLOSURE) == "preexisting"
167 assert audit_status(report, "aiohttp==3.14.1", BASE_CLOSURE, BASE_CLOSURE) == "preexisting"
168
169
170def test_findings_in_a_bumped_package_fail() -> None:
171 """A version the target branch does not install is one this pull request brings in."""
172 assert audit_status(audit_report("aiohttp==3.14.3"), "aiohttp==3.14.3", BASE_CLOSURE) == "fail"
173
174
175def test_findings_in_a_new_transitive_package_fail() -> None:
176 """A vulnerability pulled in indirectly gates too, even with no direct requirement for it."""
177 assert audit_status(audit_report("transformers==5.14.1"), "beat-this==1.1.0", BASE_CLOSURE) == (
178 "fail"
179 )
180
181
182def test_a_release_published_during_the_run_is_not_gated() -> None:
183 """The environment is installed before either branch is resolved; that skew must not gate."""
184 # Both branches resolve to the release that landed after the environment was installed
185 closure = "aiohttp==3.14.3\nclean-package==1.0.0\n"
186 report = audit_report("aiohttp==3.14.1")
187 changed = "music-assistant-models==1.1.183"
188
189 assert audit_status(report, changed, closure) == "fail"
190 assert audit_status(report, changed, closure, closure) == "preexisting"
191
192
193def test_findings_the_pull_request_resolves_to_still_fail() -> None:
194 """A bumped version the pull request's own resolution carries is genuinely introduced."""
195 head_closure = "aiohttp==3.14.3\nclean-package==1.0.0\n"
196 report = audit_report("aiohttp==3.14.3")
197
198 assert audit_status(report, "aiohttp==3.14.3", BASE_CLOSURE, head_closure) == "fail"
199
200
201def test_swapping_a_url_pin_for_a_vulnerable_release_fails() -> None:
202 """Replacing a URL requirement with a published version brings that version in."""
203 base_closure = "aiolibdatachannel @ git+https://github.com/example/aiolibdatachannel@v1"
204 report = audit_report("aiolibdatachannel==0.1.0")
205
206 assert audit_status(report, "aiolibdatachannel==0.1.0", base_closure) == "fail"
207 assert audit_status(report, "", base_closure) == "preexisting"
208
209
210def test_a_changed_url_pin_fails_against_both_resolutions() -> None:
211 """A URL requirement pins no version, so the pull request's resolution matches on name."""
212 base_closure = "aiolibdatachannel @ git+https://github.com/example/aiolibdatachannel@v1"
213 head_closure = "aiolibdatachannel @ git+https://github.com/example/aiolibdatachannel@v2"
214 report = audit_report("aiolibdatachannel==0.1.0")
215
216 assert audit_status(report, head_closure, base_closure, head_closure) == "fail"
217
218
219def test_an_unreadable_target_branch_set_raises() -> None:
220 """A resolved set no package can be read from must fail loudly rather than gate everything."""
221 with pytest.raises(ValueError, match="No packages"):
222 audit_status(audit_report("aiohttp"), "", '{"packages": [{"name": "aiohttp"}]}')
223
224
225def test_an_unreadable_pull_request_set_raises() -> None:
226 """The same holds the other way around: it would clear every finding instead."""
227 with pytest.raises(ValueError, match="No packages"):
228 audit_status(
229 audit_report("aiohttp==3.14.3"),
230 "aiohttp==3.14.3",
231 BASE_CLOSURE,
232 '{"packages": [{"name": "aiohttp"}]}',
233 )
234
235
236def test_findings_without_a_target_branch_set_compare_changed_requirements() -> None:
237 """Resolving the target branch can fail; the changed requirements remain a usable basis."""
238 assert audit_status(audit_report("aiohttp"), "some-pkg==1.0") == "preexisting"
239 assert audit_status(audit_report("aiohttp"), "aiohttp==3.14.1") == "fail"
240
241
242def test_findings_without_dependency_changes_are_preexisting() -> None:
243 """A pull request that changes no dependency can never introduce a vulnerability."""
244 assert audit_status(audit_report("aiohttp"), "") == "preexisting"
245
246
247def test_findings_match_across_name_spellings() -> None:
248 """The comparison survives the underscore/dash spellings both sides use."""
249 assert audit_status(audit_report("aiohttp_fast_zlib"), "aiohttp-fast-zlib==0.3.0") == "fail"
250
251
252def test_main_prints_status(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
253 """The status is printed for the reporting workflow to pick up."""
254 audit = tmp_path / "audit.json"
255 audit.write_text(audit_report("aiohttp==3.14.3"))
256 requirements = tmp_path / "new_deps.txt"
257 requirements.write_text("aiohttp==3.14.3\n")
258 base_closure = tmp_path / "base_closure.txt"
259 base_closure.write_text(BASE_CLOSURE)
260
261 assert main([str(audit), str(requirements), str(base_closure)]) == 0
262 assert capsys.readouterr().out.strip() == "fail"
263
264
265def test_main_reads_the_resolutions_in_order(
266 tmp_path: Path, capsys: pytest.CaptureFixture[str]
267) -> None:
268 """The target branch is the third argument, the pull request's own set the fourth."""
269 audit = tmp_path / "audit.json"
270 audit.write_text(audit_report("aiohttp==3.14.1"))
271 requirements = tmp_path / "new_deps.txt"
272 requirements.write_text("music-assistant-models==1.1.183\n")
273 without = tmp_path / "without.txt"
274 without.write_text("aiohttp==3.14.3\n")
275 carrying = tmp_path / "carrying.txt"
276 carrying.write_text("aiohttp==3.14.1\n")
277
278 assert main([str(audit), str(requirements), str(without), str(carrying)]) == 0
279 assert capsys.readouterr().out.strip() == "fail"
280
281 assert main([str(audit), str(requirements), str(carrying), str(without)]) == 0
282 assert capsys.readouterr().out.strip() == "preexisting"
283
284
285def test_main_without_a_base_closure_file(
286 tmp_path: Path, capsys: pytest.CaptureFixture[str]
287) -> None:
288 """The resolved set is absent when the target branch could not be resolved."""
289 audit = tmp_path / "audit.json"
290 audit.write_text(audit_report("transformers==5.14.1"))
291 requirements = tmp_path / "new_deps.txt"
292 requirements.write_text("beat-this==1.1.0\n")
293
294 assert main([str(audit), str(requirements), str(tmp_path / "missing.txt")]) == 0
295 assert capsys.readouterr().out.strip() == "preexisting"
296
297
298def test_main_with_an_empty_base_closure_file(
299 tmp_path: Path, capsys: pytest.CaptureFixture[str]
300) -> None:
301 """An empty resolved set carries no comparison and must not raise."""
302 audit = tmp_path / "audit.json"
303 audit.write_text(audit_report("transformers==5.14.1"))
304 base_closure = tmp_path / "base_closure.txt"
305 base_closure.write_text("")
306
307 assert main([str(audit), str(tmp_path / "missing.txt"), str(base_closure)]) == 0
308 assert capsys.readouterr().out.strip() == "preexisting"
309
310
311def test_main_without_a_requirements_file(
312 tmp_path: Path, capsys: pytest.CaptureFixture[str]
313) -> None:
314 """The requirements file is absent when the analysis found no dependency changes."""
315 audit = tmp_path / "audit.json"
316 audit.write_text(audit_report("aiohttp"))
317
318 assert main([str(audit), str(tmp_path / "missing.txt")]) == 0
319 assert capsys.readouterr().out.strip() == "preexisting"
320
321
322@pytest.mark.parametrize(
323 ("report", "expected"),
324 [
325 ("not json at all", json.JSONDecodeError),
326 # A pip-audit release that renames the key must not read as "no findings"
327 ('{"deps": [], "fixes": []}', KeyError),
328 ],
329)
330def test_an_unusable_report_raises(tmp_path: Path, report: str, expected: type[Exception]) -> None:
331 """An unreadable audit must fail loudly; the workflow gates on a non-zero exit."""
332 audit = tmp_path / "audit.json"
333 audit.write_text(report)
334
335 with pytest.raises(expected):
336 main([str(audit), str(tmp_path / "missing.txt")])
337
338
339def test_a_missing_report_raises(tmp_path: Path) -> None:
340 """pip-audit writing no report at all must fail loudly rather than pass."""
341 with pytest.raises(FileNotFoundError):
342 main([str(tmp_path / "missing.json"), str(tmp_path / "missing.txt")])
343