Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

requirement, test: Remove preresolved dependency optimization #540

Merged
merged 6 commits into from
Mar 23, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 1 addition & 8 deletions pip_audit/_dependency_source/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,7 @@
Dependency source interfaces and implementations for `pip-audit`.
"""

from .interface import (
PYPI_URL,
DependencyFixError,
DependencySource,
DependencySourceError,
InvalidRequirementSpecifier,
)
from .interface import PYPI_URL, DependencyFixError, DependencySource, DependencySourceError
from .pip import PipSource, PipSourceError
from .pyproject import PyProjectSource
from .requirement import RequirementSource
Expand All @@ -18,7 +12,6 @@
"DependencyFixError",
"DependencySource",
"DependencySourceError",
"InvalidRequirementSpecifier",
"PipSource",
"PipSourceError",
"PyProjectSource",
Expand Down
7 changes: 0 additions & 7 deletions pip_audit/_dependency_source/interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,3 @@ class DependencyFixError(Exception):
"""

pass


class InvalidRequirementSpecifier(DependencySourceError):
"""
A `DependencySourceError` specialized for the case of a non-PEP 440 requirements
specifier.
"""
86 changes: 5 additions & 81 deletions pip_audit/_dependency_source/requirement.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,17 @@
from typing import IO, Iterator

from packaging.specifiers import SpecifierSet
from packaging.version import Version
from pip_requirements_parser import InstallRequirement, InvalidRequirementLine, RequirementsFile

from pip_audit._dependency_source import (
PYPI_URL,
DependencyFixError,
DependencySource,
DependencySourceError,
InvalidRequirementSpecifier,
)
from pip_audit._fix import ResolvedFixVersion
from pip_audit._service import Dependency
from pip_audit._service.interface import ResolvedDependency, SkippedDependency
from pip_audit._service.interface import ResolvedDependency
from pip_audit._state import AuditState
from pip_audit._virtual_env import VirtualEnv, VirtualEnvError

Expand Down Expand Up @@ -84,30 +82,11 @@ def collect(self) -> Iterator[Dependency]:
Raises a `RequirementSourceError` on any errors.
"""

# Figure out whether we have a fully resolved set of dependencies.
reqs: list[InstallRequirement] = []
require_hashes: bool = self._require_hashes
for filename in self._filenames:
rf = RequirementsFile.from_file(filename)
if len(rf.invalid_lines) > 0:
invalid = rf.invalid_lines[0]
raise InvalidRequirementSpecifier(
f"requirement file {filename} contains invalid specifier at "
f"line {invalid.line_number}: {invalid.error_message}"
)

# If one or more requirements have a hash, this implies `--require-hashes`.
require_hashes = require_hashes or any(req.hash_options for req in rf.requirements)
reqs.extend(rf.requirements)

# If the user has supplied `--no-deps` or there are hashed requirements, we should assume
# that we have a fully resolved set of dependencies and we should waste time by invoking
# `pip`.
if self._no_deps or require_hashes:
yield from self._collect_preresolved_deps(iter(reqs), require_hashes)
return

ve_args = []
if self._no_deps:
ve_args.append("--no-deps")
if self._require_hashes:
ve_args.append("--require-hashes")
for filename in self._filenames:
ve_args.extend(["-r", str(filename)])

Expand Down Expand Up @@ -223,61 +202,6 @@ def _recover_files(self, tmp_files: list[IO[str]]) -> None:
logger.warning(f"encountered an exception during file recovery: {e}")
continue

def _collect_preresolved_deps(
self, reqs: Iterator[InstallRequirement], require_hashes: bool
) -> Iterator[Dependency]:
"""
Collect pre-resolved (pinned) dependencies.
"""
req_names: set[str] = set()
for req in reqs:
if not req.hash_options and require_hashes:
raise RequirementSourceError(f"requirement {req.dumps()} does not contain a hash")
if req.req is None:
# PEP 508-style URL requirements don't have a pre-declared version, even
# when hashed; the `#egg=name==version` syntax is non-standard and not supported
# by `pip` itself.
#
# In this case, we can't audit the dependency so we should signal to the
# caller that we're skipping it.
yield SkippedDependency(
name=req.requirement_line.line,
skip_reason="could not deduce package version from URL requirement",
)
continue
if self._skip_editable and req.is_editable:
yield SkippedDependency(name=req.name, skip_reason="requirement marked as editable")
if req.marker is not None and not req.marker.evaluate():
continue # pragma: no cover

# This means we have a duplicate requirement for the same package
if req.name in req_names:
raise RequirementSourceError(
f"package {req.name} has duplicate requirements: {str(req)}"
)
req_names.add(req.name)

# NOTE: URL dependencies cannot be pinned, so skipping them
# makes sense (under the same principle of skipping dependencies
# that can't be found on PyPI). This is also consistent with
# what `pip --no-deps` does (installs the URL dependency, but
# not any subdependencies).
if req.is_url:
yield SkippedDependency(
name=req.name,
skip_reason="URL requirements cannot be pinned to a specific package version",
)
elif not req.specifier:
raise RequirementSourceError(f"requirement {req.name} is not pinned: {str(req)}")
else:
pinned_specifier = PINNED_SPECIFIER_RE.match(str(req.specifier))
if pinned_specifier is None:
raise RequirementSourceError(
f"requirement {req.name} is not pinned to an exact version: {str(req)}"
)

yield ResolvedDependency(req.name, Version(pinned_specifier.group("version")))


class RequirementSourceError(DependencySourceError):
"""A requirements-parsing specific `DependencySourceError`."""
Expand Down
Loading