diff --git a/CHANGES.md b/CHANGES.md index 0f7c4d36a..767fe1223 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,7 @@ - Fix config file value interpolation for the `diff-file` option `PR #1715` - Fix diff-file being created when the option wasn't set `PR #1716` +- Provide default values for most config options in the `[mirror]` section `PR #1740` # 6.5.0 diff --git a/src/bandersnatch/config/exceptions.py b/src/bandersnatch/config/exceptions.py new file mode 100644 index 000000000..796c6e827 --- /dev/null +++ b/src/bandersnatch/config/exceptions.py @@ -0,0 +1,13 @@ +"""Exception subclasses for configuration file loading and validation.""" + + +class ConfigError(Exception): + """Base exception for configuration file exceptions.""" + + pass + + +class ConfigFileNotFound(ConfigError): + """A specified configuration file is missing or unreadable.""" + + pass diff --git a/src/bandersnatch/configuration.py b/src/bandersnatch/configuration.py index 8e2ca4da5..b05f9c594 100644 --- a/src/bandersnatch/configuration.py +++ b/src/bandersnatch/configuration.py @@ -2,24 +2,34 @@ Module containing classes to access the bandersnatch configuration file """ +import abc import configparser import importlib.resources import logging +import shutil +from configparser import ConfigParser from pathlib import Path from typing import Any, NamedTuple from .config.diff_file_reference import eval_config_reference, has_config_reference +from .config.exceptions import ConfigError, ConfigFileNotFound from .simple import SimpleDigest, SimpleFormat, get_digest_value, get_format_value logger = logging.getLogger("bandersnatch") +# Filename of example configuration file inside the bandersnatch package +_example_conf_file = "example.conf" + +# Filename of default values file inside the bandersnatch package +_defaults_conf_file = "defaults.conf" + class SetConfigValues(NamedTuple): json_save: bool root_uri: str diff_file_path: str diff_append_epoch: bool - digest_name: str + digest_name: SimpleDigest storage_backend_name: str cleanup: bool release_files_save: bool @@ -38,70 +48,108 @@ def __call__(cls, *args: Any, **kwargs: Any) -> type: return cls._instances[cls] -class BandersnatchConfig(metaclass=Singleton): +# ConfigParser's metaclass is abc.ABCMeta; we can't inherit from ConfigParser and +# also use the Singleton metaclass unless we "manually" combine the metaclasses. +class _SingletonABCMeta(Singleton, abc.ABCMeta): + pass + + +class BandersnatchConfig(ConfigParser, metaclass=_SingletonABCMeta): + """Configuration singleton. Provides global access to loaded configuration + options as a ConfigParser subclass. Always reads default mirror options when + initialized. If given a file path, that file is loaded second so its values + overwrite corresponding defaults. + """ + # Ensure we only show the deprecations once SHOWN_DEPRECATIONS = False - def __init__(self, config_file: str | None = None) -> None: - """ - Bandersnatch configuration class singleton - - This class is a singleton that parses the configuration once at the - start time. + def __init__( + self, config_file: Path | None = None, load_defaults: bool = True + ) -> None: + """Create the singleton configuration object. Default configuration values are + read from a configuration file inside the package. If a file path is given, that + file is read after reading defaults such that it's values overwrite defaults. - Parameters - ========== - config_file: str, optional - Path to the configuration file to use + :param Path | None config_file: non-default configuration file to load, defaults to None """ + super(ConfigParser, self).__init__(delimiters="=") self.found_deprecations: list[str] = [] - self.default_config_file = str( - importlib.resources.files("bandersnatch") / "default.conf" - ) - self.config_file = config_file - self.load_configuration() - # Keeping for future deprecations ... Commenting to save function call etc. - # self.check_for_deprecations() + + # ConfigParser.read can process an iterable of file paths, but separate read + # calls are used on purpose to add more information to error messages. + if load_defaults: + self._read_defaults_file() + if config_file: + self._read_user_config_file(config_file) + + def optionxform(self, optionstr: str) -> str: + return optionstr def check_for_deprecations(self) -> None: if self.SHOWN_DEPRECATIONS: return self.SHOWN_DEPRECATIONS = True - def load_configuration(self) -> None: - """ - Read the configuration from a configuration file - """ - config_file = self.default_config_file - if self.config_file: - config_file = self.config_file - self.config = configparser.ConfigParser(delimiters="=") - # mypy is unhappy with us assigning to a method - (monkeypatching?) - self.config.optionxform = lambda option: option # type: ignore - self.config.read(config_file) + def _read_defaults_file(self) -> None: + try: + defaults_file = ( + importlib.resources.files("bandersnatch") / _defaults_conf_file + ) + self.read(str(defaults_file)) + logger.debug("Read configuration defaults file.") + except OSError as err: + raise ConfigError("Error reading configuration defaults: %s", err) from err + + def _read_user_config_file(self, config_file: Path) -> None: + # Check for this case explicitly instead of letting it fall under the OSError + # case, so we can use the exception type for control flow: + if not config_file.exists(): + raise ConfigFileNotFound( + f"Specified configuration file doesn't exist: {config_file}" + ) + + # Standard configparser, but we want to add context information to an OSError + try: + self.read(config_file) + logger.info("Read configuration file '%s'", config_file) + except OSError as err: + raise ConfigError( + "Error reading configuration file '%s': %s", config_file, err + ) from err + + +def create_example_config(dest: Path) -> None: + """Create an example configuration file at the specified location. + + :param Path dest: destination path for the configuration file. + """ + example_source = importlib.resources.files("bandersnatch") / _example_conf_file + try: + shutil.copy(str(example_source), dest) + except OSError as err: + logger.error("Could not create config file '%s': %s", dest, err) def validate_config_values( # noqa: C901 config: configparser.ConfigParser, ) -> SetConfigValues: - try: - json_save = config.getboolean("mirror", "json") - except configparser.NoOptionError: - logger.error( - "Please update your config to include a json " - + "boolean in the [mirror] section. Setting to False" - ) - json_save = False - try: - root_uri = config.get("mirror", "root_uri") - except configparser.NoOptionError: - root_uri = "" + json_save = config.getboolean("mirror", "json") - try: - diff_file_path = config.get("mirror", "diff-file") - except configparser.NoOptionError: - diff_file_path = "" + root_uri = config.get("mirror", "root_uri") + + release_files_save = config.getboolean("mirror", "release-files") + + if not release_files_save and not root_uri: + root_uri = "https://files.pythonhosted.org" + logger.warning( + "Please update your config to include a root_uri in the [mirror] " + + "section when disabling release file sync. Setting to " + + root_uri + ) + + diff_file_path = config.get("mirror", "diff-file") if diff_file_path and has_config_reference(diff_file_path): try: @@ -114,27 +162,12 @@ def validate_config_values( # noqa: C901 mirror_dir = config.get("mirror", "directory") diff_file_path = (Path(mirror_dir) / "mirrored-files").as_posix() - try: - diff_append_epoch = config.getboolean("mirror", "diff-append-epoch") - except configparser.NoOptionError: - diff_append_epoch = False + diff_append_epoch = config.getboolean("mirror", "diff-append-epoch") - try: - logger.debug("Checking config for storage backend...") - storage_backend_name = config.get("mirror", "storage-backend") - logger.debug("Found storage backend in config!") - except configparser.NoOptionError: - storage_backend_name = "filesystem" - logger.debug( - "Failed to find storage backend in config, falling back to default!" - ) - logger.info(f"Selected storage backend: {storage_backend_name}") + storage_backend_name = config.get("mirror", "storage-backend") try: digest_name = get_digest_value(config.get("mirror", "digest_name")) - except configparser.NoOptionError: - digest_name = SimpleDigest.SHA256 - logger.debug(f"Using digest {digest_name} by default ...") except ValueError as e: logger.error( f"Supplied digest_name {config.get('mirror', 'digest_name')} is " @@ -144,64 +177,38 @@ def validate_config_values( # noqa: C901 raise e try: - cleanup = config.getboolean("mirror", "cleanup") - except configparser.NoOptionError: - logger.debug( - "bandersnatch is not cleaning up non PEP 503 normalized Simple " - + "API directories" - ) - cleanup = False - - release_files_save = config.getboolean("mirror", "release-files", fallback=True) - if not release_files_save and not root_uri: - root_uri = "https://files.pythonhosted.org" + simple_format_raw = config.get("mirror", "simple-format") + simple_format = get_format_value(simple_format_raw) + except ValueError as e: logger.error( - "Please update your config to include a root_uri in the [mirror] " - + "section when disabling release file sync. Setting to " - + root_uri + f"Supplied simple-format {simple_format_raw} is not supported!" + + " Please updare the simple-format in the [mirror] section of" + + " your config to a supported value." ) + raise e - try: - logger.debug("Checking config for compare method...") - compare_method = config.get("mirror", "compare-method") - logger.debug("Found compare method in config!") - except configparser.NoOptionError: - compare_method = "hash" - logger.debug( - "Failed to find compare method in config, falling back to default!" - ) + compare_method = config.get("mirror", "compare-method") if compare_method not in ("hash", "stat"): raise ValueError( f"Supplied compare_method {compare_method} is not supported! Please " + "update compare_method to one of ('hash', 'stat') in the [mirror] " + "section." ) - logger.info(f"Selected compare method: {compare_method}") - try: - logger.debug("Checking config for alternative download mirror...") - download_mirror = config.get("mirror", "download-mirror") - logger.info(f"Selected alternative download mirror {download_mirror}") - except configparser.NoOptionError: - download_mirror = "" - logger.debug("No alternative download mirror found in config.") + download_mirror = config.get("mirror", "download-mirror") if download_mirror: - try: - logger.debug( - "Checking config for only download from alternative download" - + "mirror..." - ) - download_mirror_no_fallback = config.getboolean( - "mirror", "download-mirror-no-fallback" - ) - if download_mirror_no_fallback: - logger.info("Setting to download from mirror without fallback") - else: - logger.debug("Setting to fallback to original if download mirror fails") - except configparser.NoOptionError: - download_mirror_no_fallback = False - logger.debug("No download mirror fallback setting found in config.") + + logger.debug( + "Checking config for only download from alternative download" + "mirror..." + ) + download_mirror_no_fallback = config.getboolean( + "mirror", "download-mirror-no-fallback" + ) + if download_mirror_no_fallback: + logger.info("Setting to download from mirror without fallback") + else: + logger.debug("Setting to fallback to original if download mirror fails") else: download_mirror_no_fallback = False logger.debug( @@ -209,11 +216,7 @@ def validate_config_values( # noqa: C901 + "is not set in config." ) - try: - simple_format = get_format_value(config.get("mirror", "simple-format")) - except configparser.NoOptionError: - logger.debug("Storing all Simple Formats by default ...") - simple_format = SimpleFormat.ALL + cleanup = config.getboolean("mirror", "cleanup", fallback=False) return SetConfigValues( json_save, diff --git a/src/bandersnatch/defaults.conf b/src/bandersnatch/defaults.conf new file mode 100644 index 000000000..ca1cbc709 --- /dev/null +++ b/src/bandersnatch/defaults.conf @@ -0,0 +1,34 @@ +; [ Default Config Values ] +; Bandersnatch loads this file prior to loading the user config file. +; The values in this file serve as defaults and are overriden if also +; specified in a user config. +[mirror] +storage-backend = filesystem + +master = https://pypi.org +proxy = +download-mirror = +download-mirror-no-fallback = false + +json = false +release-files = true +hash-index = false +simple-format = ALL +compare-method = hash +digest_name = sha256 +keep-index-versions = 0 +cleanup = false + +stop-on-error = false +timeout = 10 +global-timeout = 1800 +workers = 3 +verifiers = 3 + +; dynamic default: this URI used if `release-files = false` +; root_uri = https://files.pythonhosted.org +root_uri = +diff-file = +diff-append-epoch = false + +log-config = diff --git a/src/bandersnatch/default.conf b/src/bandersnatch/example.conf similarity index 100% rename from src/bandersnatch/default.conf rename to src/bandersnatch/example.conf diff --git a/src/bandersnatch/filter.py b/src/bandersnatch/filter.py index d78735825..33d72df03 100644 --- a/src/bandersnatch/filter.py +++ b/src/bandersnatch/filter.py @@ -37,7 +37,7 @@ class Filter: deprecated_name: str = "" def __init__(self, *args: Any, **kwargs: Any) -> None: - self.configuration = BandersnatchConfig().config + self.configuration = BandersnatchConfig() if ( "plugins" not in self.configuration or "enabled" not in self.configuration["plugins"] @@ -156,7 +156,7 @@ def __init__(self, load_all: bool = False) -> None: """ Loads and stores all of specified filters from the config file """ - self.config = BandersnatchConfig().config + self.config = BandersnatchConfig() self.loaded_filter_plugins: dict[str, list["Filter"]] = defaultdict(list) self.enabled_plugins = self._load_enabled() if load_all: diff --git a/src/bandersnatch/main.py b/src/bandersnatch/main.py index 92a0386e8..5b165699c 100644 --- a/src/bandersnatch/main.py +++ b/src/bandersnatch/main.py @@ -14,6 +14,7 @@ import bandersnatch.master import bandersnatch.mirror import bandersnatch.verify +from bandersnatch.config.exceptions import ConfigError, ConfigFileNotFound from bandersnatch.storage import storage_backend_plugins # See if we have uvloop and use if so @@ -202,23 +203,22 @@ def main(loop: asyncio.AbstractEventLoop | None = None) -> int: # Prepare default config file if needed. config_path = Path(args.config) - if not config_path.exists(): + try: + logger.info("Reading configuration file '%s'", config_path) + config = bandersnatch.configuration.BandersnatchConfig(config_path) + except ConfigFileNotFound: logger.warning(f"Config file '{args.config}' missing, creating default config.") logger.warning("Please review the config file, then run 'bandersnatch' again.") - - default_config_path = Path(__file__).parent / "default.conf" - try: - shutil.copy(default_config_path, args.config) - except OSError as e: - logger.error(f"Could not create config file: {e}") + bandersnatch.configuration.create_example_config(config_path) return 1 - - config = bandersnatch.configuration.BandersnatchConfig( - config_file=args.config - ).config - - if config.has_option("mirror", "log-config"): - logging.config.fileConfig(str(Path(config.get("mirror", "log-config")))) + except ConfigError as err: + logger.error("Unable to load configuration: %s", err) + logger.debug("Error details:", exc_info=err) + return 2 + + user_log_config = config.get("mirror", "log-config", fallback=None) + if user_log_config: + logging.config.fileConfig(Path(user_log_config)) if loop: loop.set_debug(args.debug) diff --git a/src/bandersnatch/storage.py b/src/bandersnatch/storage.py index d51665782..5f0709afc 100644 --- a/src/bandersnatch/storage.py +++ b/src/bandersnatch/storage.py @@ -63,11 +63,9 @@ def __init__( ) -> None: self.flock_path: PATH_TYPES = ".lock" if config is not None: - if isinstance(config, BandersnatchConfig): - config = config.config self.configuration = config else: - self.configuration = BandersnatchConfig().config + self.configuration = BandersnatchConfig() try: storage_backend = self.configuration["mirror"]["storage-backend"] except (KeyError, TypeError): @@ -342,7 +340,7 @@ def load_storage_plugins( """ global loaded_storage_plugins if config is None: - config = BandersnatchConfig().config + config = BandersnatchConfig() if not enabled_plugin: try: enabled_plugin = config["mirror"]["storage-backend"] diff --git a/src/bandersnatch/tests/mock_config.py b/src/bandersnatch/tests/mock_config.py index b9830aecb..0db5c707b 100644 --- a/src/bandersnatch/tests/mock_config.py +++ b/src/bandersnatch/tests/mock_config.py @@ -3,13 +3,18 @@ def mock_config(contents: str, filename: str = "test.conf") -> BandersnatchConfig: """ - Creates a config file with contents and loads them into a - BandersnatchConfig instance. + Creates a config file with contents and loads them into a BandersnatchConfig instance. + Because BandersnatchConfig is a singleton, it needs to be cleared before reading any + new configuration so the configuration from different tests aren't re-used. """ - with open(filename, "w") as fd: - fd.write(contents) - - instance = BandersnatchConfig() - instance.config_file = filename - instance.load_configuration() + # If this is the first time BandersnatchConfig was initialized during a test run, + # skip loading defaults in init b/c we're going to do that explicitly instead. + instance = BandersnatchConfig(load_defaults=False) + # If this *isn't* the first time BandersnatchConfig was initialized, then we've + # got to clear any previously loaded configuration from the singleton. + instance.clear() + # explicitly load defaults here + instance._read_defaults_file() + # load specified config content + instance.read_string(contents) return instance diff --git a/src/bandersnatch/tests/plugins/test_storage_plugin_s3.py b/src/bandersnatch/tests/plugins/test_storage_plugin_s3.py index a7c91741c..17c20fb6f 100644 --- a/src/bandersnatch/tests/plugins/test_storage_plugin_s3.py +++ b/src/bandersnatch/tests/plugins/test_storage_plugin_s3.py @@ -145,7 +145,7 @@ def test_scandir(s3_mock: S3Path) -> None: def test_plugin_init(s3_mock: S3Path) -> None: - config_loader = mock_config( + config = mock_config( """ [mirror] directory = /tmp/pypi @@ -168,14 +168,14 @@ def test_plugin_init(s3_mock: S3Path) -> None: signature_version = s3v4 """ ) - backend = s3.S3Storage(config=config_loader.config) + backend = s3.S3Storage(config=config) backend.initialize_plugin() path = s3.S3Path("/tmp/pypi") resource, _ = configuration_map.get_configuration(path) assert resource.meta.client.meta.endpoint_url == "http://localhost:9090" - config_loader = mock_config( + config = mock_config( """ [mirror] directory = /tmp/pypi @@ -194,7 +194,7 @@ def test_plugin_init(s3_mock: S3Path) -> None: endpoint_url = http://localhost:9090 """ ) - backend = s3.S3Storage(config=config_loader.config) + backend = s3.S3Storage(config=config) backend.initialize_plugin() path = s3.S3Path("/tmp/pypi") @@ -203,7 +203,7 @@ def test_plugin_init(s3_mock: S3Path) -> None: def test_plugin_init_with_boto3_configs(s3_mock: S3Path) -> None: - config_loader = mock_config( + config = mock_config( """ [mirror] directory = /tmp/pypi @@ -227,7 +227,7 @@ def test_plugin_init_with_boto3_configs(s3_mock: S3Path) -> None: config_param_ServerSideEncryption = AES256 """ ) - backend = s3.S3Storage(config=config_loader.config) + backend = s3.S3Storage(config=config) backend.initialize_plugin() assert backend.configuration_parameters["ServerSideEncryption"] == "AES256" diff --git a/src/bandersnatch/tests/plugins/test_storage_plugins.py b/src/bandersnatch/tests/plugins/test_storage_plugins.py index e8f8ed3b3..a383d242c 100644 --- a/src/bandersnatch/tests/plugins/test_storage_plugins.py +++ b/src/bandersnatch/tests/plugins/test_storage_plugins.py @@ -622,7 +622,7 @@ def test_plugin_type(self) -> None: self.assertTrue(self.plugin.PATH_BACKEND is self.path_backends[self.backend]) def test_json_paths(self) -> None: - config = mock_config(self.config_contents).config + config = mock_config(self.config_contents) mirror_dir = self.plugin.PATH_BACKEND(config.get("mirror", "directory")) packages = { "bandersnatch": [ diff --git a/src/bandersnatch/tests/test_configuration.py b/src/bandersnatch/tests/test_configuration.py index 89e628693..900574144 100644 --- a/src/bandersnatch/tests/test_configuration.py +++ b/src/bandersnatch/tests/test_configuration.py @@ -2,6 +2,7 @@ import importlib.resources import os import unittest +from pathlib import Path from tempfile import TemporaryDirectory from unittest import TestCase @@ -12,7 +13,7 @@ Singleton, validate_config_values, ) -from bandersnatch.simple import SimpleFormat +from bandersnatch.simple import SimpleDigest, SimpleFormat class TestBandersnatchConf(TestCase): @@ -44,34 +45,43 @@ def test_is_singleton(self) -> None: self.assertEqual(id(instance1), id(instance2)) def test_single_config__default__all_sections_present(self) -> None: - config_file = str(importlib.resources.files("bandersnatch") / "unittest.conf") - instance = BandersnatchConfig(str(config_file)) + config_file = Path( + str(importlib.resources.files("bandersnatch") / "unittest.conf") + ) + instance = BandersnatchConfig(config_file) # All default values should at least be present and be the write types for section in ["mirror", "plugins", "blocklist"]: - self.assertIn(section, instance.config.sections()) + self.assertIn(section, instance.sections()) def test_single_config__default__mirror__setting_attributes(self) -> None: instance = BandersnatchConfig() - options = [option for option in instance.config["mirror"]] - options.sort() - self.assertListEqual( + options = {option for option in instance["mirror"]} + self.assertSetEqual( options, - [ + { "cleanup", "compare-method", - "directory", + "diff-append-epoch", + "diff-file", + "digest_name", + "download-mirror", + "download-mirror-no-fallback", "global-timeout", "hash-index", "json", + "keep-index-versions", + "log-config", "master", + "proxy", "release-files", + "root_uri", "simple-format", "stop-on-error", "storage-backend", "timeout", "verifiers", "workers", - ], + }, ) def test_single_config__default__mirror__setting__types(self) -> None: @@ -92,42 +102,34 @@ def test_single_config__default__mirror__setting__types(self) -> None: ("compare-method", str), ]: self.assertIsInstance( - option_type(instance.config["mirror"].get(option)), option_type + option_type(instance["mirror"].get(option)), option_type ) def test_single_config_custom_setting_boolean(self) -> None: - with open("test.conf", "w") as testconfig_handle: - testconfig_handle.write("[mirror]\nhash-index=false\n") instance = BandersnatchConfig() - instance.config_file = "test.conf" - instance.load_configuration() - self.assertFalse(instance.config["mirror"].getboolean("hash-index")) + instance.read_string("[mirror]\nhash-index=false\n") + + self.assertFalse(instance["mirror"].getboolean("hash-index")) def test_single_config_custom_setting_int(self) -> None: - with open("test.conf", "w") as testconfig_handle: - testconfig_handle.write("[mirror]\ntimeout=999\n") instance = BandersnatchConfig() - instance.config_file = "test.conf" - instance.load_configuration() - self.assertEqual(int(instance.config["mirror"]["timeout"]), 999) + instance.read_string("[mirror]\ntimeout=999\n") + + self.assertEqual(int(instance["mirror"]["timeout"]), 999) def test_single_config_custom_setting_str(self) -> None: - with open("test.conf", "w") as testconfig_handle: - testconfig_handle.write("[mirror]\nmaster=https://foo.bar.baz\n") instance = BandersnatchConfig() - instance.config_file = "test.conf" - instance.load_configuration() - self.assertEqual(instance.config["mirror"]["master"], "https://foo.bar.baz") + instance.read_string("[mirror]\nmaster=https://foo.bar.baz\n") + + self.assertEqual(instance["mirror"]["master"], "https://foo.bar.baz") def test_multiple_instances_custom_setting_str(self) -> None: - with open("test.conf", "w") as testconfig_handle: - testconfig_handle.write("[mirror]\nmaster=https://foo.bar.baz\n") instance1 = BandersnatchConfig() - instance1.config_file = "test.conf" - instance1.load_configuration() + instance1.read_string("[mirror]\nmaster=https://foo.bar.baz\n") instance2 = BandersnatchConfig() - self.assertEqual(instance2.config["mirror"]["master"], "https://foo.bar.baz") + + self.assertEqual(instance2["mirror"]["master"], "https://foo.bar.baz") def test_validate_config_values(self) -> None: default_values = SetConfigValues( @@ -135,7 +137,7 @@ def test_validate_config_values(self) -> None: "", "", False, - "sha256", + SimpleDigest.SHA256, "filesystem", False, True, @@ -144,8 +146,7 @@ def test_validate_config_values(self) -> None: False, SimpleFormat.ALL, ) - no_options_configparser = configparser.ConfigParser() - no_options_configparser["mirror"] = {} + no_options_configparser = BandersnatchConfig(load_defaults=True) self.assertEqual( default_values, validate_config_values(no_options_configparser) ) @@ -156,7 +157,7 @@ def test_validate_config_values_release_files_false_sets_root_uri(self) -> None: "https://files.pythonhosted.org", "", False, - "sha256", + SimpleDigest.SHA256, "filesystem", False, False, @@ -165,8 +166,8 @@ def test_validate_config_values_release_files_false_sets_root_uri(self) -> None: False, SimpleFormat.ALL, ) - release_files_false_configparser = configparser.ConfigParser() - release_files_false_configparser["mirror"] = {"release-files": "false"} + release_files_false_configparser = BandersnatchConfig(load_defaults=True) + release_files_false_configparser["mirror"].update({"release-files": "false"}) self.assertEqual( default_values, validate_config_values(release_files_false_configparser) ) @@ -179,7 +180,7 @@ def test_validate_config_values_download_mirror_false_sets_no_fallback( "", "", False, - "sha256", + SimpleDigest.SHA256, "filesystem", False, True, @@ -188,10 +189,12 @@ def test_validate_config_values_download_mirror_false_sets_no_fallback( False, SimpleFormat.ALL, ) - release_files_false_configparser = configparser.ConfigParser() - release_files_false_configparser["mirror"] = { - "download-mirror-no-fallback": "true", - } + release_files_false_configparser = BandersnatchConfig(load_defaults=True) + release_files_false_configparser["mirror"].update( + { + "download-mirror-no-fallback": "true", + } + ) self.assertEqual( default_values, validate_config_values(release_files_false_configparser) ) @@ -247,7 +250,7 @@ def test_validate_config_diff_file_reference(self) -> None: expected=expected, cfg_data=cfg_data, ): - cfg = configparser.ConfigParser() + cfg = BandersnatchConfig(load_defaults=True) cfg.read_dict(cfg_data) config_values = validate_config_values(cfg) self.assertIsInstance(config_values.diff_file_path, str) diff --git a/src/bandersnatch/tests/test_delete.py b/src/bandersnatch/tests/test_delete.py index f120a4bb3..4a08b07f2 100644 --- a/src/bandersnatch/tests/test_delete.py +++ b/src/bandersnatch/tests/test_delete.py @@ -11,6 +11,7 @@ from aiohttp import ClientResponseError from pytest import MonkeyPatch +from bandersnatch.configuration import BandersnatchConfig from bandersnatch.delete import delete_packages, delete_path, delete_simple_page from bandersnatch.master import Master from bandersnatch.mirror import BandersnatchMirror @@ -77,6 +78,7 @@ def _fake_config() -> ConfigParser: @pytest.mark.asyncio async def test_delete_path() -> None: + BandersnatchConfig().read_dict(_fake_config()) with TemporaryDirectory() as td: td_path = Path(td) fake_path = td_path / "unittest-file.tgz" diff --git a/src/bandersnatch/tests/test_filter.py b/src/bandersnatch/tests/test_filter.py index 4932db05c..87d75d205 100644 --- a/src/bandersnatch/tests/test_filter.py +++ b/src/bandersnatch/tests/test_filter.py @@ -124,11 +124,9 @@ def test__filter_base_clases(self) -> None: self.assertFalse(error) def test_deprecated_keys(self) -> None: - with open("test.conf", "w") as f: - f.write("[allowlist]\npackages=foo\n[blocklist]\npackages=bar\n") instance = BandersnatchConfig() - instance.config_file = "test.conf" - instance.load_configuration() + instance.read_string("[allowlist]\npackages=foo\n[blocklist]\npackages=bar\n") + plugin = Filter() assert plugin.allowlist.name == "allowlist" assert plugin.blocklist.name == "blocklist" diff --git a/src/bandersnatch/tests/test_main.py b/src/bandersnatch/tests/test_main.py index 5fc34b340..47fcdd14d 100644 --- a/src/bandersnatch/tests/test_main.py +++ b/src/bandersnatch/tests/test_main.py @@ -108,7 +108,7 @@ def test_main_reads_custom_config_values( sys.argv = ["bandersnatch", "-c", conffile, "mirror"] main(asyncio.new_event_loop()) (log_config, _kwargs) = logging_mock.call_args_list[0] - assert log_config == (str(customconfig / "bandersnatch-log.conf"),) + assert log_config == ((customconfig / "bandersnatch-log.conf"),) def test_main_throws_exception_on_unsupported_digest_name( diff --git a/src/bandersnatch/tests/test_mirror.py b/src/bandersnatch/tests/test_mirror.py index 2b1a5d64e..1e4dbffe7 100644 --- a/src/bandersnatch/tests/test_mirror.py +++ b/src/bandersnatch/tests/test_mirror.py @@ -3,7 +3,6 @@ import time import unittest.mock as mock from collections.abc import Awaitable, Callable, Iterator, Mapping -from configparser import ConfigParser from os import sep from pathlib import Path from tempfile import TemporaryDirectory @@ -144,9 +143,9 @@ def test_mirror_filter_packages_match(tmpdir: Path) -> None: example1 """ Singleton._instances = {} - with open("test.conf", "w") as testconfig_handle: - testconfig_handle.write(test_configuration) - BandersnatchConfig("test.conf") + test_conf = Path("test.conf") + test_conf.write_text(test_configuration) + BandersnatchConfig(config_file=test_conf) m = BandersnatchMirror(tmpdir, mock.Mock()) m.packages_to_sync = {"example1": "", "example2": ""} m._filter_packages() @@ -167,9 +166,9 @@ def test_mirror_filter_packages_nomatch_package_with_spec(tmpdir: Path) -> None: example3>2.0.0 """ Singleton._instances = {} - with open("test.conf", "w") as testconfig_handle: - testconfig_handle.write(test_configuration) - BandersnatchConfig("test.conf") + test_conf = Path("test.conf") + test_conf.write_text(test_configuration) + BandersnatchConfig(config_file=test_conf) m = BandersnatchMirror(tmpdir, mock.Mock()) m.packages_to_sync = {"example1": "", "example3": ""} m._filter_packages() @@ -1332,7 +1331,7 @@ async def test_mirror_subcommand_only_creates_diff_file_if_configured( ) -> None: # Setup: create a configuration for the 'mirror' subcommand # Mirror section only contains required options and omits 'diff-file' - config = ConfigParser() + config = BandersnatchConfig(load_defaults=True) config.read_dict( { "mirror": { @@ -1415,7 +1414,7 @@ async def test_mirror_subcommand_diff_file_dir_with_epoch( # Setup: create configuration for 'mirror' subcommand that includes both # `diff-file` and `diff-append-epoch` - config = ConfigParser() + config = BandersnatchConfig(load_defaults=True) config.read_dict( { "mirror": { diff --git a/src/bandersnatch/tests/test_simple.py b/src/bandersnatch/tests/test_simple.py index 4ac9de7c1..54d2b257d 100644 --- a/src/bandersnatch/tests/test_simple.py +++ b/src/bandersnatch/tests/test_simple.py @@ -6,7 +6,7 @@ import pytest from bandersnatch import utils -from bandersnatch.configuration import validate_config_values +from bandersnatch.configuration import BandersnatchConfig, validate_config_values from bandersnatch.package import Package from bandersnatch.simple import ( InvalidDigestFormat, @@ -46,8 +46,7 @@ def test_digest_valid() -> None: def test_digest_config_default() -> None: - c = ConfigParser() - c.add_section("mirror") + c = BandersnatchConfig(load_defaults=True) config = validate_config_values(c) s = SimpleAPI(Storage(), "ALL", [], config.digest_name, False, None) assert config.digest_name.upper() in [v.name for v in SimpleDigest]