From 4c5331353b5a553b5083358d6df5085e455cc800 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 18 Jul 2023 11:40:06 -0600 Subject: [PATCH 01/19] chore: pre-commit autoupdate (#1851) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edgar Ramírez Mondragón --- .pre-commit-config.yaml | 4 ++-- pyproject.toml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3cde0b46..3c8a18afe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,14 +36,14 @@ repos: )$ - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.23.2 + rev: 0.23.3 hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.277 + rev: v0.0.278 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] diff --git a/pyproject.toml b/pyproject.toml index d4a73d704..bac2a0f7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -220,6 +220,8 @@ pytest11 = { callable = "singer_sdk:testing.pytest_plugin", extras = ["testing"] [tool.ruff] exclude = [ "cookiecutter/*", + "singer_sdk/helpers/_simpleeval.py", + "tests/core/test_simpleeval.py", ] ignore = [ "ANN101", # Missing type annotation for `self` in method From e6553866950d1104a282c3eced76958c6b834bdc Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Tue, 18 Jul 2023 17:40:10 -0600 Subject: [PATCH 02/19] fix(targets): Check against the unconformed key properties when validating record keys (#1853) --- singer_sdk/sinks/core.py | 7 +++++-- tests/conftest.py | 4 ++++ tests/core/test_target_base.py | 25 +++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/singer_sdk/sinks/core.py b/singer_sdk/sinks/core.py index ec43c4060..9928aa6f2 100644 --- a/singer_sdk/sinks/core.py +++ b/singer_sdk/sinks/core.py @@ -215,6 +215,9 @@ def datetime_error_treatment(self) -> DatetimeErrorTreatmentEnum: def key_properties(self) -> list[str]: """Return key properties. + Override this method to return a list of key properties in a format that is + compatible with the target. + Returns: A list of stream key properties. """ @@ -331,10 +334,10 @@ def _singer_validate_message(self, record: dict) -> None: Raises: MissingKeyPropertiesError: If record is missing one or more key properties. """ - if not all(key_property in record for key_property in self.key_properties): + if any(key_property not in record for key_property in self._key_properties): msg = ( f"Record is missing one or more key_properties. \n" - f"Key Properties: {self.key_properties}, " + f"Key Properties: {self._key_properties}, " f"Record Keys: {list(record.keys())}" ) raise MissingKeyPropertiesError( diff --git a/tests/conftest.py b/tests/conftest.py index 142e76fe1..cb201c9a1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -86,6 +86,10 @@ def process_batch(self, context: dict) -> None: self.target.records_written.extend(context["records"]) self.target.num_batches_processed += 1 + @property + def key_properties(self) -> list[str]: + return [key.upper() for key in super().key_properties] + class TargetMock(Target): """A mock Target class.""" diff --git a/tests/core/test_target_base.py b/tests/core/test_target_base.py index 778fab722..1fd6b9a93 100644 --- a/tests/core/test_target_base.py +++ b/tests/core/test_target_base.py @@ -2,6 +2,9 @@ import copy +import pytest + +from singer_sdk.exceptions import MissingKeyPropertiesError from tests.conftest import BatchSinkMock, TargetMock @@ -28,3 +31,25 @@ def test_get_sink(): key_properties=key_properties, ) assert sink_returned == sink + + +def test_validate_record(): + target = TargetMock() + sink = BatchSinkMock( + target=target, + stream_name="test", + schema={ + "properties": { + "id": {"type": ["integer"]}, + "name": {"type": ["string"]}, + }, + }, + key_properties=["id"], + ) + + # Test valid record + sink._singer_validate_message({"id": 1, "name": "test"}) + + # Test invalid record + with pytest.raises(MissingKeyPropertiesError): + sink._singer_validate_message({"name": "test"}) From e36da21d7fd05824de727228041bb2b19c0b4b82 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Tue, 18 Jul 2023 18:01:55 -0600 Subject: [PATCH 03/19] fix: Ensure all expected tap parameters are passed to `SQLTap` initializer (#1842) --- singer_sdk/tap_base.py | 28 ++++------------------------ tests/samples/conftest.py | 2 +- tests/samples/test_tap_sqlite.py | 21 +++++++++++++++++++++ 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/singer_sdk/tap_base.py b/singer_sdk/tap_base.py index 95800925c..08ce6f3ad 100644 --- a/singer_sdk/tap_base.py +++ b/singer_sdk/tap_base.py @@ -612,37 +612,17 @@ class SQLTap(Tap): # Stream class used to initialize new SQL streams from their catalog declarations. default_stream_class: type[SQLStream] - def __init__( - self, - *, - config: dict | PurePath | str | list[PurePath | str] | None = None, - catalog: PurePath | str | dict | None = None, - state: PurePath | str | dict | None = None, - parse_env_config: bool = False, - validate_config: bool = True, - ) -> None: + def __init__(self, *args: t.Any, **kwargs: t.Any) -> None: """Initialize the SQL tap. The SQLTap initializer additionally creates a cache variable for _catalog_dict. Args: - config: Tap configuration. Can be a dictionary, a single path to a - configuration file, or a list of paths to multiple configuration - files. - catalog: Tap catalog. Can be a dictionary or a path to the catalog file. - state: Tap state. Can be dictionary or a path to the state file. - parse_env_config: Whether to look for configuration values in environment - variables. - validate_config: True to require validation of config settings. + *args: Positional arguments for the Tap initializer. + **kwargs: Keyword arguments for the Tap initializer. """ self._catalog_dict: dict | None = None - super().__init__( - config=config, - catalog=catalog, - state=state, - parse_env_config=parse_env_config, - validate_config=validate_config, - ) + super().__init__(*args, **kwargs) @property def catalog_dict(self) -> dict: diff --git a/tests/samples/conftest.py b/tests/samples/conftest.py index 60f277412..29560a330 100644 --- a/tests/samples/conftest.py +++ b/tests/samples/conftest.py @@ -73,7 +73,7 @@ def path_to_sample_data_db(tmp_path: Path) -> Path: @pytest.fixture -def sqlite_sample_db_config(path_to_sample_data_db: str) -> dict: +def sqlite_sample_db_config(path_to_sample_data_db: Path) -> dict: """Get configuration dictionary for target-csv.""" return {"path_to_db": str(path_to_sample_data_db)} diff --git a/tests/samples/test_tap_sqlite.py b/tests/samples/test_tap_sqlite.py index 5e1349f94..bceaa34b1 100644 --- a/tests/samples/test_tap_sqlite.py +++ b/tests/samples/test_tap_sqlite.py @@ -1,7 +1,11 @@ from __future__ import annotations +import json import typing as t +from click.testing import CliRunner + +from samples.sample_tap_sqlite import SQLiteTap from samples.sample_target_csv.csv_target import SampleTargetCSV from singer_sdk import SQLStream from singer_sdk._singerlib import MetadataMapping, StreamMetadata @@ -24,6 +28,23 @@ def _discover_and_select_all(tap: SQLTap) -> None: catalog_entry["metadata"] = md.to_list() +def test_tap_sqlite_cli(sqlite_sample_db_config: dict[str, t.Any], tmp_path: Path): + runner = CliRunner() + filepath = tmp_path / "config.json" + + with filepath.open("w") as f: + json.dump(sqlite_sample_db_config, f) + + result = runner.invoke( + SQLiteTap.cli, + ["--discover", "--config", str(filepath)], + ) + assert result.exit_code == 0 + + catalog = json.loads(result.stdout) + assert "streams" in catalog + + def test_sql_metadata(sqlite_sample_tap: SQLTap): stream = t.cast(SQLStream, sqlite_sample_tap.streams["main-t1"]) detected_metadata = stream.catalog_entry["metadata"] From edb0c00bc0c57b3873b20d26cd9e0fbe43e413fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Jul 2023 13:11:34 +0000 Subject: [PATCH 04/19] chore(deps): Bump pyyaml from 6.0 to 6.0.1 (#1855) --- poetry.lock | 82 ++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8aa96235c..36bbe2da1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1797,51 +1797,51 @@ files = [ [[package]] name = "pyyaml" -version = "6.0" +version = "6.0.1" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.6" files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] [[package]] From 27d939de06b27441f46812edc54089e8a1596f5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Jul 2023 13:21:06 +0000 Subject: [PATCH 05/19] chore(deps): Bump pyjwt from 2.7.0 to 2.8.0 (#1856) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 36bbe2da1..328146c03 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1596,13 +1596,13 @@ plugins = ["importlib-metadata"] [[package]] name = "pyjwt" -version = "2.7.0" +version = "2.8.0" description = "JSON Web Token implementation in Python" optional = false python-versions = ">=3.7" files = [ - {file = "PyJWT-2.7.0-py3-none-any.whl", hash = "sha256:ba2b425b15ad5ef12f200dc67dd56af4e26de2331f965c5439994dad075876e1"}, - {file = "PyJWT-2.7.0.tar.gz", hash = "sha256:bd6ca4a3c4285c1a2d4349e5a035fdf8fb94e04ccd0fcbe6ba289dae9cc3e074"}, + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, ] [package.dependencies] From 260a15c21d9fb6572a955b14cc7cb0ec5086d102 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Jul 2023 13:30:55 +0000 Subject: [PATCH 06/19] chore(deps): Bump click from 8.1.5 to 8.1.6 (#1857) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 328146c03..9334d6645 100644 --- a/poetry.lock +++ b/poetry.lock @@ -404,13 +404,13 @@ files = [ [[package]] name = "click" -version = "8.1.5" +version = "8.1.6" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.5-py3-none-any.whl", hash = "sha256:e576aa487d679441d7d30abb87e1b43d24fc53bffb8758443b1a9e1cee504548"}, - {file = "click-8.1.5.tar.gz", hash = "sha256:4be4b1af8d665c6d942909916d31a213a106800c47d0eeba73d34da3cbc11367"}, + {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, + {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, ] [package.dependencies] From 384cff7526b2ac08413c9d1f4633404c7f7be481 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Wed, 19 Jul 2023 10:13:12 -0600 Subject: [PATCH 07/19] chore: Address deprecation warning in pytest plugin reference during Poetry build (#1859) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bac2a0f7d..bdf111925 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -215,7 +215,7 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] -pytest11 = { callable = "singer_sdk:testing.pytest_plugin", extras = ["testing"] } +pytest11 = { reference = "singer_sdk:testing.pytest_plugin", extras = ["testing"], type = "console" } [tool.ruff] exclude = [ From b5cc80516761f6ceee692cdc31deaaf39e278319 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:20:09 -0600 Subject: [PATCH 08/19] feat(deps): Bump latest supported sqlalchemy from `1.*` to `2.*` (#1484) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Edgar Ramírez Mondragón --- .github/workflows/test.yml | 10 +++++++--- noxfile.py | 11 ++++++++--- poetry.lock | 16 +--------------- pyproject.toml | 9 +-------- singer_sdk/sinks/sql.py | 6 +----- tests/conftest.py | 6 ++++++ 6 files changed, 24 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 495b4e58f..71a5adf30 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ env: jobs: tests: - name: Test on ${{ matrix.python-version }} (${{ matrix.session }}) / ${{ matrix.os }} + name: "Test on ${{ matrix.python-version }} (${{ matrix.session }}) / ${{ matrix.os }} / SQLAlchemy: ${{ matrix.sqlalchemy }}" runs-on: ${{ matrix.os }} env: NOXSESSION: ${{ matrix.session }} @@ -47,9 +47,11 @@ jobs: session: [tests] os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + sqlalchemy: ["2.*"] include: - - { session: doctest, python-version: "3.10", os: "ubuntu-latest" } - - { session: mypy, python-version: "3.8", os: "ubuntu-latest" } + - { session: tests, python-version: "3.11", os: "ubuntu-latest", sqlalchemy: "1.*" } + - { session: doctest, python-version: "3.10", os: "ubuntu-latest", sqlalchemy: "2.*" } + - { session: mypy, python-version: "3.8", os: "ubuntu-latest", sqlalchemy: "2.*" } steps: - name: Check out the repository @@ -86,6 +88,8 @@ jobs: nox --version - name: Run Nox + env: + SQLALCHEMY_VERSION: ${{ matrix.sqlalchemy }} run: | nox --python=${{ matrix.python-version }} diff --git a/noxfile.py b/noxfile.py index 4c4949413..7f65b8f21 100644 --- a/noxfile.py +++ b/noxfile.py @@ -85,6 +85,14 @@ def tests(session: Session) -> None: session.install(".[s3]") session.install(*test_dependencies) + sqlalchemy_version = os.environ.get("SQLALCHEMY_VERSION") + if sqlalchemy_version: + # Bypass nox-poetry use of --constraint so we can install a version of + # SQLAlchemy that doesn't match what's in poetry.lock. + session.poetry.session.install( # type: ignore[attr-defined] + f"sqlalchemy=={sqlalchemy_version}", + ) + try: session.run( "coverage", @@ -95,9 +103,6 @@ def tests(session: Session) -> None: "-v", "--durations=10", *session.posargs, - env={ - "SQLALCHEMY_WARN_20": "1", - }, ) finally: if session.interactive: diff --git a/poetry.lock b/poetry.lock index 9334d6645..96a31a297 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2346,20 +2346,6 @@ postgresql-psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql", "pymysql (<1)"] sqlcipher = ["sqlcipher3-binary"] -[[package]] -name = "sqlalchemy2-stubs" -version = "0.0.2a35" -description = "Typing Stubs for SQLAlchemy 1.4" -optional = false -python-versions = ">=3.6" -files = [ - {file = "sqlalchemy2-stubs-0.0.2a35.tar.gz", hash = "sha256:bd5d530697d7e8c8504c7fe792ef334538392a5fb7aa7e4f670bfacdd668a19d"}, - {file = "sqlalchemy2_stubs-0.0.2a35-py3-none-any.whl", hash = "sha256:593784ff9fc0dc2ded1895e3322591689db3be06f3ca006e3ef47640baf2d38a"}, -] - -[package.dependencies] -typing-extensions = ">=3.7.4" - [[package]] name = "termcolor" version = "2.3.0" @@ -2712,4 +2698,4 @@ testing = ["pytest", "pytest-durations"] [metadata] lock-version = "2.0" python-versions = "<3.12,>=3.7.1" -content-hash = "ef713a1192d52c92e45d00697c48d51df216f225476d2a5744954302b438dc8e" +content-hash = "9ac1398cdb5cd011452f152cbd95412139f320881bb2d27855dc52253b313332" diff --git a/pyproject.toml b/pyproject.toml index bdf111925..f70055d1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ memoization = ">=0.3.2,<0.5.0" jsonpath-ng = "^1.5.3" joblib = "^1.0.1" inflection = "^0.5.1" -sqlalchemy = "^1.4" +sqlalchemy = ">=1.4,<3.0" python-dotenv = ">=0.20,<0.22" typing-extensions = "^4.2.0" simplejson = "^3.17.6" @@ -109,7 +109,6 @@ numpy = [ { version = ">=1.22", python = ">=3.8" }, ] requests-mock = "^1.10.0" -sqlalchemy2-stubs = {version = "^0.0.2a32", allow-prereleases = true} types-jsonschema = "^4.17.0.6" types-python-dateutil = "^2.8.19" types-pytz = ">=2022.7.1.2,<2024.0.0.0" @@ -132,9 +131,6 @@ exclude = ".*simpleeval.*" [tool.pytest.ini_options] addopts = '-vvv --ignore=singer_sdk/helpers/_simpleeval.py -m "not external"' -filterwarnings = [ - "error::sqlalchemy.exc.RemovedIn20Warning", -] markers = [ "external: Tests relying on external resources", "windows: Tests that only run on Windows", @@ -190,9 +186,6 @@ fail_under = 82 [tool.mypy] exclude = "tests" files = "singer_sdk" -plugins = [ - "sqlalchemy.ext.mypy.plugin", -] python_version = "3.8" warn_unused_configs = true warn_unused_ignores = true diff --git a/singer_sdk/sinks/sql.py b/singer_sdk/sinks/sql.py index 6b6f8d121..02fd307b9 100644 --- a/singer_sdk/sinks/sql.py +++ b/singer_sdk/sinks/sql.py @@ -322,11 +322,7 @@ def bulk_insert_records( if isinstance(insert_sql, str): insert_sql = sqlalchemy.text(insert_sql) - conformed_records = ( - [self.conform_record(record) for record in records] - if isinstance(records, list) - else (self.conform_record(record) for record in records) - ) + conformed_records = [self.conform_record(record) for record in records] self.logger.info("Inserting with SQL: %s", insert_sql) with self.connector._connect() as conn, conn.begin(): conn.execute(insert_sql, conformed_records) diff --git a/tests/conftest.py b/tests/conftest.py index cb201c9a1..25319c015 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ import typing as t import pytest +from sqlalchemy import __version__ as sqlalchemy_version from singer_sdk import typing as th from singer_sdk.sinks import BatchSink @@ -39,6 +40,11 @@ def pytest_runtest_setup(item): pytest.skip(f"cannot run on platform {system}") +def pytest_report_header() -> list[str]: + """Return a list of strings to be displayed in the header of the report.""" + return [f"sqlalchemy: {sqlalchemy_version}"] + + @pytest.fixture(scope="class") def outdir() -> t.Generator[str, None, None]: """Create a temporary directory for cookiecutters and target output.""" From b2b04461cb3d0a0cca59caa37382c4f08afd3844 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 14:00:03 +0000 Subject: [PATCH 09/19] chore(deps): bump sqlalchemy from 1.4.48 to 2.0.19 (#1869) --- poetry.lock | 108 +++++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/poetry.lock b/poetry.lock index 96a31a297..4c1eb80d2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2273,77 +2273,81 @@ test = ["pytest"] [[package]] name = "sqlalchemy" -version = "1.4.48" +version = "2.0.19" description = "Database Abstraction Library" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" -files = [ - {file = "SQLAlchemy-1.4.48-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:4bac3aa3c3d8bc7408097e6fe8bf983caa6e9491c5d2e2488cfcfd8106f13b6a"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:dbcae0e528d755f4522cad5842f0942e54b578d79f21a692c44d91352ea6d64e"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27m-win32.whl", hash = "sha256:cbbe8b8bffb199b225d2fe3804421b7b43a0d49983f81dc654d0431d2f855543"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27m-win_amd64.whl", hash = "sha256:627e04a5d54bd50628fc8734d5fc6df2a1aa5962f219c44aad50b00a6cdcf965"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9af1db7a287ef86e0f5cd990b38da6bd9328de739d17e8864f1817710da2d217"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:ce7915eecc9c14a93b73f4e1c9d779ca43e955b43ddf1e21df154184f39748e5"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5381ddd09a99638f429f4cbe1b71b025bed318f6a7b23e11d65f3eed5e181c33"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:87609f6d4e81a941a17e61a4c19fee57f795e96f834c4f0a30cee725fc3f81d9"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb0808ad34167f394fea21bd4587fc62f3bd81bba232a1e7fbdfa17e6cfa7cd7"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-win32.whl", hash = "sha256:d53cd8bc582da5c1c8c86b6acc4ef42e20985c57d0ebc906445989df566c5603"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-win_amd64.whl", hash = "sha256:4355e5915844afdc5cf22ec29fba1010166e35dd94a21305f49020022167556b"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:066c2b0413e8cb980e6d46bf9d35ca83be81c20af688fedaef01450b06e4aa5e"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c99bf13e07140601d111a7c6f1fc1519914dd4e5228315bbda255e08412f61a4"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ee26276f12614d47cc07bc85490a70f559cba965fb178b1c45d46ffa8d73fda"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-win32.whl", hash = "sha256:49c312bcff4728bffc6fb5e5318b8020ed5c8b958a06800f91859fe9633ca20e"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-win_amd64.whl", hash = "sha256:cef2e2abc06eab187a533ec3e1067a71d7bbec69e582401afdf6d8cad4ba3515"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:3509159e050bd6d24189ec7af373359f07aed690db91909c131e5068176c5a5d"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc2ab4d9f6d9218a5caa4121bdcf1125303482a1cdcfcdbd8567be8518969c0"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e1ddbbcef9bcedaa370c03771ebec7e39e3944782bef49e69430383c376a250b"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f82d8efea1ca92b24f51d3aea1a82897ed2409868a0af04247c8c1e4fef5890"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-win32.whl", hash = "sha256:e3e98d4907805b07743b583a99ecc58bf8807ecb6985576d82d5e8ae103b5272"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-win_amd64.whl", hash = "sha256:25887b4f716e085a1c5162f130b852f84e18d2633942c8ca40dfb8519367c14f"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:0817c181271b0ce5df1aa20949f0a9e2426830fed5ecdcc8db449618f12c2730"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe1dd2562313dd9fe1778ed56739ad5d9aae10f9f43d9f4cf81d65b0c85168bb"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:68413aead943883b341b2b77acd7a7fe2377c34d82e64d1840860247cec7ff7c"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbde5642104ac6e95f96e8ad6d18d9382aa20672008cf26068fe36f3004491df"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-win32.whl", hash = "sha256:11c6b1de720f816c22d6ad3bbfa2f026f89c7b78a5c4ffafb220e0183956a92a"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-win_amd64.whl", hash = "sha256:eb5464ee8d4bb6549d368b578e9529d3c43265007193597ddca71c1bae6174e6"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:92e6133cf337c42bfee03ca08c62ba0f2d9695618c8abc14a564f47503157be9"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44d29a3fc6d9c45962476b470a81983dd8add6ad26fdbfae6d463b509d5adcda"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:005e942b451cad5285015481ae4e557ff4154dde327840ba91b9ac379be3b6ce"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c8cfe951ed074ba5e708ed29c45397a95c4143255b0d022c7c8331a75ae61f3"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-win32.whl", hash = "sha256:2b9af65cc58726129d8414fc1a1a650dcdd594ba12e9c97909f1f57d48e393d3"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-win_amd64.whl", hash = "sha256:2b562e9d1e59be7833edf28b0968f156683d57cabd2137d8121806f38a9d58f4"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:a1fc046756cf2a37d7277c93278566ddf8be135c6a58397b4c940abf837011f4"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d9b55252d2ca42a09bcd10a697fa041e696def9dfab0b78c0aaea1485551a08"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6dab89874e72a9ab5462997846d4c760cdb957958be27b03b49cf0de5e5c327c"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd8b5ee5a3acc4371f820934b36f8109ce604ee73cc668c724abb054cebcb6e"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-win32.whl", hash = "sha256:eee09350fd538e29cfe3a496ec6f148504d2da40dbf52adefb0d2f8e4d38ccc4"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-win_amd64.whl", hash = "sha256:7ad2b0f6520ed5038e795cc2852eb5c1f20fa6831d73301ced4aafbe3a10e1f6"}, - {file = "SQLAlchemy-1.4.48.tar.gz", hash = "sha256:b47bc287096d989a0838ce96f7d8e966914a24da877ed41a7531d44b55cdb8df"}, +python-versions = ">=3.7" +files = [ + {file = "SQLAlchemy-2.0.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9deaae357edc2091a9ed5d25e9ee8bba98bcfae454b3911adeaf159c2e9ca9e3"}, + {file = "SQLAlchemy-2.0.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0bf0fd65b50a330261ec7fe3d091dfc1c577483c96a9fa1e4323e932961aa1b5"}, + {file = "SQLAlchemy-2.0.19-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d90ccc15ba1baa345796a8fb1965223ca7ded2d235ccbef80a47b85cea2d71a"}, + {file = "SQLAlchemy-2.0.19-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb4e688f6784427e5f9479d1a13617f573de8f7d4aa713ba82813bcd16e259d1"}, + {file = "SQLAlchemy-2.0.19-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:584f66e5e1979a7a00f4935015840be627e31ca29ad13f49a6e51e97a3fb8cae"}, + {file = "SQLAlchemy-2.0.19-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2c69ce70047b801d2aba3e5ff3cba32014558966109fecab0c39d16c18510f15"}, + {file = "SQLAlchemy-2.0.19-cp310-cp310-win32.whl", hash = "sha256:96f0463573469579d32ad0c91929548d78314ef95c210a8115346271beeeaaa2"}, + {file = "SQLAlchemy-2.0.19-cp310-cp310-win_amd64.whl", hash = "sha256:22bafb1da60c24514c141a7ff852b52f9f573fb933b1e6b5263f0daa28ce6db9"}, + {file = "SQLAlchemy-2.0.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d6894708eeb81f6d8193e996257223b6bb4041cb05a17cd5cf373ed836ef87a2"}, + {file = "SQLAlchemy-2.0.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8f2afd1aafded7362b397581772c670f20ea84d0a780b93a1a1529da7c3d369"}, + {file = "SQLAlchemy-2.0.19-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b15afbf5aa76f2241184c1d3b61af1a72ba31ce4161013d7cb5c4c2fca04fd6e"}, + {file = "SQLAlchemy-2.0.19-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fc05b59142445a4efb9c1fd75c334b431d35c304b0e33f4fa0ff1ea4890f92e"}, + {file = "SQLAlchemy-2.0.19-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5831138f0cc06b43edf5f99541c64adf0ab0d41f9a4471fd63b54ae18399e4de"}, + {file = "SQLAlchemy-2.0.19-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3afa8a21a9046917b3a12ffe016ba7ebe7a55a6fc0c7d950beb303c735c3c3ad"}, + {file = "SQLAlchemy-2.0.19-cp311-cp311-win32.whl", hash = "sha256:c896d4e6ab2eba2afa1d56be3d0b936c56d4666e789bfc59d6ae76e9fcf46145"}, + {file = "SQLAlchemy-2.0.19-cp311-cp311-win_amd64.whl", hash = "sha256:024d2f67fb3ec697555e48caeb7147cfe2c08065a4f1a52d93c3d44fc8e6ad1c"}, + {file = "SQLAlchemy-2.0.19-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:89bc2b374ebee1a02fd2eae6fd0570b5ad897ee514e0f84c5c137c942772aa0c"}, + {file = "SQLAlchemy-2.0.19-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd4d410a76c3762511ae075d50f379ae09551d92525aa5bb307f8343bf7c2c12"}, + {file = "SQLAlchemy-2.0.19-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f469f15068cd8351826df4080ffe4cc6377c5bf7d29b5a07b0e717dddb4c7ea2"}, + {file = "SQLAlchemy-2.0.19-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:cda283700c984e699e8ef0fcc5c61f00c9d14b6f65a4f2767c97242513fcdd84"}, + {file = "SQLAlchemy-2.0.19-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:43699eb3f80920cc39a380c159ae21c8a8924fe071bccb68fc509e099420b148"}, + {file = "SQLAlchemy-2.0.19-cp37-cp37m-win32.whl", hash = "sha256:61ada5831db36d897e28eb95f0f81814525e0d7927fb51145526c4e63174920b"}, + {file = "SQLAlchemy-2.0.19-cp37-cp37m-win_amd64.whl", hash = "sha256:57d100a421d9ab4874f51285c059003292433c648df6abe6c9c904e5bd5b0828"}, + {file = "SQLAlchemy-2.0.19-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:16a310f5bc75a5b2ce7cb656d0e76eb13440b8354f927ff15cbaddd2523ee2d1"}, + {file = "SQLAlchemy-2.0.19-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cf7b5e3856cbf1876da4e9d9715546fa26b6e0ba1a682d5ed2fc3ca4c7c3ec5b"}, + {file = "SQLAlchemy-2.0.19-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e7b69d9ced4b53310a87117824b23c509c6fc1f692aa7272d47561347e133b6"}, + {file = "SQLAlchemy-2.0.19-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f9eb4575bfa5afc4b066528302bf12083da3175f71b64a43a7c0badda2be365"}, + {file = "SQLAlchemy-2.0.19-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6b54d1ad7a162857bb7c8ef689049c7cd9eae2f38864fc096d62ae10bc100c7d"}, + {file = "SQLAlchemy-2.0.19-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5d6afc41ca0ecf373366fd8e10aee2797128d3ae45eb8467b19da4899bcd1ee0"}, + {file = "SQLAlchemy-2.0.19-cp38-cp38-win32.whl", hash = "sha256:430614f18443b58ceb9dedec323ecddc0abb2b34e79d03503b5a7579cd73a531"}, + {file = "SQLAlchemy-2.0.19-cp38-cp38-win_amd64.whl", hash = "sha256:eb60699de43ba1a1f77363f563bb2c652f7748127ba3a774f7cf2c7804aa0d3d"}, + {file = "SQLAlchemy-2.0.19-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a752b7a9aceb0ba173955d4f780c64ee15a1a991f1c52d307d6215c6c73b3a4c"}, + {file = "SQLAlchemy-2.0.19-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7351c05db355da112e056a7b731253cbeffab9dfdb3be1e895368513c7d70106"}, + {file = "SQLAlchemy-2.0.19-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa51ce4aea583b0c6b426f4b0563d3535c1c75986c4373a0987d84d22376585b"}, + {file = "SQLAlchemy-2.0.19-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae7473a67cd82a41decfea58c0eac581209a0aa30f8bc9190926fbf628bb17f7"}, + {file = "SQLAlchemy-2.0.19-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:851a37898a8a39783aab603c7348eb5b20d83c76a14766a43f56e6ad422d1ec8"}, + {file = "SQLAlchemy-2.0.19-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539010665c90e60c4a1650afe4ab49ca100c74e6aef882466f1de6471d414be7"}, + {file = "SQLAlchemy-2.0.19-cp39-cp39-win32.whl", hash = "sha256:f82c310ddf97b04e1392c33cf9a70909e0ae10a7e2ddc1d64495e3abdc5d19fb"}, + {file = "SQLAlchemy-2.0.19-cp39-cp39-win_amd64.whl", hash = "sha256:8e712cfd2e07b801bc6b60fdf64853bc2bd0af33ca8fa46166a23fe11ce0dbb0"}, + {file = "SQLAlchemy-2.0.19-py3-none-any.whl", hash = "sha256:314145c1389b021a9ad5aa3a18bac6f5d939f9087d7fc5443be28cba19d2c972"}, + {file = "SQLAlchemy-2.0.19.tar.gz", hash = "sha256:77a14fa20264af73ddcdb1e2b9c5a829b8cc6b8304d0f093271980e36c200a3f"}, ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\")"} +greenlet = {version = "!=0.4.17", markers = "platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\""} importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +typing-extensions = ">=4.2.0" [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] -asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] -mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] mssql = ["pyodbc"] mssql-pymssql = ["pymssql"] mssql-pyodbc = ["pyodbc"] -mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] -mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] +mypy = ["mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0)"] mysql-connector = ["mysql-connector-python"] -oracle = ["cx-oracle (>=7)", "cx-oracle (>=7,<8)"] +oracle = ["cx-oracle (>=7)"] +oracle-oracledb = ["oracledb (>=1.0.1)"] postgresql = ["psycopg2 (>=2.7)"] postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] -postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] +postgresql-pg8000 = ["pg8000 (>=1.29.1)"] +postgresql-psycopg = ["psycopg (>=3.0.7)"] postgresql-psycopg2binary = ["psycopg2-binary"] postgresql-psycopg2cffi = ["psycopg2cffi"] -pymysql = ["pymysql", "pymysql (<1)"] +postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] +pymysql = ["pymysql"] sqlcipher = ["sqlcipher3-binary"] [[package]] From 4ece8a1091b5898173b067329ddd14217b5fdfba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 14:10:35 +0000 Subject: [PATCH 10/19] chore(deps-dev): bump types-python-dateutil from 2.8.19.13 to 2.8.19.14 (#1868) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4c1eb80d2..80ae08c2b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2463,13 +2463,13 @@ files = [ [[package]] name = "types-python-dateutil" -version = "2.8.19.13" +version = "2.8.19.14" description = "Typing stubs for python-dateutil" optional = false python-versions = "*" files = [ - {file = "types-python-dateutil-2.8.19.13.tar.gz", hash = "sha256:09a0275f95ee31ce68196710ed2c3d1b9dc42e0b61cc43acc369a42cb939134f"}, - {file = "types_python_dateutil-2.8.19.13-py3-none-any.whl", hash = "sha256:0b0e7c68e7043b0354b26a1e0225cb1baea7abb1b324d02b50e2d08f1221043f"}, + {file = "types-python-dateutil-2.8.19.14.tar.gz", hash = "sha256:1f4f10ac98bb8b16ade9dbee3518d9ace017821d94b057a425b069f834737f4b"}, + {file = "types_python_dateutil-2.8.19.14-py3-none-any.whl", hash = "sha256:f977b8de27787639986b4e28963263fd0e5158942b3ecef91b9335c130cb1ce9"}, ] [[package]] From 9287c0df11fb3da2429962354e36d5d7aa9332c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 14:30:54 +0000 Subject: [PATCH 11/19] chore(deps-dev): bump types-requests from 2.31.0.1 to 2.31.0.2 (#1870) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 80ae08c2b..11c64a016 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2496,13 +2496,13 @@ files = [ [[package]] name = "types-requests" -version = "2.31.0.1" +version = "2.31.0.2" description = "Typing stubs for requests" optional = false python-versions = "*" files = [ - {file = "types-requests-2.31.0.1.tar.gz", hash = "sha256:3de667cffa123ce698591de0ad7db034a5317457a596eb0b4944e5a9d9e8d1ac"}, - {file = "types_requests-2.31.0.1-py3-none-any.whl", hash = "sha256:afb06ef8f25ba83d59a1d424bd7a5a939082f94b94e90ab5e6116bd2559deaa3"}, + {file = "types-requests-2.31.0.2.tar.gz", hash = "sha256:6aa3f7faf0ea52d728bb18c0a0d1522d9bfd8c72d26ff6f61bfc3d06a411cf40"}, + {file = "types_requests-2.31.0.2-py3-none-any.whl", hash = "sha256:56d181c85b5925cbc59f4489a57e72a8b2166f18273fd8ba7b6fe0c0b986f12a"}, ] [package.dependencies] From 33e9df990476e640052994d8dadd8ae1334d4ef4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 15:49:10 +0000 Subject: [PATCH 12/19] chore(deps-dev): bump types-pyyaml from 6.0.12.10 to 6.0.12.11 (#1866) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 11c64a016..88825a59b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2485,13 +2485,13 @@ files = [ [[package]] name = "types-pyyaml" -version = "6.0.12.10" +version = "6.0.12.11" description = "Typing stubs for PyYAML" optional = false python-versions = "*" files = [ - {file = "types-PyYAML-6.0.12.10.tar.gz", hash = "sha256:ebab3d0700b946553724ae6ca636ea932c1b0868701d4af121630e78d695fc97"}, - {file = "types_PyYAML-6.0.12.10-py3-none-any.whl", hash = "sha256:662fa444963eff9b68120d70cda1af5a5f2aa57900003c2006d7626450eaae5f"}, + {file = "types-PyYAML-6.0.12.11.tar.gz", hash = "sha256:7d340b19ca28cddfdba438ee638cd4084bde213e501a3978738543e27094775b"}, + {file = "types_PyYAML-6.0.12.11-py3-none-any.whl", hash = "sha256:a461508f3096d1d5810ec5ab95d7eeecb651f3a15b71959999988942063bf01d"}, ] [[package]] From c444601380d2261426095db77538d4cfabe76876 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 16:00:08 +0000 Subject: [PATCH 13/19] chore(deps-dev): bump types-simplejson from 3.19.0.1 to 3.19.0.2 (#1867) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 88825a59b..6f155534b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2510,13 +2510,13 @@ types-urllib3 = "*" [[package]] name = "types-simplejson" -version = "3.19.0.1" +version = "3.19.0.2" description = "Typing stubs for simplejson" optional = false python-versions = "*" files = [ - {file = "types-simplejson-3.19.0.1.tar.gz", hash = "sha256:0233df016477bd58a2525df79ac8a34b079910d51ca45ec4f09a94ae58222f02"}, - {file = "types_simplejson-3.19.0.1-py3-none-any.whl", hash = "sha256:0083e84d43b6b36e8af6eb77e6b41440f2aec8842d16cee0f828fb5622196f4f"}, + {file = "types-simplejson-3.19.0.2.tar.gz", hash = "sha256:ebc81f886f89d99d6b80c726518aa2228bc77c26438f18fd81455e4f79f8ee1b"}, + {file = "types_simplejson-3.19.0.2-py3-none-any.whl", hash = "sha256:8ba093dc7884f59b3e62aed217144085e675a269debc32678fd80e0b43b2b86f"}, ] [[package]] From 52802e648cc1b97de5abfdbefcb8764445a9ff7b Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Fri, 21 Jul 2023 14:40:50 -0600 Subject: [PATCH 14/19] fix: Add deprecation warning when importing legacy testing helpers (#1838) * fix: Add deprecation warning when import legacy testing helpers * Document deprecation * Fix names * Test missing module attributes --- docs/deprecation.md | 2 ++ singer_sdk/testing/__init__.py | 38 ++++++++++++++++++++++++++++++---- tests/core/test_testing.py | 21 +++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 tests/core/test_testing.py diff --git a/docs/deprecation.md b/docs/deprecation.md index 089355fd9..b27e61b2b 100644 --- a/docs/deprecation.md +++ b/docs/deprecation.md @@ -11,3 +11,5 @@ incompatible way, following their deprecation, as indicated in the [`RESTStream.get_new_paginator`](singer_sdk.RESTStream.get_new_paginator). See the [migration guide](./guides/pagination-classes.md) for more information. + +- The `singer_sdk.testing.get_standard_tap_tests` and `singer_sdk.testing.get_standard_target_tests` functions will be removed. Replace them with `singer_sdk.testing.get_tap_test_class` and `singer_sdk.testing.get_target_test_class` functions respective to generate a richer test suite. diff --git a/singer_sdk/testing/__init__.py b/singer_sdk/testing/__init__.py index 24ce4ac9f..83ca9aacc 100644 --- a/singer_sdk/testing/__init__.py +++ b/singer_sdk/testing/__init__.py @@ -2,13 +2,14 @@ from __future__ import annotations +import typing as t +import warnings + from .config import SuiteConfig from .factory import get_tap_test_class, get_target_test_class from .legacy import ( _get_tap_catalog, _select_all, - get_standard_tap_tests, - get_standard_target_tests, sync_end_to_end, tap_sync_test, tap_to_target_sync_test, @@ -16,13 +17,42 @@ ) from .runners import SingerTestRunner, TapTestRunner, TargetTestRunner + +def __getattr__(name: str) -> t.Any: # noqa: ANN401 + if name == "get_standard_tap_tests": + warnings.warn( + "The function singer_sdk.testing.get_standard_tap_tests is deprecated " + "and will be removed in a future release. Use get_tap_test_class instead.", + DeprecationWarning, + stacklevel=2, + ) + + from .legacy import get_standard_tap_tests + + return get_standard_tap_tests + + if name == "get_standard_target_tests": + warnings.warn( + "The function singer_sdk.testing.get_standard_target_tests is deprecated " + "and will be removed in a future release. Use get_target_test_class " + "instead.", + DeprecationWarning, + stacklevel=2, + ) + + from .legacy import get_standard_target_tests + + return get_standard_target_tests + + msg = f"module {__name__} has no attribute {name}" + raise AttributeError(msg) + + __all__ = [ "get_tap_test_class", "get_target_test_class", "_get_tap_catalog", "_select_all", - "get_standard_tap_tests", - "get_standard_target_tests", "sync_end_to_end", "tap_sync_test", "tap_to_target_sync_test", diff --git a/tests/core/test_testing.py b/tests/core/test_testing.py new file mode 100644 index 000000000..02672d9ad --- /dev/null +++ b/tests/core/test_testing.py @@ -0,0 +1,21 @@ +"""Test the plugin testing helpers.""" + +from __future__ import annotations + +import pytest + + +def test_module_deprecations(): + with pytest.deprecated_call(): + from singer_sdk.testing import get_standard_tap_tests # noqa: F401 + + with pytest.deprecated_call(): + from singer_sdk.testing import get_standard_target_tests # noqa: F401 + + from singer_sdk import testing + + with pytest.raises( + AttributeError, + match="module singer_sdk.testing has no attribute", + ): + testing.foo # noqa: B018 From d0efb8ac3cb3975426d121bc98a2ea106e570de0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jul 2023 14:23:14 +0000 Subject: [PATCH 15/19] chore(deps-dev): bump types-jsonschema from 4.17.0.9 to 4.17.0.10 (#1874) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6f155534b..02cfef947 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2452,13 +2452,13 @@ files = [ [[package]] name = "types-jsonschema" -version = "4.17.0.9" +version = "4.17.0.10" description = "Typing stubs for jsonschema" optional = false python-versions = "*" files = [ - {file = "types-jsonschema-4.17.0.9.tar.gz", hash = "sha256:ddbbf84a37ba19f486e43d2a4ab239c9e49aebb5cc99a17a5d59f54568373376"}, - {file = "types_jsonschema-4.17.0.9-py3-none-any.whl", hash = "sha256:ec83f48c5ce5d3ea6955c3617d8c903e5ba3db8debea0c7f5c8e9bd60d782a9e"}, + {file = "types-jsonschema-4.17.0.10.tar.gz", hash = "sha256:8e979db34d69bc9f9b3d6e8b89bdbc60b3a41cfce4e1fb87bf191d205c7f5098"}, + {file = "types_jsonschema-4.17.0.10-py3-none-any.whl", hash = "sha256:3aa2a89afbd9eaa6ce0c15618b36f02692a621433889ce73014656f7d8caf971"}, ] [[package]] From aece0a3851398b457650911e82cdcdda6f8949a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jul 2023 14:32:58 +0000 Subject: [PATCH 16/19] chore(deps): bump pip from 23.2 to 23.2.1 in /.github/workflows (#1873) --- .github/workflows/constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt index 246d39c69..26aef2cc7 100644 --- a/.github/workflows/constraints.txt +++ b/.github/workflows/constraints.txt @@ -1,4 +1,4 @@ -pip==23.2 +pip==23.2.1 poetry==1.5.1 pre-commit==3.3.3 nox==2023.4.22 From 019902ed7015c20b2b8b7dad04edd7e8b16468bd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 25 Jul 2023 10:22:36 -0600 Subject: [PATCH 17/19] chore: pre-commit autoupdate (#1875) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edgar Ramírez Mondragón --- .pre-commit-config.yaml | 2 +- singer_sdk/connectors/sql.py | 2 +- singer_sdk/helpers/_flattening.py | 18 +++++++++--------- singer_sdk/streams/core.py | 9 ++++++--- singer_sdk/streams/graphql.py | 6 ++++-- singer_sdk/typing.py | 5 ++--- tests/core/test_jsonschema_helpers.py | 2 +- tests/samples/test_tap_sqlite.py | 2 +- 8 files changed, 25 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c8a18afe..95bc2bad7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,7 @@ repos: - id: check-readthedocs - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.278 + rev: v0.0.280 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index aecfbb0c1..e9a65cf80 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -179,7 +179,7 @@ def get_sqlalchemy_url(self, config: dict[str, t.Any]) -> str: @staticmethod def to_jsonschema_type( sql_type: ( - str + str # noqa: ANN401 | sqlalchemy.types.TypeEngine | type[sqlalchemy.types.TypeEngine] | t.Any diff --git a/singer_sdk/helpers/_flattening.py b/singer_sdk/helpers/_flattening.py index 29ef35cc2..02585dd80 100644 --- a/singer_sdk/helpers/_flattening.py +++ b/singer_sdk/helpers/_flattening.py @@ -252,15 +252,15 @@ def _flatten_schema( # noqa: C901 else: items.append((new_key, v)) elif len(v.values()) > 0: - if list(v.values())[0][0]["type"] == "string": - list(v.values())[0][0]["type"] = ["null", "string"] - items.append((new_key, list(v.values())[0][0])) - elif list(v.values())[0][0]["type"] == "array": - list(v.values())[0][0]["type"] = ["null", "array"] - items.append((new_key, list(v.values())[0][0])) - elif list(v.values())[0][0]["type"] == "object": - list(v.values())[0][0]["type"] = ["null", "object"] - items.append((new_key, list(v.values())[0][0])) + if next(iter(v.values()))[0]["type"] == "string": + next(iter(v.values()))[0]["type"] = ["null", "string"] + items.append((new_key, next(iter(v.values()))[0])) + elif next(iter(v.values()))[0]["type"] == "array": + next(iter(v.values()))[0]["type"] = ["null", "array"] + items.append((new_key, next(iter(v.values()))[0])) + elif next(iter(v.values()))[0]["type"] == "object": + next(iter(v.values()))[0]["type"] = ["null", "object"] + items.append((new_key, next(iter(v.values()))[0])) # Sort and check for duplicates def _key_func(item): diff --git a/singer_sdk/streams/core.py b/singer_sdk/streams/core.py index 35bc2e79e..53e9533ff 100644 --- a/singer_sdk/streams/core.py +++ b/singer_sdk/streams/core.py @@ -217,7 +217,10 @@ def is_timestamp_replication_key(self) -> bool: type_dict = self.schema.get("properties", {}).get(self.replication_key) return is_datetime_type(type_dict) - def get_starting_replication_key_value(self, context: dict | None) -> t.Any | None: + def get_starting_replication_key_value( + self, + context: dict | None, + ) -> t.Any | None: # noqa: ANN401 """Get starting replication key. Will return the value of the stream's replication key when `--state` is passed. @@ -385,7 +388,7 @@ def _write_starting_replication_value(self, context: dict | None) -> None: def get_replication_key_signpost( self, context: dict | None, # noqa: ARG002 - ) -> datetime.datetime | t.Any | None: + ) -> datetime.datetime | t.Any | None: # noqa: ANN401 """Get the replication signpost. For timestamp-based replication keys, this defaults to `utc_now()`. For @@ -1255,7 +1258,7 @@ def get_child_context(self, record: dict, context: dict | None) -> dict | None: Raises: NotImplementedError: If the stream has children but this method is not - overriden. + overridden. """ if context is None: for child_stream in self.child_streams: diff --git a/singer_sdk/streams/graphql.py b/singer_sdk/streams/graphql.py index 01e5d41ee..fde4f99b9 100644 --- a/singer_sdk/streams/graphql.py +++ b/singer_sdk/streams/graphql.py @@ -8,8 +8,10 @@ from singer_sdk.helpers._classproperty import classproperty from singer_sdk.streams.rest import RESTStream +_TToken = t.TypeVar("_TToken") -class GraphQLStream(RESTStream, metaclass=abc.ABCMeta): + +class GraphQLStream(RESTStream, t.Generic[_TToken], metaclass=abc.ABCMeta): """Abstract base class for API-type streams. GraphQL streams inherit from the class `GraphQLStream`, which in turn inherits from @@ -43,7 +45,7 @@ def query(self) -> str: def prepare_request_payload( self, context: dict | None, - next_page_token: t.Any | None, + next_page_token: _TToken | None, ) -> dict | None: """Prepare the data payload for the GraphQL API request. diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index a8f654c90..850e30e8e 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -965,9 +965,8 @@ def _jsonschema_type_check(jsonschema_type: dict, type_check: tuple[str]) -> boo for schema_type in jsonschema_type["type"]: if schema_type in type_check: return True - else: - if jsonschema_type.get("type") in type_check: # noqa: PLR5501 - return True + elif jsonschema_type.get("type") in type_check: + return True if any( _jsonschema_type_check(t, type_check) for t in jsonschema_type.get("anyOf", ()) diff --git a/tests/core/test_jsonschema_helpers.py b/tests/core/test_jsonschema_helpers.py index 3e4ba6eca..e1369dcba 100644 --- a/tests/core/test_jsonschema_helpers.py +++ b/tests/core/test_jsonschema_helpers.py @@ -490,7 +490,7 @@ def test_property_creation( property_dict = property_obj.to_dict() assert property_dict == expected_jsonschema for check_fn in TYPE_FN_CHECKS: - property_name = list(property_dict.keys())[0] + property_name = next(iter(property_dict.keys())) property_node = property_dict[property_name] if check_fn in type_fn_checks_true: assert ( diff --git a/tests/samples/test_tap_sqlite.py b/tests/samples/test_tap_sqlite.py index bceaa34b1..e2b1940da 100644 --- a/tests/samples/test_tap_sqlite.py +++ b/tests/samples/test_tap_sqlite.py @@ -48,7 +48,7 @@ def test_tap_sqlite_cli(sqlite_sample_db_config: dict[str, t.Any], tmp_path: Pat def test_sql_metadata(sqlite_sample_tap: SQLTap): stream = t.cast(SQLStream, sqlite_sample_tap.streams["main-t1"]) detected_metadata = stream.catalog_entry["metadata"] - detected_root_md = [md for md in detected_metadata if md["breadcrumb"] == []][0] + detected_root_md = next(md for md in detected_metadata if md["breadcrumb"] == []) detected_root_md = detected_root_md["metadata"] translated_metadata = StreamMetadata.from_dict(detected_root_md) assert detected_root_md["schema-name"] == translated_metadata.schema_name From 6576f5276e46fc634806ceaa25b71fc7a93a94c3 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Wed, 26 Jul 2023 14:00:32 -0600 Subject: [PATCH 18/19] fix(targets): Handle missing record properties in SQL sinks (#1865) * fix(targets): Handle missing record properties in SQL sinks * Update singer_sdk/sinks/sql.py * Add test to target suite --- singer_sdk/sinks/sql.py | 14 +++++++-- singer_sdk/testing/suites.py | 2 ++ .../record_missing_fields.singer | 4 +++ singer_sdk/testing/target_tests.py | 6 ++++ tests/samples/test_target_sqlite.py | 29 +++++++++++++++++++ 5 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 singer_sdk/testing/target_test_streams/record_missing_fields.singer diff --git a/singer_sdk/sinks/sql.py b/singer_sdk/sinks/sql.py index 02fd307b9..9dac99b1e 100644 --- a/singer_sdk/sinks/sql.py +++ b/singer_sdk/sinks/sql.py @@ -323,10 +323,20 @@ def bulk_insert_records( insert_sql = sqlalchemy.text(insert_sql) conformed_records = [self.conform_record(record) for record in records] + property_names = list(self.conform_schema(schema)["properties"].keys()) + + # Create new record dicts with missing properties filled in with None + new_records = [ + {name: record.get(name) for name in property_names} + for record in conformed_records + ] + self.logger.info("Inserting with SQL: %s", insert_sql) + with self.connector._connect() as conn, conn.begin(): - conn.execute(insert_sql, conformed_records) - return len(conformed_records) if isinstance(conformed_records, list) else None + result = conn.execute(insert_sql, new_records) + + return result.rowcount def merge_upsert_from_table( self, diff --git a/singer_sdk/testing/suites.py b/singer_sdk/testing/suites.py index 0f8a9fabe..d795cf153 100644 --- a/singer_sdk/testing/suites.py +++ b/singer_sdk/testing/suites.py @@ -37,6 +37,7 @@ TargetOptionalAttributes, TargetRecordBeforeSchemaTest, TargetRecordMissingKeyProperty, + TargetRecordMissingOptionalFields, TargetSchemaNoProperties, TargetSchemaUpdates, TargetSpecialCharsInAttributes, @@ -103,6 +104,7 @@ class TestSuite: TargetOptionalAttributes, TargetRecordBeforeSchemaTest, TargetRecordMissingKeyProperty, + TargetRecordMissingOptionalFields, TargetSchemaNoProperties, TargetSchemaUpdates, TargetSpecialCharsInAttributes, diff --git a/singer_sdk/testing/target_test_streams/record_missing_fields.singer b/singer_sdk/testing/target_test_streams/record_missing_fields.singer new file mode 100644 index 000000000..a398f6bd6 --- /dev/null +++ b/singer_sdk/testing/target_test_streams/record_missing_fields.singer @@ -0,0 +1,4 @@ +{"type": "SCHEMA", "stream": "record_missing_fields", "key_properties": ["id"], "schema": {"type": "object", "properties": {"id": {"type": "integer"}, "optional": {"type": "string"}}, "required": ["id"]}} +{"type": "RECORD", "stream": "record_missing_fields", "record": {"id": 1, "optional": "now you see me"}} +{"type": "RECORD", "stream": "record_missing_fields", "record": {"id": 2}} +{"type": "STATE", "value": {}} diff --git a/singer_sdk/testing/target_tests.py b/singer_sdk/testing/target_tests.py index 8412329c5..96e0b0d59 100644 --- a/singer_sdk/testing/target_tests.py +++ b/singer_sdk/testing/target_tests.py @@ -139,3 +139,9 @@ class TargetSpecialCharsInAttributes(TargetFileTestTemplate): """Test Target handles special chars in attributes.""" name = "special_chars_in_attributes" + + +class TargetRecordMissingOptionalFields(TargetFileTestTemplate): + """Test Target handles record missing optional fields.""" + + name = "record_missing_fields" diff --git a/tests/samples/test_target_sqlite.py b/tests/samples/test_target_sqlite.py index a7ca3b3c5..abf1bccaf 100644 --- a/tests/samples/test_target_sqlite.py +++ b/tests/samples/test_target_sqlite.py @@ -396,6 +396,35 @@ def test_sqlite_column_no_morph(sqlite_sample_target: SQLTarget): target_sync_test(sqlite_sample_target, input=StringIO(tap_output_b), finalize=True) +def test_record_with_missing_properties( + sqlite_sample_target: SQLTarget, +): + """Test handling of records with missing properties.""" + tap_output = "\n".join( + json.dumps(msg) + for msg in [ + { + "type": "SCHEMA", + "stream": "test_stream", + "schema": { + "type": "object", + "properties": { + "id": {"type": "integer"}, + "name": {"type": "string"}, + }, + }, + "key_properties": ["id"], + }, + { + "type": "RECORD", + "stream": "test_stream", + "record": {"id": 1}, + }, + ] + ) + target_sync_test(sqlite_sample_target, input=StringIO(tap_output), finalize=True) + + @pytest.mark.parametrize( "stream_name,schema,key_properties,expected_dml", [ From b0a86213683a311a47640823fa52dc2afa3af8f6 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Thu, 27 Jul 2023 10:39:50 -0600 Subject: [PATCH 19/19] refactor: Use `importlib.resources` instead of `__file__` to retrieve sample Singer output files (#1877) --- singer_sdk/helpers/_compat.py | 7 ++++++- singer_sdk/testing/target_test_streams/__init__.py | 1 + singer_sdk/testing/templates.py | 6 ++++-- 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 singer_sdk/testing/target_test_streams/__init__.py diff --git a/singer_sdk/helpers/_compat.py b/singer_sdk/helpers/_compat.py index 87033ea4c..20b7a399a 100644 --- a/singer_sdk/helpers/_compat.py +++ b/singer_sdk/helpers/_compat.py @@ -11,4 +11,9 @@ from importlib import metadata from typing import final # noqa: ICN003 -__all__ = ["metadata", "final"] +if sys.version_info < (3, 9): + import importlib_resources as resources +else: + from importlib import resources + +__all__ = ["metadata", "final", "resources"] diff --git a/singer_sdk/testing/target_test_streams/__init__.py b/singer_sdk/testing/target_test_streams/__init__.py new file mode 100644 index 000000000..14d313288 --- /dev/null +++ b/singer_sdk/testing/target_test_streams/__init__.py @@ -0,0 +1 @@ +"""Singer output samples, used for testing target behavior.""" diff --git a/singer_sdk/testing/templates.py b/singer_sdk/testing/templates.py index b43d37830..bfb54c360 100644 --- a/singer_sdk/testing/templates.py +++ b/singer_sdk/testing/templates.py @@ -7,6 +7,9 @@ import warnings from pathlib import Path +from singer_sdk.helpers._compat import resources +from singer_sdk.testing import target_test_streams + if t.TYPE_CHECKING: from singer_sdk.streams import Stream @@ -334,5 +337,4 @@ def singer_filepath(self) -> Path: Returns: The expected Path to this tests singer file. """ - current_dir = Path(__file__).resolve().parent - return current_dir / "target_test_streams" / f"{self.name}.singer" + return resources.files(target_test_streams).joinpath(f"{self.name}.singer") # type: ignore[no-any-return] # noqa: E501