From e663489a8b30197222fa3a4a1478f69d3fb4d0fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BCl=20Bonet?= Date: Sun, 31 Mar 2024 00:07:33 +0100 Subject: [PATCH 1/4] feat: prepare_table_columns --- singer_sdk/connectors/sql.py | 38 +++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index 814fdc8b2..65380d3bd 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -797,13 +797,41 @@ def prepare_table( as_temp_table=as_temp_table, ) return + + self.prepare_table_columns( + full_table_name=full_table_name, + schema=schema, + ) + def prepare_table_columns( + self, + full_table_name: str, + schema: dict, + ) -> None: + """Adapt target table columns to provided schema if possible. The advantage of encapsulating this + in a separate method is that we reduce the number of calls to the database by calling get_table_columns() + only once per table. + + Args: + full_table_name: the target table name. + schema: the JSON Schema for the table. + """ + columns = self.get_table_columns( + full_table_name=full_table_name, + ) for property_name, property_def in schema["properties"].items(): - self.prepare_column( - full_table_name, - property_name, - self.to_sql_type(property_def), - ) + if property_name not in columns: + self._create_empty_column( + full_table_name=full_table_name, + column_name=property_name, + sql_type=self.to_sql_type(property_def), + ) + else: + self._adapt_column_type( + full_table_name, + column_name=property_name, + sql_type=self.to_sql_type(property_def), + ) def prepare_column( self, From c1999dfcd32d670769f5c2242e0aff241e8d5e7d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 30 Mar 2024 23:23:38 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- singer_sdk/connectors/sql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index 65380d3bd..81ca8f333 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -797,7 +797,7 @@ def prepare_table( as_temp_table=as_temp_table, ) return - + self.prepare_table_columns( full_table_name=full_table_name, schema=schema, @@ -810,8 +810,8 @@ def prepare_table_columns( ) -> None: """Adapt target table columns to provided schema if possible. The advantage of encapsulating this in a separate method is that we reduce the number of calls to the database by calling get_table_columns() - only once per table. - + only once per table. + Args: full_table_name: the target table name. schema: the JSON Schema for the table. From c68264131ae23b1eaee8e0e675b331f432f18035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Fri, 12 Apr 2024 17:11:38 -0600 Subject: [PATCH 3/4] Update singer_sdk/connectors/sql.py --- singer_sdk/connectors/sql.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index b887b094d..d89f55829 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -808,9 +808,11 @@ def prepare_table_columns( full_table_name: str, schema: dict, ) -> None: - """Adapt target table columns to provided schema if possible. The advantage of encapsulating this - in a separate method is that we reduce the number of calls to the database by calling get_table_columns() - only once per table. + """Adapt target table columns to provided schema if possible. + + The advantage of encapsulating this in a separate method is that we reduce + the number of calls to the database by calling get_table_columns() only once + per table. Args: full_table_name: the target table name. From ba2d93f5d942919f879aa08681036c59b2f16b17 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 23:12:02 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- singer_sdk/connectors/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py index d89f55829..6755483f4 100644 --- a/singer_sdk/connectors/sql.py +++ b/singer_sdk/connectors/sql.py @@ -809,7 +809,7 @@ def prepare_table_columns( schema: dict, ) -> None: """Adapt target table columns to provided schema if possible. - + The advantage of encapsulating this in a separate method is that we reduce the number of calls to the database by calling get_table_columns() only once per table.