Skip to content

Commit

Permalink
less ignored lines == better coverage
Browse files Browse the repository at this point in the history
  • Loading branch information
usefulalgorithm committed Oct 30, 2024
1 parent d15b5e5 commit 960d970
Showing 1 changed file with 15 additions and 32 deletions.
47 changes: 15 additions & 32 deletions metaphor/great_expectations/extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,19 +124,7 @@ def _parse_suite_result(
execution_engine = datasource.get_execution_engine()

# TODO: support PandasExecutionEngine
if isinstance(execution_engine, PandasExecutionEngine):
logger.warning(
"PandasExecutionEngine not supported, not parsing this validation result"
)
return

# TODO: support SparkDFExecutionEngine
if isinstance(execution_engine, SparkDFExecutionEngine):
logger.warning(
"SparkDFExecution not supported, not parsing this validation result"
)
return

if not isinstance(execution_engine, SqlAlchemyExecutionEngine):
logger.warning(
f"Cannot process execution engine: {execution_engine}, not parsing this validation result"
Expand All @@ -159,26 +147,21 @@ def _parse_sql_execution_engine_result(
batch_spec: BatchSpec = validation_result.meta["batch_spec"]
logger.info(f"batch spec: {batch_spec}")

if "query" in batch_spec:
# This is a RuntimeQueryBatchSpec, we should parse the query and see what datasets
# are referenced in it.
logger.warning(
"RuntimeQueryBatchSpec not supported, not parsing this validation result"
)
return

if "batch_data" in batch_spec:
logger.warning(
"RuntimeDataBatchSpec not supported, not parsing this validation result"
)
return

if "schema_name" not in batch_spec and "table_name" not in batch_spec:
# At this point the only batch spec we care is SqlAlchemyDatasourceBatchSpec,
# anything else we are not parsing.
logger.warning(
f"Cannot parse batch spec {batch_spec}, ignoring this validation result"
)
if (
"query" in batch_spec
or "batch_data" in batch_spec
or "schema_name" not in batch_spec
and "table_name" not in batch_spec
):
# If "query" is in batch spec, then it is a RuntimeQueryBatchSpec, we should parse
# the query and see what datasets are referenced in it.
#
# If "batch_data" is in batch spec, then it is a RuntimeDataBatchSpec and it's just
# a file in the filesystem.
#
# If batch spec does not have "schema_name" nor "table_name", there's no way for us
# to get a fully qualified name for the dataset, and we're just gonna ignore it.
logger.warning("Unsupported batch spec, not parsing this validation result")
return

url = execution_engine.engine.url
Expand Down

0 comments on commit 960d970

Please sign in to comment.