dataset crawler filtering not working with [sc-29743] (#1035)

metaphor/bigquery/README.md

@@ -112,6 +112,7 @@ See [Output Config](../common/docs/output.md) for more information.
 #### Filtering
 
 See [Filter Config](../common/docs/filter.md) for more information on the optional `filter` config.
+> NOTE: While the filter config uses `database -> schema -> table/view` hierarchy, BigQuery uses `project -> database -> table/view` hierarchy. Please use project id at the top level in the include/exclude of the filter config.
 
 #### Tag Assignment
 

metaphor/snowflake/extractor.py

@@ -141,11 +141,11 @@ async def extract(self) -> Collection[ENTITY_TYPES]:
         with self._conn:
             cursor = self._conn.cursor()
 
-            databases = (
-                self.fetch_databases(cursor)
-                if self._filter.includes is None
-                else list(self._filter.includes.keys())
-            )
+            databases = [
+                db
+                for db in self.fetch_databases(cursor)
+                if self._filter.include_database(db)
+            ]
             logger.info(f"Databases to include: {databases}")
 
             shared_databases = self._fetch_shared_databases(cursor)

metaphor/snowflake/profile/extractor.py

@@ -86,11 +86,11 @@ async def extract(self) -> Collection[ENTITY_TYPES]:
         with self._conn:
             cursor = self._conn.cursor()
 
-            databases = (
-                SnowflakeExtractor.fetch_databases(cursor)
-                if self._filter.includes is None
-                else list(self._filter.includes.keys())
-            )
+            databases = [
+                db
+                for db in SnowflakeExtractor.fetch_databases(cursor)
+                if self._filter.include_database(db)
+            ]
 
             for database in databases:
                 tables = self._fetch_tables(cursor, database)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.150"
+version = "0.14.151"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]