-
Notifications
You must be signed in to change notification settings - Fork 4
/
config.py
60 lines (36 loc) · 1.4 KB
/
config.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
from datetime import datetime, timezone
from typing import List, Optional
from pydantic.dataclasses import dataclass
from metaphor.common.base_extractor import BaseConfig
from metaphor.common.dataclass import ConnectorConfig
from metaphor.common.models import DeserializableDatasetLogicalID
@dataclass(config=ConnectorConfig)
class DataQualityMonitorTarget:
dataset: Optional[str]
column: Optional[str]
@dataclass(config=ConnectorConfig)
class DataQualityMonitor:
title: str
# Change type to Literal["PASSED", "WARNING", "ERROR"] in Python 3.8+
status: str
targets: List[DataQualityMonitorTarget]
description: Optional[str] = None
url: Optional[str] = None
owner: Optional[str] = None
# Change type to Optional[Literal["LOW", "MEDIUM", "HIGH"]] in Python 3.8+
severity: Optional[str] = None
last_run: Optional[datetime] = datetime.now(tz=timezone.utc)
value: Optional[float] = None
@dataclass(config=ConnectorConfig)
class DataQuality:
monitors: List[DataQualityMonitor]
# Change type to Optional[Literal["SODA", "LIGHTUP", "BIGEYE"]]
provider: Optional[str] = None
url: Optional[str] = None
@dataclass(config=ConnectorConfig)
class DatasetDataQuality:
id: DeserializableDatasetLogicalID
data_quality: DataQuality
@dataclass(config=ConnectorConfig)
class CustomDataQualityConfig(BaseConfig):
datasets: List[DatasetDataQuality]