diff --git a/great-docs.yml b/great-docs.yml index 249b4e605c..6121aa9604 100644 --- a/great-docs.yml +++ b/great-docs.yml @@ -271,7 +271,11 @@ reference: - Validate.all_passed - Validate.assert_passing - Validate.assert_below_threshold + - Validate.assert_dimension_scores - Validate.above_threshold + - Validate.get_dimension_scores + - Validate.get_health_score + - Validate.get_scorecard - Validate.n - Validate.n_passed - Validate.n_failed diff --git a/pointblank/_constants.py b/pointblank/_constants.py index 9b67872a2d..ed74d0075e 100644 --- a/pointblank/_constants.py +++ b/pointblank/_constants.py @@ -62,6 +62,89 @@ "col_sum_eq": "sum_eq", } +# The canonical set of data quality dimension names used for health scoring. These are the +# categories that each validation step is tagged with (either inferred automatically from the +# `assertion_type` or set explicitly via the `dimension=` parameter on a validation method). +DIMENSION_NAMES: list[str] = [ + "completeness", + "consistency", + "validity", + "uniqueness", + "timeliness", + "volume", +] + +# Default mapping of `assertion_type` to a data quality dimension. This enables automatic +# dimension inference for every validation step with no user effort. Users can remap globally +# via `pb.config(dimension_map=...)` or override per step via a validation method's `dimension=` +# parameter. Any assertion type not present here falls back to `"unknown"`. +ASSERTION_TYPE_TO_DIMENSION: dict[str, str] = { + # Completeness: presence/absence of values + "col_vals_null": "completeness", + "col_vals_not_null": "completeness", + "col_pct_null": "completeness", + "col_pct_missing": "completeness", + "col_missing_coded": "completeness", + "col_missing_only_coded": "completeness", + "rows_complete": "completeness", + # Consistency: internal agreement across columns/rows/tables + "col_missing_consistent": "consistency", + "conjointly": "consistency", + "tbl_match": "consistency", + # Validity: values conform to expected rules, ranges, formats, or schema + "col_vals_gt": "validity", + "col_vals_lt": "validity", + "col_vals_eq": "validity", + "col_vals_ne": "validity", + "col_vals_ge": "validity", + "col_vals_le": "validity", + "col_vals_between": "validity", + "col_vals_outside": "validity", + "col_vals_in_set": "validity", + "col_vals_not_in_set": "validity", + "col_vals_regex": "validity", + "col_vals_within_spec": "validity", + "col_vals_increasing": "validity", + "col_vals_decreasing": "validity", + "col_vals_expr": "validity", + "col_exists": "validity", + "col_schema_match": "validity", + "col_sum_eq": "validity", + "prompt": "validity", + "specially": "validity", + # Uniqueness: absence of duplicate rows + "rows_distinct": "uniqueness", + # Timeliness: data recency/freshness + "data_freshness": "timeliness", + # Volume: expected row/column counts + "row_count_match": "volume", + "col_count_match": "volume", +} + +# Two-letter abbreviations used for the compact, color-coded dimension badge shown in the corner of +# the step-number cell in the validation report. +DIMENSION_ABBR: dict[str, str] = { + "completeness": "CM", + "consistency": "CS", + "validity": "VA", + "uniqueness": "UQ", + "timeliness": "TM", + "volume": "VO", + "unknown": "??", +} + +# Accent colors used to color-code the dimension badge in the validation report. Hues are chosen +# to read as categorical (distinct from the warning/error/critical severity palette). +DIMENSION_COLORS: dict[str, str] = { + "completeness": "#3C6E9A", # blue + "consistency": "#C57B3C", # amber + "validity": "#4E9A6B", # green + "uniqueness": "#8E6FB5", # purple + "timeliness": "#3F9C9C", # teal + "volume": "#7A7A8C", # slate + "unknown": "#B0B0B0", # gray +} + COMPARISON_OPERATORS = { "col_vals_gt": ">", "col_vals_ge": ">=", @@ -130,6 +213,7 @@ "thresholds", "label", "brief", + "dimension", "active", "all_passed", "n", diff --git a/pointblank/_constants_translations.py b/pointblank/_constants_translations.py index 14f57c58a3..2b3c49e017 100644 --- a/pointblank/_constants_translations.py +++ b/pointblank/_constants_translations.py @@ -3447,6 +3447,468 @@ "th": "ล้มเหลว", "fa": "ناموفق", }, + "report_col_dimension": { + "en": "DIMENSION", + "fr": "DIMENSION", + "de": "DIMENSION", + "it": "DIMENSIONE", + "es": "DIMENSIÓN", + "pt": "DIMENSÃO", + "ro": "DIMENSIUNE", + "tr": "BOYUT", + "zh-Hans": "维度", + "zh-Hant": "維度", + "ja": "ディメンション", + "ko": "차원", + "vi": "CHIỀU", + "ru": "ИЗМЕРЕНИЕ", + "cs": "DIMENZE", + "pl": "WYMIAR", + "da": "DIMENSION", + "sv": "DIMENSION", + "nb": "DIMENSJON", + "nl": "DIMENSIE", + "fi": "ULOTTUVUUS", + "is": "VÍDD", + "ar": "البُعد", + "hi": "आयाम", + "el": "ΔΙΑΣΤΑΣΗ", + "id": "DIMENSI", + "uk": "ВИМІР", + "bg": "ИЗМЕРЕНИЕ", + "hr": "DIMENZIJA", + "et": "MÕÕDE", + "hu": "DIMENZIÓ", + "ga": "TOISE", + "lv": "DIMENSIJA", + "lt": "DIMENSIJA", + "mt": "DIMENSJONI", + "sk": "DIMENZIA", + "sl": "DIMENZIJA", + "he": "מימד", + "th": "มิติ", + "fa": "بُعد", + }, + "report_col_score": { + "en": "SCORE", + "fr": "SCORE", + "de": "WERT", + "it": "PUNTEGGIO", + "es": "PUNTUACIÓN", + "pt": "PONTUAÇÃO", + "ro": "SCOR", + "tr": "PUAN", + "zh-Hans": "评分", + "zh-Hant": "評分", + "ja": "スコア", + "ko": "점수", + "vi": "ĐIỂM", + "ru": "ОЦЕНКА", + "cs": "SKÓRE", + "pl": "WYNIK", + "da": "SCORE", + "sv": "POÄNG", + "nb": "POENG", + "nl": "SCORE", + "fi": "PISTEET", + "is": "STIG", + "ar": "الدرجة", + "hi": "स्कोर", + "el": "ΒΑΘΜΟΛΟΓΙΑ", + "id": "SKOR", + "uk": "ОЦІНКА", + "bg": "РЕЗУЛТАТ", + "hr": "REZULTAT", + "et": "SKOOR", + "hu": "PONTSZÁM", + "ga": "SCÓR", + "lv": "VĒRTĒJUMS", + "lt": "BALAS", + "mt": "PUNTEĠĠ", + "sk": "SKÓRE", + "sl": "OCENA", + "he": "ציון", + "th": "คะแนน", + "fa": "امتیاز", + }, + "report_health_score": { + "en": "Health Score", + "fr": "Score de santé", + "de": "Gesundheitswert", + "it": "Punteggio di salute", + "es": "Puntuación de salud", + "pt": "Pontuação de saúde", + "ro": "Scor de sănătate", + "tr": "Sağlık Puanı", + "zh-Hans": "健康评分", + "zh-Hant": "健康評分", + "ja": "ヘルススコア", + "ko": "상태 점수", + "vi": "Điểm sức khỏe", + "ru": "Оценка состояния", + "cs": "Skóre stavu", + "pl": "Wskaźnik kondycji", + "da": "Sundhedsscore", + "sv": "Hälsopoäng", + "nb": "Helsescore", + "nl": "Gezondheidsscore", + "fi": "Terveyspisteet", + "is": "Heilsustig", + "ar": "درجة الصحة", + "hi": "स्वास्थ्य स्कोर", + "el": "Βαθμολογία υγείας", + "id": "Skor Kesehatan", + "uk": "Оцінка стану", + "bg": "Здравен резултат", + "hr": "Ocjena zdravlja", + "et": "Terviseskoor", + "hu": "Egészségpontszám", + "ga": "Scór Sláinte", + "lv": "Veselības rādītājs", + "lt": "Būklės įvertis", + "mt": "Punteġġ tas-Saħħa", + "sk": "Skóre stavu", + "sl": "Ocena zdravja", + "he": "ציון תקינות", + "th": "คะแนนสุขภาพ", + "fa": "امتیاز سلامت", + }, + "report_dimension_scores": { + "en": "Dimension Scores", + "fr": "Scores par dimension", + "de": "Dimensionswerte", + "it": "Punteggi per dimensione", + "es": "Puntuaciones por dimensión", + "pt": "Pontuações por dimensão", + "ro": "Scoruri pe dimensiuni", + "tr": "Boyut Puanları", + "zh-Hans": "各维度评分", + "zh-Hant": "各維度評分", + "ja": "ディメンションスコア", + "ko": "차원별 점수", + "vi": "Điểm theo chiều", + "ru": "Оценки по измерениям", + "cs": "Skóre podle dimenzí", + "pl": "Wyniki wymiarów", + "da": "Dimensionsscorer", + "sv": "Dimensionspoäng", + "nb": "Dimensjonsscorer", + "nl": "Dimensiescores", + "fi": "Ulottuvuuspisteet", + "is": "Víddastig", + "ar": "درجات الأبعاد", + "hi": "आयाम स्कोर", + "el": "Βαθμολογίες διαστάσεων", + "id": "Skor Dimensi", + "uk": "Оцінки за вимірами", + "bg": "Резултати по измерения", + "hr": "Rezultati po dimenzijama", + "et": "Mõõtmete skoorid", + "hu": "Dimenziópontszámok", + "ga": "Scóir Toise", + "lv": "Dimensiju rādītāji", + "lt": "Dimensijų balai", + "mt": "Punteġġi tad-Dimensjonijiet", + "sk": "Skóre podľa dimenzií", + "sl": "Ocene po dimenzijah", + "he": "ציוני מימדים", + "th": "คะแนนตามมิติ", + "fa": "امتیازهای ابعاد", + }, + "dimension_completeness": { + "en": "Completeness", + "fr": "Complétude", + "de": "Vollständigkeit", + "it": "Completezza", + "es": "Completitud", + "pt": "Completude", + "ro": "Completitudine", + "tr": "Eksiksizlik", + "zh-Hans": "完整性", + "zh-Hant": "完整性", + "ja": "完全性", + "ko": "완전성", + "vi": "Tính đầy đủ", + "ru": "Полнота", + "cs": "Úplnost", + "pl": "Kompletność", + "da": "Fuldstændighed", + "sv": "Fullständighet", + "nb": "Fullstendighet", + "nl": "Volledigheid", + "fi": "Täydellisyys", + "is": "Heilleiki", + "ar": "الاكتمال", + "hi": "पूर्णता", + "el": "Πληρότητα", + "id": "Kelengkapan", + "uk": "Повнота", + "bg": "Пълнота", + "hr": "Potpunost", + "et": "Terviklikkus", + "hu": "Teljesség", + "ga": "Iomláine", + "lv": "Pilnīgums", + "lt": "Išsamumas", + "mt": "Kompletezza", + "sk": "Úplnosť", + "sl": "Popolnost", + "he": "שלמות", + "th": "ความครบถ้วน", + "fa": "کامل بودن", + }, + "dimension_consistency": { + "en": "Consistency", + "fr": "Cohérence", + "de": "Konsistenz", + "it": "Coerenza", + "es": "Consistencia", + "pt": "Consistência", + "ro": "Consistență", + "tr": "Tutarlılık", + "zh-Hans": "一致性", + "zh-Hant": "一致性", + "ja": "一貫性", + "ko": "일관성", + "vi": "Tính nhất quán", + "ru": "Согласованность", + "cs": "Konzistence", + "pl": "Spójność", + "da": "Konsistens", + "sv": "Konsistens", + "nb": "Konsistens", + "nl": "Consistentie", + "fi": "Johdonmukaisuus", + "is": "Samkvæmni", + "ar": "الاتساق", + "hi": "संगति", + "el": "Συνέπεια", + "id": "Konsistensi", + "uk": "Узгодженість", + "bg": "Съгласуваност", + "hr": "Dosljednost", + "et": "Järjepidevus", + "hu": "Konzisztencia", + "ga": "Comhsheasmhacht", + "lv": "Konsekvence", + "lt": "Nuoseklumas", + "mt": "Konsistenza", + "sk": "Konzistentnosť", + "sl": "Doslednost", + "he": "עקביות", + "th": "ความสอดคล้อง", + "fa": "سازگاری", + }, + "dimension_validity": { + "en": "Validity", + "fr": "Validité", + "de": "Gültigkeit", + "it": "Validità", + "es": "Validez", + "pt": "Validade", + "ro": "Validitate", + "tr": "Geçerlilik", + "zh-Hans": "有效性", + "zh-Hant": "有效性", + "ja": "妥当性", + "ko": "유효성", + "vi": "Tính hợp lệ", + "ru": "Валидность", + "cs": "Platnost", + "pl": "Poprawność", + "da": "Gyldighed", + "sv": "Giltighet", + "nb": "Gyldighet", + "nl": "Geldigheid", + "fi": "Kelvollisuus", + "is": "Gildi", + "ar": "الصلاحية", + "hi": "वैधता", + "el": "Εγκυρότητα", + "id": "Validitas", + "uk": "Валідність", + "bg": "Валидност", + "hr": "Valjanost", + "et": "Kehtivus", + "hu": "Érvényesség", + "ga": "Bailíocht", + "lv": "Derīgums", + "lt": "Galiojimas", + "mt": "Validità", + "sk": "Platnosť", + "sl": "Veljavnost", + "he": "תקפות", + "th": "ความถูกต้อง", + "fa": "اعتبار", + }, + "dimension_uniqueness": { + "en": "Uniqueness", + "fr": "Unicité", + "de": "Eindeutigkeit", + "it": "Unicità", + "es": "Unicidad", + "pt": "Unicidade", + "ro": "Unicitate", + "tr": "Benzersizlik", + "zh-Hans": "唯一性", + "zh-Hant": "唯一性", + "ja": "一意性", + "ko": "고유성", + "vi": "Tính duy nhất", + "ru": "Уникальность", + "cs": "Jedinečnost", + "pl": "Unikalność", + "da": "Unikhed", + "sv": "Unikhet", + "nb": "Unikhet", + "nl": "Uniekheid", + "fi": "Ainutlaatuisuus", + "is": "Einkvæmni", + "ar": "التفرد", + "hi": "विशिष्टता", + "el": "Μοναδικότητα", + "id": "Keunikan", + "uk": "Унікальність", + "bg": "Уникалност", + "hr": "Jedinstvenost", + "et": "Unikaalsus", + "hu": "Egyediség", + "ga": "Uathúlacht", + "lv": "Unikalitāte", + "lt": "Unikalumas", + "mt": "Uniċità", + "sk": "Jedinečnosť", + "sl": "Edinstvenost", + "he": "ייחודיות", + "th": "ความไม่ซ้ำ", + "fa": "یکتایی", + }, + "dimension_timeliness": { + "en": "Timeliness", + "fr": "Actualité", + "de": "Aktualität", + "it": "Tempestività", + "es": "Actualidad", + "pt": "Atualidade", + "ro": "Actualitate", + "tr": "Güncellik", + "zh-Hans": "时效性", + "zh-Hant": "時效性", + "ja": "適時性", + "ko": "적시성", + "vi": "Tính kịp thời", + "ru": "Своевременность", + "cs": "Včasnost", + "pl": "Aktualność", + "da": "Aktualitet", + "sv": "Aktualitet", + "nb": "Aktualitet", + "nl": "Tijdigheid", + "fi": "Ajantasaisuus", + "is": "Tímanleiki", + "ar": "الحداثة", + "hi": "समयबद्धता", + "el": "Επικαιρότητα", + "id": "Ketepatan Waktu", + "uk": "Своєчасність", + "bg": "Навременност", + "hr": "Pravovremenost", + "et": "Ajakohasus", + "hu": "Időszerűség", + "ga": "Tráthúlacht", + "lv": "Savlaicīgums", + "lt": "Savalaikiškumas", + "mt": "Puntwalità", + "sk": "Včasnosť", + "sl": "Pravočasnost", + "he": "עדכניות", + "th": "ความทันเวลา", + "fa": "به‌هنگام بودن", + }, + "dimension_volume": { + "en": "Volume", + "fr": "Volume", + "de": "Volumen", + "it": "Volume", + "es": "Volumen", + "pt": "Volume", + "ro": "Volum", + "tr": "Hacim", + "zh-Hans": "数据量", + "zh-Hant": "資料量", + "ja": "データ量", + "ko": "볼륨", + "vi": "Khối lượng", + "ru": "Объём", + "cs": "Objem", + "pl": "Wolumen", + "da": "Volumen", + "sv": "Volym", + "nb": "Volum", + "nl": "Volume", + "fi": "Määrä", + "is": "Umfang", + "ar": "الحجم", + "hi": "मात्रा", + "el": "Όγκος", + "id": "Volume", + "uk": "Обсяг", + "bg": "Обем", + "hr": "Volumen", + "et": "Maht", + "hu": "Mennyiség", + "ga": "Toirt", + "lv": "Apjoms", + "lt": "Apimtis", + "mt": "Volum", + "sk": "Objem", + "sl": "Obseg", + "he": "נפח", + "th": "ปริมาณ", + "fa": "حجم", + }, + "dimension_unknown": { + "en": "Unknown", + "fr": "Inconnu", + "de": "Unbekannt", + "it": "Sconosciuto", + "es": "Desconocido", + "pt": "Desconhecido", + "ro": "Necunoscut", + "tr": "Bilinmeyen", + "zh-Hans": "未知", + "zh-Hant": "未知", + "ja": "不明", + "ko": "알 수 없음", + "vi": "Không xác định", + "ru": "Неизвестно", + "cs": "Neznámé", + "pl": "Nieznane", + "da": "Ukendt", + "sv": "Okänd", + "nb": "Ukjent", + "nl": "Onbekend", + "fi": "Tuntematon", + "is": "Óþekkt", + "ar": "غير معروف", + "hi": "अज्ञात", + "el": "Άγνωστο", + "id": "Tidak diketahui", + "uk": "Невідомо", + "bg": "Неизвестно", + "hr": "Nepoznato", + "et": "Teadmata", + "hu": "Ismeretlen", + "ga": "Anaithnid", + "lv": "Nezināms", + "lt": "Nežinoma", + "mt": "Mhux magħruf", + "sk": "Neznáme", + "sl": "Neznano", + "he": "לא ידוע", + "th": "ไม่ทราบ", + "fa": "نامشخص", + }, "report_column_schema": { "en": "SCHEMA", "fr": "SCHÉMA", diff --git a/pointblank/validate.py b/pointblank/validate.py index 3ffa68543a..31daadda35 100644 --- a/pointblank/validate.py +++ b/pointblank/validate.py @@ -36,11 +36,15 @@ ) from pointblank._constants import ( ASSERTION_TYPE_METHOD_MAP, + ASSERTION_TYPE_TO_DIMENSION, CHECK_MARK_SPAN, COMPARISON_OPERATORS, COMPARISON_OPERATORS_AR, COMPATIBLE_DTYPES, CROSS_MARK_SPAN, + DIMENSION_ABBR, + DIMENSION_COLORS, + DIMENSION_NAMES, IBIS_BACKENDS, LOG_LEVELS_MAP, MODEL_PROVIDERS, @@ -318,6 +322,10 @@ def get_validation_summary() -> dict | None: - `tbl_column_count` (`int`): The number of columns in the target table. - `tbl_name` (`str`): The name of the target table. - `validation_duration` (`float`): The duration of the validation in seconds. + - `dimension_scores` (`dict`): The test-unit-weighted health score (`0`-`100`) for each data + quality dimension present in the validation (e.g., `{"completeness": 99.2, "validity": 97.8}`). + - `overall_health_score` (`float`): The overall, test-unit-weighted health score (`0`-`100`) + across all dimensions. Note that the summary dictionary is only available within the context of a final action. If called outside of a final action (i.e., when no final action is being executed), this function @@ -400,7 +408,11 @@ class PointblankConfig: report_incl_footer: bool = True report_incl_footer_timings: bool = True report_incl_footer_notes: bool = True + report_incl_dimensions: bool = False preview_incl_header: bool = True + dimension_map: dict[str, str] | None = None + dimension_weights: dict[str, float] | None = None + dimension_thresholds: dict[str, float] | None = None def __repr__(self) -> str: return ( @@ -408,7 +420,11 @@ def __repr__(self) -> str: f"report_incl_footer={self.report_incl_footer}, " f"report_incl_footer_timings={self.report_incl_footer_timings}, " f"report_incl_footer_notes={self.report_incl_footer_notes}, " - f"preview_incl_header={self.preview_incl_header})" + f"report_incl_dimensions={self.report_incl_dimensions}, " + f"preview_incl_header={self.preview_incl_header}, " + f"dimension_map={self.dimension_map}, " + f"dimension_weights={self.dimension_weights}, " + f"dimension_thresholds={self.dimension_thresholds})" ) @@ -421,7 +437,11 @@ def config( report_incl_footer: bool = True, report_incl_footer_timings: bool = True, report_incl_footer_notes: bool = True, + report_incl_dimensions: bool = False, preview_incl_header: bool = True, + dimension_map: dict[str, str] | None = None, + dimension_weights: dict[str, float] | None = None, + dimension_thresholds: dict[str, float] | None = None, ) -> PointblankConfig: """ Configuration settings for the Pointblank library. @@ -441,9 +461,32 @@ def config( report_incl_footer_notes Controls whether the notes from validation steps should be displayed in the footer. Only applies when `report_incl_footer=True`. + report_incl_dimensions + Controls whether the data quality dimension display is included in the validation report by + default. When `True`, each step's report shows a color-coded dimension badge on the step + number and a health-score summary block in the footer. This is `False` by default (opt-in); + it can also be toggled per-report via `get_tabular_report(incl_dimensions=...)`. preview_incl_header Whether the header should be present in any preview table (generated via the [`preview()`](`pointblank.preview`) function). + dimension_map + An optional mapping of `assertion_type` (e.g., `"col_vals_gt"`) to a data quality dimension + name (e.g., `"validity"`). This is merged on top of the built-in inference map, letting you + remap how validation steps are categorized for health scoring. Only affects steps where the + dimension is not set explicitly via a validation method's `dimension=` parameter. + dimension_weights + An optional mapping of dimension name to a relative weight (a positive number) used when + computing the overall health score. A weight scales that dimension's *test-unit* + contribution to the overall score, so a dimension's influence is its weight multiplied by + its number of test units (a heavily-weighted dimension with few test units still + contributes modestly). Dimensions not present in the mapping default to a weight of `1.0`; + when omitted entirely, the overall score is a pure test-unit-weighted pass rate. + dimension_thresholds + An optional mapping of dimension name to a minimum acceptable health score (`0`-`100`). + This is used as the default by + [`assert_dimension_scores()`](`pointblank.Validate.assert_dimension_scores`), which raises + an `AssertionError` when any dimension's score falls below its minimum (e.g., to fail a CI + run when the completeness score drops below `95`). Returns ------- @@ -456,10 +499,49 @@ def config( global_config.report_incl_footer = report_incl_footer # pragma: no cover global_config.report_incl_footer_timings = report_incl_footer_timings # pragma: no cover global_config.report_incl_footer_notes = report_incl_footer_notes # pragma: no cover + global_config.report_incl_dimensions = report_incl_dimensions # pragma: no cover global_config.preview_incl_header = preview_incl_header # pragma: no cover + global_config.dimension_map = dimension_map # pragma: no cover + global_config.dimension_weights = dimension_weights # pragma: no cover + global_config.dimension_thresholds = dimension_thresholds # pragma: no cover return global_config # pragma: no cover +def _base_dimension_from_assertion_type(assertion_type: str | None) -> str | None: + """ + Infer the data quality dimension from the built-in defaults only (ignoring any config override). + + Uses `ASSERTION_TYPE_TO_DIMENSION`, with a regex fallback for the dynamically-generated + aggregate comparison methods (e.g., `col_sum_eq`, `col_avg_gt`, `col_sd_le`). Assertion types + not covered are categorized as `"unknown"`. Returns `None` if `assertion_type` is `None`. + """ + if assertion_type is None: + return None + if assertion_type in ASSERTION_TYPE_TO_DIMENSION: + return ASSERTION_TYPE_TO_DIMENSION[assertion_type] + if re.match( + r"^col_(sum|avg|mean|median|min|max|sd|std|var)_(eq|ne|gt|ge|lt|le|between|outside)$", + assertion_type, + ): + return "validity" + return "unknown" + + +def _infer_dimension_from_assertion_type(assertion_type: str | None) -> str | None: + """ + Infer the data quality dimension for a validation step from its `assertion_type`. + + Any `pb.config(dimension_map=...)` override takes precedence over the built-in + `ASSERTION_TYPE_TO_DIMENSION` defaults. Returns `None` if `assertion_type` is `None`. + """ + if assertion_type is None: + return None + override = getattr(global_config, "dimension_map", None) or {} + if assertion_type in override: + return override[assertion_type] + return _base_dimension_from_assertion_type(assertion_type) + + def load_dataset( dataset: Literal["small_table", "game_revenue", "nycflights", "global_sales"] = "small_table", tbl_type: Literal["polars", "pandas", "duckdb"] = "polars", @@ -3977,6 +4059,9 @@ class _ValidationInfo: A brief description of the validation step. autobrief An automatically-generated brief for the validation step. + dimension + The data quality dimension for the validation step (e.g., `"completeness"`, `"validity"`, + `"uniqueness"`, etc.). If not set explicitly, it is inferred from the `assertion_type`. active Whether the validation step is active. all_passed @@ -4026,6 +4111,7 @@ def from_agg_validator( brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> _ValidationInfo: # This factory method creates a `_ValidationInfo` instance for aggregate # methods. The reason this is created, is because all agg methods share the same @@ -4041,6 +4127,7 @@ def from_agg_validator( brief=_transform_auto_brief(brief=brief), actions=actions, active=active, + dimension=dimension, ) # Validation plan @@ -4061,6 +4148,7 @@ def from_agg_validator( label: str | None = None brief: str | None = None autobrief: str | None = None + dimension: str | None = None active: bool | Callable | None = None # Interrogation results eval_error: bool | None = None @@ -4522,6 +4610,12 @@ def _columns_param() -> None: if vi.brief is not None and vi.brief is not False: params["brief"] = True if vi.brief == "{auto}" else vi.brief + # Emit `dimension=` only when it deviates from the built-in default inference (i.e., an + # explicit per-step override or a `dimension_map` remap), so generated plans reproduce the + # same dimensions without depending on global config + if vi.dimension is not None and vi.dimension != _base_dimension_from_assertion_type(at): + params["dimension"] = vi.dimension + if vi.active is not None and vi.active is not True: if callable(vi.active): params["active"] = _placeholder( @@ -5653,6 +5747,7 @@ def _add_agg_validation( brief=False, actions=None, active=True, + dimension: str | None = None, ): """ Add an aggregation-based validation step to the validation plan. @@ -5701,6 +5796,12 @@ def _add_agg_validation( active Whether this validation step is active. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -5735,6 +5836,7 @@ def _add_agg_validation( actions=self.actions if actions is None else actions, brief=self.brief if brief is None else brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -5890,6 +5992,7 @@ def col_vals_gt( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Are column data greater than a fixed value or data in another column? @@ -5952,6 +6055,12 @@ def col_vals_gt( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -6163,6 +6272,7 @@ def col_vals_gt( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -6181,6 +6291,7 @@ def col_vals_lt( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Are column data less than a fixed value or data in another column? @@ -6243,6 +6354,12 @@ def col_vals_lt( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -6461,6 +6578,7 @@ def col_vals_lt( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -6479,6 +6597,7 @@ def col_vals_eq( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Are column data equal to a fixed value or data in another column? @@ -6541,6 +6660,12 @@ def col_vals_eq( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -6759,6 +6884,7 @@ def col_vals_eq( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -6777,6 +6903,7 @@ def col_vals_ne( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Are column data not equal to a fixed value or data in another column? @@ -6839,6 +6966,12 @@ def col_vals_ne( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -7055,6 +7188,7 @@ def col_vals_ne( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -7073,6 +7207,7 @@ def col_vals_ge( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Are column data greater than or equal to a fixed value or data in another column? @@ -7135,6 +7270,12 @@ def col_vals_ge( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -7354,6 +7495,7 @@ def col_vals_ge( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -7372,6 +7514,7 @@ def col_vals_le( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Are column data less than or equal to a fixed value or data in another column? @@ -7434,6 +7577,12 @@ def col_vals_le( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -7653,6 +7802,7 @@ def col_vals_le( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -7673,6 +7823,7 @@ def col_vals_between( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Do column data lie between two specified values or data in other columns? @@ -7745,6 +7896,12 @@ def col_vals_between( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -7980,6 +8137,7 @@ def col_vals_between( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -8000,6 +8158,7 @@ def col_vals_outside( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Do column data lie outside of two specified values or data in other columns? @@ -8072,6 +8231,12 @@ def col_vals_outside( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -8307,6 +8472,7 @@ def col_vals_outside( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -8324,6 +8490,7 @@ def col_vals_in_set( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether column values are in a set of values. @@ -8381,6 +8548,12 @@ def col_vals_in_set( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -8631,6 +8804,7 @@ class Color(Enum): actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -8648,6 +8822,7 @@ def col_vals_not_in_set( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether column values are not in a set of values. @@ -8705,6 +8880,12 @@ def col_vals_not_in_set( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -8927,6 +9108,7 @@ class InvalidStatus(Enum): actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -8946,6 +9128,7 @@ def col_vals_increasing( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Are column data increasing by row? @@ -9014,6 +9197,12 @@ def col_vals_increasing( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -9125,6 +9314,7 @@ def col_vals_increasing( actions=actions, brief=brief, active=active, + dimension=dimension, val_info={ "allow_stationary": allow_stationary, "decreasing_tol": decreasing_tol if decreasing_tol else 0.0, @@ -9148,6 +9338,7 @@ def col_vals_decreasing( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Are column data decreasing by row? @@ -9216,6 +9407,12 @@ def col_vals_decreasing( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -9327,6 +9524,7 @@ def col_vals_decreasing( actions=actions, brief=brief, active=active, + dimension=dimension, val_info={ "allow_stationary": allow_stationary, "increasing_tol": increasing_tol if increasing_tol else 0.0, @@ -9346,6 +9544,7 @@ def col_vals_null( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether values in a column are Null. @@ -9397,6 +9596,12 @@ def col_vals_null( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -9579,6 +9784,7 @@ def col_vals_null( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -9594,6 +9800,7 @@ def col_vals_not_null( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether values in a column are not Null. @@ -9645,6 +9852,12 @@ def col_vals_not_null( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -9827,6 +10040,7 @@ def col_vals_not_null( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -9846,6 +10060,7 @@ def col_vals_regex( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether column values match a regular expression pattern. @@ -9906,6 +10121,12 @@ def col_vals_regex( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -10098,6 +10319,7 @@ def col_vals_regex( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -10116,6 +10338,7 @@ def col_vals_within_spec( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether column values fit within a specification. @@ -10177,6 +10400,12 @@ def col_vals_within_spec( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -10390,6 +10619,7 @@ def col_vals_within_spec( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -10405,6 +10635,7 @@ def col_vals_expr( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate column values using a custom expression. @@ -10457,6 +10688,12 @@ def col_vals_expr( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -10621,6 +10858,7 @@ def col_vals_expr( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -10634,6 +10872,7 @@ def col_exists( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether one or more columns exist in the table. @@ -10675,6 +10914,12 @@ def col_exists( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -10800,6 +11045,7 @@ def col_exists( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -10815,6 +11061,7 @@ def col_pct_null( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether a column has a specific percentage of Null values. @@ -10867,6 +11114,12 @@ def col_pct_null( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -11101,6 +11354,7 @@ def col_pct_null( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -11118,6 +11372,7 @@ def col_pct_missing( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate that the percentage of *structured* missing values stays within a limit. @@ -11173,6 +11428,12 @@ def col_pct_missing( active. Using `False` will make the validation step inactive (still reporting its presence and keeping indexes for the steps unchanged). + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -11301,6 +11562,7 @@ def col_pct_missing( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -11317,6 +11579,7 @@ def col_missing_coded( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate that all missing values in a column are *coded* (no uncoded nulls). @@ -11367,6 +11630,12 @@ def col_missing_coded( active. Using `False` will make the validation step inactive (still reporting its presence and keeping indexes for the steps unchanged). + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -11481,6 +11750,7 @@ def col_missing_coded( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -11500,6 +11770,7 @@ def col_missing_only_coded( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate that a column contains only documented codes and legitimate values. @@ -11555,6 +11826,12 @@ def col_missing_only_coded( active. Using `False` will make the validation step inactive (still reporting its presence and keeping indexes for the steps unchanged). + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -11685,6 +11962,7 @@ def col_missing_only_coded( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -11700,6 +11978,7 @@ def rows_distinct( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether rows in the table are distinct. @@ -11751,6 +12030,12 @@ def rows_distinct( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -11931,6 +12216,7 @@ def rows_distinct( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -11946,6 +12232,7 @@ def rows_complete( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether row data are complete by having no missing values. @@ -11997,6 +12284,12 @@ def rows_complete( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -12177,6 +12470,7 @@ def rows_complete( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -12194,6 +12488,7 @@ def col_missing_consistent( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate that related columns share a consistent missingness pattern for a given reason. @@ -12246,6 +12541,12 @@ def col_missing_consistent( active. Using `False` will make the validation step inactive (still reporting its presence and keeping indexes for the steps unchanged). + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -12378,6 +12679,7 @@ def col_missing_consistent( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -12398,6 +12700,7 @@ def prompt( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate rows using AI/LLM-powered analysis. @@ -12490,6 +12793,12 @@ def prompt( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -12857,6 +13166,7 @@ def prompt( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -12876,6 +13186,7 @@ def col_schema_match( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Do columns in the table (and their types) match a predefined schema? @@ -12943,6 +13254,12 @@ def col_schema_match( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -13085,6 +13402,7 @@ def col_schema_match( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -13101,6 +13419,7 @@ def row_count_match( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether the row count of the table matches a specified count. @@ -13161,6 +13480,12 @@ def row_count_match( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -13309,6 +13634,7 @@ def row_count_match( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -13327,6 +13653,7 @@ def data_freshness( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate that data in a datetime column is not older than a specified maximum age. @@ -13392,6 +13719,12 @@ def data_freshness( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -13677,6 +14010,7 @@ def data_freshness( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -13692,6 +14026,7 @@ def col_count_match( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether the column count of the table matches a specified count. @@ -13744,6 +14079,12 @@ def col_count_match( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -13858,6 +14199,7 @@ def col_count_match( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -13872,6 +14214,7 @@ def tbl_match( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Validate whether the target table matches a comparison table. @@ -13928,6 +14271,12 @@ def tbl_match( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -14132,6 +14481,7 @@ def tbl_match( actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -14146,6 +14496,7 @@ def conjointly( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Perform multiple row-wise validations for joint validity. @@ -14198,6 +14549,12 @@ def conjointly( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -14385,6 +14742,7 @@ def filter_by_c_gt_5(df): actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -14399,6 +14757,7 @@ def specially( actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """ Perform a specialized validation with customized logic. @@ -14466,6 +14825,12 @@ def specially( [`has_rows()`](`pointblank.has_rows`) can be used here to conditionally activate a step based on properties of the target table. + dimension + An optional data quality dimension to categorize this validation step for health + scoring. One of `"completeness"`, `"validity"`, `"uniqueness"`, `"consistency"`, + `"timeliness"`, or `"volume"` (or any custom string). If `None` (the default), the + dimension is inferred automatically from the assertion type. This label appears in the + validation report and feeds the overall and per-dimension health scores. Returns ------- Validate @@ -14735,6 +15100,7 @@ def validate_pointblank_version(): actions=actions, brief=brief, active=active, + dimension=dimension, ) self._add_validation(validation_info=val_info) @@ -17955,6 +18321,359 @@ def from_prompt( max_reprompts=max_reprompts, ) + def get_dimension_scores(self) -> dict[str, float]: + """ + Get per-dimension health scores from the validation results. + + Each validation step is associated with a data quality dimension (e.g., `"completeness"`, + `"validity"`, `"uniqueness"`, `"consistency"`, `"timeliness"`, or `"volume"`), either + inferred automatically from the assertion type or set explicitly via the `dimension=` + parameter on a validation method. This method rolls the per-step results up into a + test-unit-weighted pass rate (`0`-`100`) for each dimension present in the plan. + + Scores are weighted by test units, so larger tables and steps with more test units + contribute proportionally more to each dimension's score. Only steps that have been + interrogated contribute; inactive steps are excluded. + + Returns + ------- + dict[str, float] + A dictionary mapping each dimension name to its score (a percentage from `0` to `100`). + Returns an empty dictionary if the validation has not been interrogated. + + Examples + -------- + ```python + import pointblank as pb + + validation = ( + pb.Validate(data=pb.load_dataset("small_table")) + .col_vals_not_null(columns="c") + .col_vals_gt(columns="d", value=0) + .rows_distinct() + .interrogate() + ) + + validation.get_dimension_scores() + ``` + + See Also + -------- + Use [`get_health_score()`](`pointblank.Validate.get_health_score`) for a single overall + score across all dimensions. + """ + return _compute_dimension_scores(self.validation_info) + + def get_health_score(self) -> float: + """ + Get the overall data quality health score from the validation results. + + The health score is a single number (a percentage from `0` to `100`) that summarizes the + overall quality of the data across all validation steps. It is computed as a test-unit + weighted pass rate: the total number of passing test units divided by the total number of + test units. This means larger tables and steps operating over more rows contribute + proportionally more to the score, so it tracks data volume rather than mere step count. + + Per-dimension weights can be set globally via + [`config(dimension_weights=...)`](`pointblank.config`) for organizations that consider some + dimensions (e.g., completeness) more critical than others. A weight scales that dimension's + test-unit contribution to the score (so a dimension's influence is its weight times its + test-unit count); the returned score is always within `[0, 100]`. + + Returns + ------- + float + The overall health score as a percentage from `0` to `100`. Returns `100.0` if the + validation has not been interrogated or contains no scorable steps. + + Examples + -------- + ```python + import pointblank as pb + + validation = ( + pb.Validate(data=pb.load_dataset("small_table")) + .col_vals_not_null(columns="c") + .col_vals_gt(columns="d", value=0) + .interrogate() + ) + + validation.get_health_score() + ``` + + See Also + -------- + Use [`get_dimension_scores()`](`pointblank.Validate.get_dimension_scores`) for a per-dimension + breakdown of the score. + """ + weights = getattr(global_config, "dimension_weights", None) + return _compute_health_score(self.validation_info, dimension_weights=weights) + + def assert_dimension_scores( + self, + thresholds: dict[str, float] | None = None, + message: str | None = None, + ) -> None: + """ + Raise an `AssertionError` if any dimension's health score falls below a minimum. + + The `assert_dimension_scores()` method checks each data quality dimension's score (from + [`get_dimension_scores()`](`pointblank.Validate.get_dimension_scores`)) against a minimum + acceptable value. This is useful in automated testing and CI environments where you want to + fail the run when, say, the completeness score drops below `95`. + + Parameters + ---------- + thresholds + A mapping of dimension name to a minimum acceptable score (`0`-`100`). If `None`, the + minimums set via [`config(dimension_thresholds=...)`](`pointblank.config`) are used. A + dimension present in the thresholds but absent from the validation is ignored. + message + Custom error message to use if the assertion fails. If `None`, a default message that + lists the offending dimensions (with actual vs. required scores) is generated. + + Returns + ------- + None + + Raises + ------ + AssertionError + If any dimension's score is below its specified minimum. + + Examples + -------- + ```python + import pointblank as pb + + validation = ( + pb.Validate(data=pb.load_dataset("small_table")) + .col_vals_not_null(columns="c") + .interrogate() + ) + + # Fail if the completeness score is below 95 + validation.assert_dimension_scores(thresholds={"completeness": 95}) + ``` + + See Also + -------- + Use [`get_dimension_scores()`](`pointblank.Validate.get_dimension_scores`) to retrieve the + scores without raising, and [`config()`](`pointblank.config`) to set default per-dimension + thresholds globally. + """ + if thresholds is None: + thresholds = getattr(global_config, "dimension_thresholds", None) or {} + + if not thresholds: + return + + # Auto-interrogate with default parameters if not already done (matches the behavior of + # `assert_below_threshold()`), so scores reflect actual results rather than an empty plan + if not hasattr(self, "time_start") or self.time_start is None: + self.interrogate() + + dimension_scores = self.get_dimension_scores() + + failures = [] + for dimension, minimum in thresholds.items(): + score = dimension_scores.get(dimension) + if score is not None and score < minimum: + failures.append((dimension, score, minimum)) + + if failures: + if message is None: + parts = [ + f"{dimension} ({score:g}% < {minimum:g}%)" + for dimension, score, minimum in failures + ] + message = "Dimension health score(s) below the required minimum: " + ", ".join( + parts + ) + raise AssertionError(message) + + def get_scorecard(self, title: str | None = ":default:") -> GT: + """ + Get a data quality scorecard as a GT table. + + The `get_scorecard()` method produces a compact, standalone scorecard that summarizes data + quality across dimensions. It shows the overall health score prominently, along with a + per-dimension breakdown (a color-coded bar, the dimension's score, and its passing/total + test units). Unlike the full validation report, the scorecard focuses purely on the + aggregate health picture, making it well-suited for dashboards and executive summaries. + + The returned object is a Great Tables `GT` object, so it can be displayed directly, exported + to HTML (via `.as_raw_html()`), or saved to an image file (via `.save()`). + + Parameters + ---------- + title + Options for customizing the title of the scorecard. The default `":default:"` produces + a generic title (optionally including the table name). Use `":tbl_name:"` to show just + the table name, `":none:"` for no title, or provide your own Markdown text. + + Returns + ------- + GT + A `GT` object representing the scorecard. + + Examples + -------- + ```python + import pointblank as pb + + validation = ( + pb.Validate(data=pb.load_dataset("small_table"), tbl_name="small_table") + .col_vals_not_null(columns="c") + .col_vals_gt(columns="d", value=0) + .rows_distinct() + .interrogate() + ) + + validation.get_scorecard() + ``` + + See Also + -------- + Use [`get_dimension_scores()`](`pointblank.Validate.get_dimension_scores`) and + [`get_health_score()`](`pointblank.Validate.get_health_score`) for the underlying numbers, + and [`get_tabular_report()`](`pointblank.Validate.get_tabular_report`) for the full per-step + validation report. + """ + # Do we have a DataFrame library to work with? + _check_any_df_lib(method_used="get_scorecard") + + # Select the DataFrame library + df_lib = _select_df_lib(preference="polars") + + lang = self.lang or "en" + + dimension_scores = _compute_dimension_scores(self.validation_info) + scorecard_title = _get_report_text("report_dimension_scores", lang) + + # Resolve the title text + if title == ":default:": + if self.tbl_name: + title_text = f"{scorecard_title} — {self.tbl_name}" + else: + title_text = scorecard_title + elif title == ":tbl_name:": + title_text = f"{self.tbl_name}" if self.tbl_name else scorecard_title + elif title in (":none:", None): + title_text = None + else: + title_text = commonmark.commonmark(title) + + # If there are no scorable steps, return a minimal table with an informative message. + # Distinguish "not interrogated yet" from "interrogated but nothing scorable" (an empty + # plan, or all steps inactive/errored) so the message is accurate in each case. + if not dimension_scores: + interrogated = getattr(self, "time_start", None) is not None + msg_key = ( + "no_validation_steps_text" if interrogated else "no_interrogation_performed_text" + ) + no_steps_text = VALIDATION_REPORT_TEXT[msg_key].get( + lang, VALIDATION_REPORT_TEXT[msg_key]["en"] + ) + df = df_lib.DataFrame({"scorecard": [no_steps_text]}) + gt_tbl = ( + GT(df, id="pb_scorecard") + .opt_table_font(font=google_font(name="IBM Plex Sans")) + .cols_label(cases={"scorecard": ""}) + .tab_style(style=style.text(color="#666666", weight="bold"), locations=loc.body()) + ) + if title_text is not None: + gt_tbl = gt_tbl.tab_header(title=html(title_text)) + if version("great_tables") >= "0.17.0": + gt_tbl = gt_tbl.tab_options(quarto_disable_processing=True) + return gt_tbl + + weights = getattr(global_config, "dimension_weights", None) + overall = _compute_health_score(self.validation_info, dimension_weights=weights) + + agg = _aggregate_dimension_units(self.validation_info) + + # Order dimensions using the canonical order, appending any custom dimensions at the end + ordered_dimensions = [d for d in DIMENSION_NAMES if d in dimension_scores] + ordered_dimensions += [d for d in dimension_scores if d not in DIMENSION_NAMES] + + dimension_cells: list[str] = [] + score_cells: list[str] = [] + units_cells: list[str] = [] + for dimension in ordered_dimensions: + n_passed, n = agg[dimension] + score = dimension_scores[dimension] + color = DIMENSION_COLORS.get(dimension, DIMENSION_COLORS["unknown"]) + label = html_module.escape(_get_dimension_label(dimension, lang)) + dimension_cells.append( + f'' + f"{label}" + ) + score_cells.append( + '
' + '
' + f'
' + "{score:g}%' + "
" + ) + units_cells.append(f"{n_passed} / {n}") + + df = df_lib.DataFrame( + { + "dimension": dimension_cells, + "score": score_cells, + "test_units": units_cells, + } + ) + + health_label = _get_report_text("report_health_score", lang) + subtitle_html = ( + '
' + f'{health_label}:' + " " + f'' + f"{overall:.0f}%" + "
" + ) + + gt_tbl = ( + GT(df, id="pb_scorecard") + .opt_table_font(font=google_font(name="IBM Plex Sans")) + .cols_label( + cases={ + "dimension": _get_report_text("report_col_dimension", lang), + "score": _get_report_text("report_col_score", lang), + "test_units": _get_report_text("report_col_units", lang), + } + ) + .cols_align(align="left", columns=["dimension", "score"]) + .cols_align(align="right", columns=["test_units"]) + .cols_width(cases={"dimension": "130px", "score": "200px", "test_units": "90px"}) + .tab_style( + style=style.text(weight="bold", color="#666666"), locations=loc.column_labels() + ) + .tab_style(style=style.css("height: 34px;"), locations=loc.body()) + .tab_style( + style=style.text( + color="black", font=google_font(name="IBM Plex Mono"), size="12px" + ), + locations=loc.body(columns="test_units"), + ) + .tab_options(table_font_size="90%") + ) + + if title_text is not None: + gt_tbl = gt_tbl.tab_header(title=html(title_text), subtitle=html(subtitle_html)) + + if version("great_tables") >= "0.17.0": + gt_tbl = gt_tbl.tab_options(quarto_disable_processing=True) + + return gt_tbl + def get_json_report( self, use_fields: list[str] | None = None, exclude_fields: list[str] | None = None ) -> str: @@ -18453,6 +19172,7 @@ def get_tabular_report( incl_footer: bool | None = None, incl_footer_timings: bool | None = None, incl_footer_notes: bool | None = None, + incl_dimensions: bool | None = None, ) -> GT: """ Validation report as a GT table. @@ -18489,6 +19209,11 @@ def get_tabular_report( incl_footer_notes Controls whether notes from validation steps should be displayed in the footer. If `None`, uses the global configuration setting. Only applies when `incl_footer=True`. + incl_dimensions + Controls whether the data quality dimension display is shown: a color-coded dimension + badge on each step number and a health-score summary block in the footer. If `None`, + uses the global configuration setting (which defaults to `False`, i.e., opt-in). Set to + `True` to include it for this report. Returns ------- @@ -18552,6 +19277,8 @@ def get_tabular_report( incl_footer_timings = global_config.report_incl_footer_timings if incl_footer_notes is None: incl_footer_notes = global_config.report_incl_footer_notes + if incl_dimensions is None: + incl_dimensions = global_config.report_incl_dimensions # Do we have a DataFrame library to work with? _check_any_df_lib(method_used="get_tabular_report") @@ -19324,6 +20051,19 @@ def get_tabular_report( if not interrogation_performed: validation_info_dict["i"] = list(range(1, len(validation_info_dict["type_upd"]) + 1)) + # Overlay a small, color-coded two-letter dimension badge on the top-left of each step + # number cell (opt-in via `incl_dimensions`). This is positioned absolutely so it doesn't + # affect the layout of the numeral (regardless of digit count); the full dimension name is + # shown via a tooltip. This is done after the `i` values are finalized (they are regenerated + # above when not interrogated). + if incl_dimensions: + validation_info_dict["i"] = _transform_step_number_with_dimension( + i_values=validation_info_dict["i"], + dimensions=validation_info_dict["dimension"], + lang=lang, + ) + validation_info_dict.pop("dimension") + # Create a table time string table_time = _create_table_time_html(time_start=self.time_start, time_end=self.time_end) @@ -19502,6 +20242,20 @@ def get_tabular_report( gt_tbl = gt_tbl.tab_header(title=html(title_text), subtitle=html(combined_subtitle)) if incl_footer: + # Add the health-score summary block (overall + per-dimension) as the first source + # note when the dimension display is enabled, the interrogation has been performed, and + # there are scorable steps. The block's dotted divider is only drawn when the timings + # row follows it (otherwise it would double up with the next footer separator). + if incl_dimensions and interrogation_performed: + health_score_html = _create_health_score_html( + validation_info=self.validation_info, + lang=lang, + dimension_weights=getattr(global_config, "dimension_weights", None), + show_divider=incl_footer_timings, + ) + if health_score_html: + gt_tbl = gt_tbl.tab_source_note(source_note=html(health_score_html)) + # Add table time as HTML source note if enabled if incl_footer_timings: gt_tbl = gt_tbl.tab_source_note(source_note=html(table_time)) @@ -20225,6 +20979,13 @@ def _add_validation(self, validation_info): # Set the `i_o` attribute to the largest value of `i_o` plus 1 validation_info.i_o = max_i_o + 1 + # Automatically infer the data quality dimension from the assertion type if it was not + # set explicitly via the `dimension=` parameter on the validation method + if validation_info.dimension is None: + validation_info.dimension = _infer_dimension_from_assertion_type( + validation_info.assertion_type + ) + self.validation_info.append(validation_info) return self @@ -20485,6 +21246,8 @@ def _execute_final_actions(self) -> None: "tbl_column_count": column_count, "tbl_name": self.tbl_name or "Unknown", "validation_duration": validation_duration, + "dimension_scores": _compute_dimension_scores(self.validation_info), + "overall_health_score": self.get_health_score(), } # Extract the actions from FinalActions object and execute @@ -22238,6 +23001,7 @@ def _validation_info_as_dict(validation_info: _ValidationInfo) -> dict: "label", "brief", "autobrief", + "dimension", "active", "eval_error", "all_passed", @@ -22271,6 +23035,79 @@ def _validation_info_as_dict(validation_info: _ValidationInfo) -> dict: return validation_info_dict +def _aggregate_dimension_units(validation_info: list[_ValidationInfo]) -> dict[str, list[int]]: + """ + Aggregate passing/total test units per data quality dimension. + + Only steps that produced a pass/fail result contribute to the aggregation. Steps that were not + interrogated, inactive steps (`active=False`), and steps that could not be evaluated + (`eval_error`) all have `n_passed` unset and are therefore excluded, so a broken check (e.g., a + reference to a nonexistent column) doesn't distort the score. Steps with no assigned dimension + are grouped under `"unknown"`. + + Returns + ------- + dict[str, list[int]] + A mapping of dimension name to a two-element list `[n_passed, n]` (summed test units). + """ + agg: dict[str, list[int]] = {} + for step in validation_info: + # Skip steps without a computed result: not interrogated, inactive, or `eval_error` + # (these have `n_passed` as `None`) + if step.n is None or step.n_passed is None: + continue + dimension = step.dimension or "unknown" + entry = agg.setdefault(dimension, [0, 0]) + entry[0] += step.n_passed + entry[1] += step.n + return agg + + +def _compute_dimension_scores(validation_info: list[_ValidationInfo]) -> dict[str, float]: + """ + Compute the test-unit-weighted pass rate (0-100) for each data quality dimension. + + A dimension's score is the total number of passing test units divided by the total number of + test units across all of its steps, expressed as a percentage. Dimensions with zero test units + score `100.0`. + """ + agg = _aggregate_dimension_units(validation_info) + return { + dimension: (round(n_passed / n * 100, 2) if n else 100.0) + for dimension, (n_passed, n) in agg.items() + } + + +def _compute_health_score( + validation_info: list[_ValidationInfo], + dimension_weights: dict[str, float] | None = None, +) -> float: + """ + Compute the overall, test-unit-weighted health score (0-100) across all dimensions. + + With uniform weights this reduces to the overall pass rate (total passing test units over + total test units). Optional per-dimension weights scale each dimension's test-unit + contribution; a dimension not present in `dimension_weights` uses a weight of `1.0`. Weights + are expected to be positive; negative weights are treated as `0` (the dimension is dropped from + the overall score), and the returned score is always clamped to the `[0, 100]` range. + """ + agg = _aggregate_dimension_units(validation_info) + if not agg: + return 100.0 + weights = dimension_weights or {} + numerator = 0.0 + denominator = 0.0 + for dimension, (n_passed, n) in agg.items(): + # Guard against invalid (negative) weights, which could otherwise push the score + # out of the [0, 100] range + weight = max(0.0, weights.get(dimension, 1.0)) + numerator += weight * n_passed + denominator += weight * n + if not denominator: + return 100.0 + return round(min(100.0, max(0.0, numerator / denominator * 100)), 2) + + def _get_assertion_icon(icon: list[str], length_val: int = 30) -> list[str]: # For each icon, get the assertion icon SVG test from SVG_ICONS_FOR_ASSERTION_TYPES dictionary icon_svg: list[str] = [SVG_ICONS_FOR_ASSERTION_TYPES[icon] for icon in icon] @@ -22557,6 +23394,172 @@ def _transform_w_e_c(values, color, interrogation_performed): ] +def _get_dimension_label(dimension: str | None, lang: str) -> str: + """ + Get the localized display label for a data quality dimension. + + Falls back to English (and finally to a title-cased version of the raw dimension name for + custom/unmapped dimensions) when a translation is not available for the requested language. + """ + key = dimension or "unknown" + entry = VALIDATION_REPORT_TEXT.get(f"dimension_{key}") + if entry: + return entry.get(lang, entry.get("en", key)) + # Custom or unmapped dimension: present the raw name in a readable form + return str(key).replace("_", " ").title() + + +def _transform_step_number_with_dimension( + i_values: list, dimensions: list[str | None], lang: str +) -> list[str]: + """ + Render each step number with a small, color-coded two-letter dimension badge in its top-left + corner. + + The badge is absolutely positioned so it does not shift the step numeral (regardless of how + many digits it has). The full dimension name is exposed via a `title` tooltip, and the badge + color matches the per-dimension colors used in the report's health-score summary. + """ + # Right-align the numeral (GT auto-aligns integer columns to the right, but treats these HTML + # strings as text); place the badge on the leading edge, nudged inward to clear the status bar + is_rtl = lang in RTL_LANGUAGES + text_align = "left" if is_rtl else "right" + badge_side = "right" if is_rtl else "left" + + cells: list[str] = [] + for i_value, dimension in zip(i_values, dimensions): + key = dimension or "unknown" + color = DIMENSION_COLORS.get(key, DIMENSION_COLORS["unknown"]) + if key in DIMENSION_ABBR: + abbr = DIMENSION_ABBR[key] + else: + # Derive a two-letter code from a custom dimension name (first two letters) + letters = "".join(c for c in key if c.isalpha()) + abbr = (letters[:2].upper()) or DIMENSION_ABBR["unknown"] + # Escape the tooltip text since a custom dimension name is user-provided + name = html_module.escape(_get_dimension_label(key, lang)) + badge = ( + f'{abbr}" + ) + cells.append( + f'
{badge}{i_value}
' + ) + return cells + + +def _get_report_text(key: str, lang: str) -> str: + """Safely look up a validation report string with English fallback.""" + entry = VALIDATION_REPORT_TEXT.get(key, {}) + return entry.get(lang, entry.get("en", key)) + + +def _health_score_color(score: float) -> str: + """Map a health score (0-100) to a status color for display. + + Uses darker shades than the warning/error/critical severity palette so the score reads with + sufficient contrast against a white background. + """ + if score >= 90: + return "#2E7D32" # green + if score >= 75: + return "#A15C00" # amber + return "#C62828" # red + + +def _create_health_score_html( + validation_info: list[_ValidationInfo], + lang: str, + dimension_weights: dict[str, float] | None = None, + show_divider: bool = True, +) -> str: + """ + Build the health-score summary block (overall score + per-dimension breakdown) for the + validation report footer. Returns an empty string when there are no scorable steps. + + When `show_divider` is `True`, a dotted bottom border is drawn to separate the block from the + timings row that follows it. It should be `False` when no timings row is displayed, so the + divider doesn't double up with the next footer separator. + """ + dimension_scores = _compute_dimension_scores(validation_info) + if not dimension_scores: + return "" + + overall = _compute_health_score(validation_info, dimension_weights=dimension_weights) + health_label = _get_report_text("report_health_score", lang) + dimensions_label = _get_report_text("report_dimension_scores", lang) + + # Order the per-dimension chips using the canonical dimension order, appending any custom + # dimensions (not in the canonical list) at the end + ordered_dimensions = [d for d in DIMENSION_NAMES if d in dimension_scores] + ordered_dimensions += [d for d in dimension_scores if d not in DIMENSION_NAMES] + + chips: list[str] = [] + for dimension in ordered_dimensions: + score = dimension_scores[dimension] + color = DIMENSION_COLORS.get(dimension, DIMENSION_COLORS["unknown"]) + label = html_module.escape(_get_dimension_label(dimension, lang)) + chips.append( + '' + f'' + f"{label}" + "{score:g}%' + "" + ) + + # A shared caption style so the "HEALTH SCORE" and "DIMENSION SCORES" lines sit in parallel + caption_style = ( + "font-weight: 600; color: #888888; font-size: 10px; text-transform: uppercase; " + "letter-spacing: 0.5px; padding-right: 8px; vertical-align: middle;" + ) + + # The score sits in a box that pulses (glow + scale). The animation gets more vigorous — faster + # and with a larger pulse/glow — as the score approaches 100%. The keyframes reference CSS + # custom properties so a single definition adapts to the per-report values set inline. + color = _health_score_color(overall) + intensity = max(0.0, min(1.0, overall / 100)) + duration = round(2.3 - 1.6 * intensity, 2) # 2.3s (low) -> 0.7s (100%) + scale = round(1 + 0.09 * intensity, 3) # up to 1.09x + glow = round(3 + 15 * intensity) # 3px -> 18px halo + + style_block = ( + "" + ) + score_box = ( + f'{overall:.0f}%' + ) + + # Only draw the dotted divider when a timings row follows this block + outer_style = ( + "padding: 3px 2px 8px 2px; border-bottom: 1px dotted #D3D3D3;" + if show_divider + else "padding: 3px 2px;" + ) + return ( + f"{style_block}" + f'
' + f'{health_label}' + f"{score_box}" + '
' + f'{dimensions_label}' + f"{''.join(chips)}" + "
" + "
" + ) + + def _transform_assertion_str( assertion_str: list[str], brief_str: list[str | None], @@ -25723,6 +26726,7 @@ def agg_validator( brief: str | bool | None = None, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: # Dynamically generated aggregate validator. # This method is generated per assertion type and forwards all arguments @@ -25736,6 +26740,7 @@ def agg_validator( brief=brief, actions=actions, active=active, + dimension=dimension, ) # Manually set function identity so this behaves like a real method. diff --git a/pointblank/validate.pyi b/pointblank/validate.pyi index 8f07892a6c..55240dd472 100644 --- a/pointblank/validate.pyi +++ b/pointblank/validate.pyi @@ -41,7 +41,11 @@ class PointblankConfig: report_incl_footer: bool = ... report_incl_footer_timings: bool = ... report_incl_footer_notes: bool = ... + report_incl_dimensions: bool = ... preview_incl_header: bool = ... + dimension_map: dict[str, str] | None = ... + dimension_weights: dict[str, float] | None = ... + dimension_thresholds: dict[str, float] | None = ... def __repr__(self) -> str: ... def config( @@ -49,7 +53,11 @@ def config( report_incl_footer: bool = True, report_incl_footer_timings: bool = True, report_incl_footer_notes: bool = True, + report_incl_dimensions: bool = False, preview_incl_header: bool = True, + dimension_map: dict[str, str] | None = None, + dimension_weights: dict[str, float] | None = None, + dimension_thresholds: dict[str, float] | None = None, ) -> PointblankConfig: ... def load_dataset( dataset: Literal["small_table", "game_revenue", "nycflights", "global_sales"] = "small_table", @@ -97,6 +105,7 @@ class _ValidationInfo: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> _ValidationInfo: ... i: int | None = ... i_o: int | None = ... @@ -115,6 +124,7 @@ class _ValidationInfo: label: str | None = ... brief: str | None = ... autobrief: str | None = ... + dimension: str | None = ... active: bool | Callable | None = ... eval_error: bool | None = ... all_passed: bool | None = ... @@ -182,6 +192,7 @@ class Validate: brief: bool = False, actions=None, active: bool = True, + dimension: str | None = None, ): ... def set_tbl( self, tbl: Any, tbl_name: str | None = None, label: str | None = None @@ -199,6 +210,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_lt( self, @@ -212,6 +224,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_eq( self, @@ -225,6 +238,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_ne( self, @@ -238,6 +252,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_ge( self, @@ -251,6 +266,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_le( self, @@ -264,6 +280,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_between( self, @@ -279,6 +296,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_outside( self, @@ -294,6 +312,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_in_set( self, @@ -306,6 +325,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_not_in_set( self, @@ -318,6 +338,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_increasing( self, @@ -332,6 +353,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_decreasing( self, @@ -346,6 +368,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_null( self, @@ -356,6 +379,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_not_null( self, @@ -366,6 +390,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_regex( self, @@ -380,6 +405,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_within_spec( self, @@ -393,6 +419,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_vals_expr( self, @@ -403,6 +430,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_exists( self, @@ -411,6 +439,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_pct_null( self, @@ -421,6 +450,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_pct_missing( self, @@ -433,6 +463,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_missing_coded( self, @@ -444,6 +475,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_missing_only_coded( self, @@ -458,6 +490,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def rows_distinct( self, @@ -468,6 +501,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def rows_complete( self, @@ -478,6 +512,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_missing_consistent( self, @@ -490,6 +525,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def prompt( self, @@ -505,6 +541,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_schema_match( self, @@ -519,6 +556,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def row_count_match( self, @@ -530,6 +568,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def data_freshness( self, @@ -543,6 +582,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def col_count_match( self, @@ -553,6 +593,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def tbl_match( self, @@ -562,6 +603,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def conjointly( self, @@ -571,6 +613,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def specially( self, @@ -580,6 +623,7 @@ class Validate: actions: Actions | None = None, brief: str | bool | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: ... def interrogate( self, @@ -641,6 +685,12 @@ class Validate: verify_ssl: bool = True, max_reprompts: int = 1, ) -> Any: ... + def get_dimension_scores(self) -> dict[str, float]: ... + def get_health_score(self) -> float: ... + def assert_dimension_scores( + self, thresholds: dict[str, float] | None = None, message: str | None = None + ) -> None: ... + def get_scorecard(self, title: str | None = ":default:") -> GT: ... def get_json_report( self, use_fields: list[str] | None = None, exclude_fields: list[str] | None = None ) -> str: ... @@ -656,6 +706,7 @@ class Validate: incl_footer: bool | None = None, incl_footer_timings: bool | None = None, incl_footer_notes: bool | None = None, + incl_dimensions: bool | None = None, ) -> GT: ... def get_step_report( self, @@ -683,6 +734,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column sum to a value eq some `value`. @@ -697,6 +749,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -719,6 +773,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column sum to a value gt some `value`. @@ -733,6 +788,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -755,6 +812,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column sum to a value ge some `value`. @@ -769,6 +827,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -791,6 +851,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column sum to a value lt some `value`. @@ -805,6 +866,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -827,6 +890,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column sum to a value le some `value`. @@ -841,6 +905,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -863,6 +929,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column avg to a value eq some `value`. @@ -877,6 +944,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -899,6 +968,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column avg to a value gt some `value`. @@ -913,6 +983,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -935,6 +1007,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column avg to a value ge some `value`. @@ -949,6 +1022,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -971,6 +1046,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column avg to a value lt some `value`. @@ -985,6 +1061,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -1007,6 +1085,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column avg to a value le some `value`. @@ -1021,6 +1100,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -1043,6 +1124,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column sd to a value eq some `value`. @@ -1057,6 +1139,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -1079,6 +1163,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column sd to a value gt some `value`. @@ -1093,6 +1178,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -1115,6 +1202,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column sd to a value ge some `value`. @@ -1129,6 +1217,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -1151,6 +1241,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column sd to a value lt some `value`. @@ -1165,6 +1256,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. @@ -1187,6 +1280,7 @@ class Validate: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, ) -> Validate: """Assert the values in a column sd to a value le some `value`. @@ -1201,6 +1295,8 @@ class Validate: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. diff --git a/scripts/generate_agg_validate_pyi.py b/scripts/generate_agg_validate_pyi.py index 50fb1b79ec..fc8d9eddf0 100644 --- a/scripts/generate_agg_validate_pyi.py +++ b/scripts/generate_agg_validate_pyi.py @@ -51,6 +51,7 @@ def _extract_body(func) -> str: brief: str | bool = False, actions: Actions | None = None, active: bool | Callable = True, + dimension: str | None = None, """ DOCSTRING = """ @@ -65,6 +66,8 @@ def _extract_body(func) -> str: brief (str | bool, optional): Explanation of validation operation. Defaults to False. actions (Actions | None, optional): Actions to take after validation. Defaults to None. active (bool, optional): Whether to activate the validation. Defaults to True. + dimension (str | None, optional): Data quality dimension for health scoring. If None, + inferred automatically from the assertion type. Defaults to None. Returns: Validate: A `Validate` instance with the new validation method added. diff --git a/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_duckdb/validation_report.json b/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_duckdb/validation_report.json index 8c0c26cc39..7f8be5d8c9 100644 --- a/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_duckdb/validation_report.json +++ b/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_duckdb/validation_report.json @@ -12,6 +12,7 @@ "thresholds": "Thresholds(warning=None, error=None, critical=None)", "label": null, "brief": null, + "dimension": "validity", "active": true, "all_passed": true, "n": 4, diff --git a/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_parquet/validation_report.json b/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_parquet/validation_report.json index 8c0c26cc39..7f8be5d8c9 100644 --- a/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_parquet/validation_report.json +++ b/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_parquet/validation_report.json @@ -12,6 +12,7 @@ "thresholds": "Thresholds(warning=None, error=None, critical=None)", "label": null, "brief": null, + "dimension": "validity", "active": true, "all_passed": true, "n": 4, diff --git a/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_pd/validation_report.json b/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_pd/validation_report.json index 8c0c26cc39..7f8be5d8c9 100644 --- a/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_pd/validation_report.json +++ b/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_pd/validation_report.json @@ -12,6 +12,7 @@ "thresholds": "Thresholds(warning=None, error=None, critical=None)", "label": null, "brief": null, + "dimension": "validity", "active": true, "all_passed": true, "n": 4, diff --git a/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_pl/validation_report.json b/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_pl/validation_report.json index 8c0c26cc39..7f8be5d8c9 100644 --- a/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_pl/validation_report.json +++ b/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_pl/validation_report.json @@ -12,6 +12,7 @@ "thresholds": "Thresholds(warning=None, error=None, critical=None)", "label": null, "brief": null, + "dimension": "validity", "active": true, "all_passed": true, "n": 4, diff --git a/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_pyspark/validation_report.json b/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_pyspark/validation_report.json index 8c0c26cc39..7f8be5d8c9 100644 --- a/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_pyspark/validation_report.json +++ b/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_pyspark/validation_report.json @@ -12,6 +12,7 @@ "thresholds": "Thresholds(warning=None, error=None, critical=None)", "label": null, "brief": null, + "dimension": "validity", "active": true, "all_passed": true, "n": 4, diff --git a/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_sqlite/validation_report.json b/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_sqlite/validation_report.json index 8c0c26cc39..7f8be5d8c9 100644 --- a/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_sqlite/validation_report.json +++ b/tests/snapshots/test_validate/test_validation_report_interrogate_snap/tbl_sqlite/validation_report.json @@ -12,6 +12,7 @@ "thresholds": "Thresholds(warning=None, error=None, critical=None)", "label": null, "brief": null, + "dimension": "validity", "active": true, "all_passed": true, "n": 4, diff --git a/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_duckdb/validation_report.json b/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_duckdb/validation_report.json index bc749c15dc..67631cad38 100644 --- a/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_duckdb/validation_report.json +++ b/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_duckdb/validation_report.json @@ -12,6 +12,7 @@ "thresholds": "Thresholds(warning=None, error=None, critical=None)", "label": null, "brief": null, + "dimension": "validity", "active": true, "all_passed": null, "n": null, diff --git a/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_parquet/validation_report.json b/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_parquet/validation_report.json index bc749c15dc..67631cad38 100644 --- a/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_parquet/validation_report.json +++ b/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_parquet/validation_report.json @@ -12,6 +12,7 @@ "thresholds": "Thresholds(warning=None, error=None, critical=None)", "label": null, "brief": null, + "dimension": "validity", "active": true, "all_passed": null, "n": null, diff --git a/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_pd/validation_report.json b/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_pd/validation_report.json index bc749c15dc..67631cad38 100644 --- a/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_pd/validation_report.json +++ b/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_pd/validation_report.json @@ -12,6 +12,7 @@ "thresholds": "Thresholds(warning=None, error=None, critical=None)", "label": null, "brief": null, + "dimension": "validity", "active": true, "all_passed": null, "n": null, diff --git a/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_pl/validation_report.json b/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_pl/validation_report.json index bc749c15dc..67631cad38 100644 --- a/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_pl/validation_report.json +++ b/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_pl/validation_report.json @@ -12,6 +12,7 @@ "thresholds": "Thresholds(warning=None, error=None, critical=None)", "label": null, "brief": null, + "dimension": "validity", "active": true, "all_passed": null, "n": null, diff --git a/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_pyspark/validation_report.json b/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_pyspark/validation_report.json index bc749c15dc..67631cad38 100644 --- a/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_pyspark/validation_report.json +++ b/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_pyspark/validation_report.json @@ -12,6 +12,7 @@ "thresholds": "Thresholds(warning=None, error=None, critical=None)", "label": null, "brief": null, + "dimension": "validity", "active": true, "all_passed": null, "n": null, diff --git a/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_sqlite/validation_report.json b/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_sqlite/validation_report.json index bc749c15dc..67631cad38 100644 --- a/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_sqlite/validation_report.json +++ b/tests/snapshots/test_validate/test_validation_report_no_interrogate_snap/tbl_sqlite/validation_report.json @@ -12,6 +12,7 @@ "thresholds": "Thresholds(warning=None, error=None, critical=None)", "label": null, "brief": null, + "dimension": "validity", "active": true, "all_passed": null, "n": null, diff --git a/tests/test_quality_dimensions.py b/tests/test_quality_dimensions.py new file mode 100644 index 0000000000..eb4f36e081 --- /dev/null +++ b/tests/test_quality_dimensions.py @@ -0,0 +1,650 @@ +"""Tests for data quality dimensions and health scoring (PLAN_09).""" + +import pytest + +import pointblank as pb +from pointblank.validate import ( + Validate, + FinalActions, + get_validation_summary, + global_config, + _infer_dimension_from_assertion_type, + _base_dimension_from_assertion_type, + _compute_dimension_scores, + _compute_health_score, + _aggregate_dimension_units, + _validation_info_as_dict, + _get_dimension_label, +) + + +@pytest.fixture(autouse=True) +def reset_global_config(): + """Save and restore the dimension-related global config around each test.""" + saved = ( + global_config.dimension_map, + global_config.dimension_weights, + global_config.dimension_thresholds, + global_config.report_incl_dimensions, + ) + global_config.dimension_map = None + global_config.dimension_weights = None + global_config.dimension_thresholds = None + global_config.report_incl_dimensions = False + yield + ( + global_config.dimension_map, + global_config.dimension_weights, + global_config.dimension_thresholds, + global_config.report_incl_dimensions, + ) = saved + + +@pytest.fixture +def simple_df(): + pl = pytest.importorskip("polars") + # `a`: 1 null (completeness 4/5); `b`: 2 non-positive (validity 3/5); `id`: distinct + return pl.DataFrame( + { + "a": [1, 2, 3, None, 5], + "b": [10, -3, 5, 0, 8], + "id": [1, 2, 3, 4, 5], + } + ) + + +# --------------------------------------------------------------------------- +# Dimension inference +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "assertion_type,expected", + [ + ("col_vals_not_null", "completeness"), + ("col_pct_null", "completeness"), + ("rows_complete", "completeness"), + ("col_vals_gt", "validity"), + ("col_vals_regex", "validity"), + ("col_schema_match", "validity"), + ("rows_distinct", "uniqueness"), + ("col_missing_consistent", "consistency"), + ("conjointly", "consistency"), + ("tbl_match", "consistency"), + ("data_freshness", "timeliness"), + ("row_count_match", "volume"), + ("col_count_match", "volume"), + # dynamically generated aggregate methods -> validity via regex fallback + ("col_sum_eq", "validity"), + ("col_avg_gt", "validity"), + ("col_sd_le", "validity"), + ], +) +def test_infer_dimension_from_assertion_type(assertion_type, expected): + assert _infer_dimension_from_assertion_type(assertion_type) == expected + + +def test_infer_dimension_none_and_unknown(): + assert _infer_dimension_from_assertion_type(None) is None + assert _infer_dimension_from_assertion_type("not_a_real_assertion") == "unknown" + + +def test_auto_inference_on_steps(simple_df): + v = ( + Validate(data=simple_df) + .col_vals_not_null(columns="a") + .col_vals_gt(columns="b", value=0) + .rows_distinct(columns_subset=["id"]) + .col_sum_eq(columns="id", value=15) + ) + dims = [s.dimension for s in v.validation_info] + assert dims == ["completeness", "validity", "uniqueness", "validity"] + + +def test_per_step_dimension_override(simple_df): + v = ( + Validate(data=simple_df) + .col_vals_gt(columns="b", value=0, dimension="consistency") + .col_vals_gt(columns="b", value=0) + ) + assert v.validation_info[0].dimension == "consistency" + assert v.validation_info[1].dimension == "validity" + + +def test_agg_method_dimension_override(simple_df): + v = Validate(data=simple_df).col_sum_eq(columns="id", value=15, dimension="volume") + assert v.validation_info[0].dimension == "volume" + + +def test_config_dimension_map_remap(simple_df): + pb.config(dimension_map={"col_vals_gt": "consistency"}) + v = Validate(data=simple_df).col_vals_gt(columns="b", value=0) + assert v.validation_info[0].dimension == "consistency" + + +def test_config_dimension_map_does_not_override_explicit(simple_df): + pb.config(dimension_map={"col_vals_gt": "consistency"}) + v = Validate(data=simple_df).col_vals_gt(columns="b", value=0, dimension="validity") + # Explicit per-step dimension wins over the config remap + assert v.validation_info[0].dimension == "validity" + + +# --------------------------------------------------------------------------- +# Scoring +# --------------------------------------------------------------------------- + + +def test_dimension_scores(simple_df): + v = ( + Validate(data=simple_df) + .col_vals_not_null(columns="a") # completeness 4/5 = 80 + .col_vals_gt(columns="b", value=0) # validity 3/5 = 60 + .rows_distinct(columns_subset=["id"]) # uniqueness 5/5 = 100 + .interrogate() + ) + scores = v.get_dimension_scores() + assert scores == {"completeness": 80.0, "validity": 60.0, "uniqueness": 100.0} + + +def test_health_score_unweighted(simple_df): + v = ( + Validate(data=simple_df) + .col_vals_not_null(columns="a") + .col_vals_gt(columns="b", value=0) + .rows_distinct(columns_subset=["id"]) + .interrogate() + ) + # (4 + 3 + 5) / (5 + 5 + 5) = 12/15 = 80.0 + assert v.get_health_score() == 80.0 + + +def test_health_score_weighted(simple_df): + pb.config(dimension_weights={"validity": 3.0}) + v = ( + Validate(data=simple_df) + .col_vals_not_null(columns="a") # completeness 4/5 + .col_vals_gt(columns="b", value=0) # validity 3/5, weight 3 + .interrogate() + ) + # num = 1*4 + 3*3 = 13 ; den = 1*5 + 3*5 = 20 -> 65.0 + assert v.get_health_score() == 65.0 + + +def test_scores_empty_when_not_interrogated(simple_df): + v = Validate(data=simple_df).col_vals_gt(columns="b", value=0) + assert v.get_dimension_scores() == {} + assert v.get_health_score() == 100.0 + + +def test_inactive_steps_excluded_from_scoring(simple_df): + v = ( + Validate(data=simple_df) + .col_vals_gt(columns="b", value=0, active=False) # would be validity 3/5 + .col_vals_not_null(columns="a") # completeness 4/5 + .interrogate() + ) + scores = v.get_dimension_scores() + assert "validity" not in scores + assert scores == {"completeness": 80.0} + + +def test_aggregate_dimension_units(simple_df): + v = ( + Validate(data=simple_df) + .col_vals_not_null(columns="a") + .col_vals_gt(columns="b", value=0) + .interrogate() + ) + agg = _aggregate_dimension_units(v.validation_info) + assert agg == {"completeness": [4, 5], "validity": [3, 5]} + + +def test_compute_helpers_directly(simple_df): + v = ( + Validate(data=simple_df) + .col_vals_not_null(columns="a") + .col_vals_gt(columns="b", value=0) + .interrogate() + ) + assert _compute_dimension_scores(v.validation_info) == { + "completeness": 80.0, + "validity": 60.0, + } + assert _compute_health_score(v.validation_info) == 70.0 + assert _compute_health_score(v.validation_info, dimension_weights={"validity": 3.0}) == 65.0 + + +# --------------------------------------------------------------------------- +# assert_dimension_scores +# --------------------------------------------------------------------------- + + +def test_assert_dimension_scores_pass(simple_df): + v = Validate(data=simple_df).col_vals_not_null(columns="a").interrogate() + # completeness is 80, so a 75 minimum passes (no exception) + v.assert_dimension_scores(thresholds={"completeness": 75}) + + +def test_assert_dimension_scores_raises(simple_df): + v = Validate(data=simple_df).col_vals_gt(columns="b", value=0).interrogate() + with pytest.raises(AssertionError, match="validity"): + v.assert_dimension_scores(thresholds={"validity": 90}) + + +def test_assert_dimension_scores_uses_config_default(simple_df): + pb.config(dimension_thresholds={"validity": 90}) + v = Validate(data=simple_df).col_vals_gt(columns="b", value=0).interrogate() + with pytest.raises(AssertionError): + v.assert_dimension_scores() + + +def test_assert_dimension_scores_no_thresholds_is_noop(simple_df): + v = Validate(data=simple_df).col_vals_gt(columns="b", value=0).interrogate() + # No thresholds set anywhere -> no exception + v.assert_dimension_scores() + + +def test_assert_dimension_scores_custom_message(simple_df): + v = Validate(data=simple_df).col_vals_gt(columns="b", value=0).interrogate() + with pytest.raises(AssertionError, match="custom failure"): + v.assert_dimension_scores(thresholds={"validity": 90}, message="custom failure") + + +def test_assert_dimension_scores_ignores_absent_dimension(simple_df): + v = Validate(data=simple_df).col_vals_gt(columns="b", value=0).interrogate() + # 'timeliness' is not present in the plan, so it is ignored + v.assert_dimension_scores(thresholds={"timeliness": 100}) + + +# --------------------------------------------------------------------------- +# Report + dict integration +# --------------------------------------------------------------------------- + + +def test_dimension_in_validation_info_dict(simple_df): + v = Validate(data=simple_df).col_vals_not_null(columns="a").interrogate() + d = _validation_info_as_dict(v.validation_info) + assert "dimension" in d + assert d["dimension"] == ["completeness"] + + +def test_dimension_in_json_report(simple_df): + import json + + v = Validate(data=simple_df).col_vals_gt(columns="b", value=0).interrogate() + report = json.loads(v.get_json_report()) + assert report[0]["dimension"] == "validity" + + +def test_dimension_via_yaml(simple_df): + yaml_cfg = """ +tbl: small_table +steps: + - col_vals_gt: + columns: d + value: 0 + dimension: consistency +""" + v = pb.yaml_interrogate(yaml_cfg) + assert v.validation_info[0].dimension == "consistency" + + +def test_report_includes_dimension_badges_and_scores(simple_df): + v = ( + Validate(data=simple_df, tbl_name="demo") + .col_vals_not_null(columns="a") + .col_vals_gt(columns="b", value=0) + .interrogate() + ) + # The dimension display is opt-in + html = v.get_tabular_report(incl_dimensions=True).as_raw_html() + # No standalone dimension column; instead a compact badge with a tooltip on the step number + assert 'title="Completeness"' in html + assert 'title="Validity"' in html + assert ">CM<" in html # completeness abbreviation badge + assert ">VA<" in html # validity abbreviation badge + assert "Health Score" in html + + +def test_dimension_display_off_by_default(simple_df): + v = ( + Validate(data=simple_df, tbl_name="demo") + .col_vals_not_null(columns="a") + .col_vals_gt(columns="b", value=0) + .interrogate() + ) + # By default the dimension badge and health-score block are not shown (opt-in) + html = v.get_tabular_report().as_raw_html() + assert 'title="Completeness"' not in html + assert ">CM<" not in html + assert "Health Score" not in html + + +def test_dimension_display_via_global_config(simple_df): + pb.config(report_incl_dimensions=True) + v = Validate(data=simple_df, tbl_name="demo").col_vals_not_null(columns="a").interrogate() + html = v.get_tabular_report().as_raw_html() + assert 'title="Completeness"' in html + assert "Health Score" in html + + +def test_health_block_divider_only_with_timings(simple_df): + v = Validate(data=simple_df).col_vals_not_null(columns="a").interrogate() + border = "border-bottom: 1px dotted #D3D3D3" + with_timings = v.get_tabular_report( + incl_dimensions=True, incl_footer_timings=True + ).as_raw_html() + without_timings = v.get_tabular_report( + incl_dimensions=True, incl_footer_timings=False + ).as_raw_html() + assert border in with_timings + assert border not in without_timings + + +def test_report_renders_without_interrogation(simple_df): + v = Validate(data=simple_df).col_vals_not_null(columns="a") + html = v.get_tabular_report(incl_dimensions=True).as_raw_html() + # Dimension badge still shown for the plan, but no health-score block yet + assert 'title="Completeness"' in html + assert "Health Score" not in html + + +# --------------------------------------------------------------------------- +# Scorecard +# --------------------------------------------------------------------------- + + +def test_get_scorecard_renders(simple_df): + v = ( + Validate(data=simple_df, tbl_name="orders") + .col_vals_not_null(columns="a") + .col_vals_gt(columns="b", value=0) + .rows_distinct(columns_subset=["id"]) + .interrogate() + ) + html = v.get_scorecard().as_raw_html() + assert "Health Score" in html + assert "80%" in html + assert "Completeness" in html + assert "Dimension Scores" in html + + +def test_get_scorecard_empty(simple_df): + v = Validate(data=simple_df).col_vals_gt(columns="b", value=0) # not interrogated + # Should render a minimal table without raising + assert len(v.get_scorecard().as_raw_html()) > 0 + + +def test_get_scorecard_title_none(simple_df): + v = Validate(data=simple_df).col_vals_not_null(columns="a").interrogate() + html = v.get_scorecard(title=":none:").as_raw_html() + assert len(html) > 0 + + +# --------------------------------------------------------------------------- +# Validation summary integration +# --------------------------------------------------------------------------- + + +def test_validation_summary_includes_scores(): + pl = pytest.importorskip("polars") + captured = {} + + def capture(): + summary = get_validation_summary() + captured["dimension_scores"] = summary["dimension_scores"] + captured["overall_health_score"] = summary["overall_health_score"] + + df = pl.DataFrame({"a": [1, 2, None, 4, 5], "b": [10, -3, 5, 0, 8]}) + ( + Validate(data=df, final_actions=FinalActions(capture)) + .col_vals_not_null(columns="a") + .col_vals_gt(columns="b", value=0) + .interrogate() + ) + + assert captured["dimension_scores"] == {"completeness": 80.0, "validity": 60.0} + assert captured["overall_health_score"] == 70.0 + + +# --------------------------------------------------------------------------- +# Labels +# --------------------------------------------------------------------------- + + +def test_dimension_label_english(): + assert _get_dimension_label("completeness", "en") == "Completeness" + assert _get_dimension_label("validity", "en") == "Validity" + + +def test_dimension_label_localized(): + assert _get_dimension_label("completeness", "fr") == "Complétude" + assert _get_dimension_label("validity", "de") == "Gültigkeit" + + +def test_dimension_label_localized_non_latin(): + # Non-Latin languages are also translated (not falling back to English) + assert _get_dimension_label("completeness", "ja") == "完全性" + assert _get_dimension_label("timeliness", "ru") == "Своевременность" + + +def test_dimension_label_fallback_to_english_for_missing_language(): + # A language code without an explicit translation falls back to English + assert _get_dimension_label("completeness", "xx") == "Completeness" + + +def test_dimension_translation_keys_cover_all_languages(): + # Every dimension/scoring report string must be translated for all supported languages + from pointblank._constants import REPORTING_LANGUAGES + from pointblank._constants_translations import VALIDATION_REPORT_TEXT + + keys = [ + "report_col_dimension", + "report_col_score", + "report_health_score", + "report_dimension_scores", + "dimension_completeness", + "dimension_consistency", + "dimension_validity", + "dimension_uniqueness", + "dimension_timeliness", + "dimension_volume", + "dimension_unknown", + ] + for key in keys: + langs = set(VALIDATION_REPORT_TEXT[key]) + missing = [lang for lang in REPORTING_LANGUAGES if lang not in langs] + assert not missing, f"{key} is missing translations for: {missing}" + + +def test_dimension_label_custom_dimension_title_cased(): + # A custom, unmapped dimension is presented in a readable title-cased form + assert _get_dimension_label("business_rules", "en") == "Business Rules" + + +def test_custom_dimension_scored_and_rendered(): + pl = pytest.importorskip("polars") + df = pl.DataFrame({"a": [1, 2, 3, None, 5]}) + v = ( + Validate(data=df, tbl_name="t") + .col_vals_not_null(columns="a", dimension="business_rules") + .interrogate() + ) + assert v.get_dimension_scores() == {"business_rules": 80.0} + html = v.get_tabular_report(incl_dimensions=True).as_raw_html() + # Custom dimension: tooltip shows the title-cased name; badge derives a two-letter code + assert 'title="Business Rules"' in html + assert ">BU<" in html + + +def test_eval_error_step_excluded_from_scoring(): + pl = pytest.importorskip("polars") + df = pl.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + v = ( + Validate(df) + .col_vals_gt(columns="nonexistent", value=0) # eval_error -> excluded + .col_vals_gt(columns="b", value=0) # validity 3/3 + .interrogate() + ) + + # The broken step must not drag the score to 0 + assert v.get_dimension_scores() == {"validity": 100.0} + assert v.get_health_score() == 100.0 + + +def test_only_eval_error_yields_empty_scores(): + pl = pytest.importorskip("polars") + df = pl.DataFrame({"a": [1, 2, 3]}) + v = Validate(df).col_vals_gt(columns="nonexistent", value=0).interrogate() + + assert v.get_dimension_scores() == {} + assert v.get_health_score() == 100.0 + + +def test_base_dimension_ignores_global_config(simple_df): + pb.config(dimension_map={"col_vals_gt": "consistency"}) + + # The base inference ignores the global override... + assert _base_dimension_from_assertion_type("col_vals_gt") == "validity" + + # ...while the effective inference respects it + assert _infer_dimension_from_assertion_type("col_vals_gt") == "consistency" + + +def test_to_code_emits_dimension_only_when_overridden(simple_df): + v = ( + Validate(data=simple_df) + .col_vals_gt(columns="b", value=0) # inferred validity -> no dimension= + .col_vals_gt(columns="b", value=0, dimension="consistency") # override -> dimension= + .col_vals_not_null(columns="a") # inferred completeness -> no dimension= + ) + code = v.to_code() + + assert code.count("dimension=") == 1 + assert 'dimension="consistency"' in code + + +def test_to_code_round_trip_preserves_dimension(simple_df): + v = ( + Validate(data=simple_df) + .col_vals_gt(columns="b", value=0, dimension="consistency") + .col_vals_not_null(columns="a") + ) + code = v.to_code() + ns = {"pb": pb, "your_data": simple_df} + exec(code.replace("validation\n", ""), ns) + rebuilt = ns["validation"].interrogate() + dims = [s.dimension for s in rebuilt.validation_info] + + assert dims == ["consistency", "completeness"] + + +def test_to_yaml_preserves_dimension_override(simple_df): + v = Validate(data=simple_df).col_vals_gt(columns="b", value=0, dimension="consistency") + yaml_text = v.to_yaml() + + assert "dimension: consistency" in yaml_text + + +def test_assert_dimension_scores_auto_interrogates(simple_df): + # Not interrogated yet; the assertion should interrogate and then evaluate + v = Validate(data=simple_df).col_vals_gt(columns="b", value=0) # validity 3/5 = 60 + with pytest.raises(AssertionError, match="validity"): + v.assert_dimension_scores(thresholds={"validity": 90}) + + +def test_custom_dimension_name_is_html_escaped(): + pl = pytest.importorskip("polars") + df = pl.DataFrame({"a": [1, 2, 3]}) + v = ( + Validate(df, tbl_name="t") + .col_vals_not_null(columns="a", dimension='weird"') + .interrogate() + ) + html = v.get_tabular_report(incl_dimensions=True).as_raw_html() + + # The custom name is title-cased for display; the raw (unescaped) form must not leak into the + # HTML, and the escaped entities must be present instead + assert 'Weird"' not in html + assert "Weird"<X>" in html + + +def test_health_score_negative_weight_clamped(simple_df): + v = ( + Validate(data=simple_df) + .col_vals_not_null(columns="a") + .col_vals_gt(columns="b", value=0) + .interrogate() + ) + # A negative weight must not push the score outside [0, 100] + score = _compute_health_score(v.validation_info, dimension_weights={"validity": -1.0}) + + assert 0.0 <= score <= 100.0 + + +def test_health_score_all_zero_weights(simple_df): + v = ( + Validate(data=simple_df) + .col_vals_not_null(columns="a") + .col_vals_gt(columns="b", value=0) + .interrogate() + ) + + # All-zero weights -> no denominator -> defaults to 100.0 (same as the no-data case) + assert ( + _compute_health_score( + v.validation_info, dimension_weights={"completeness": 0.0, "validity": 0.0} + ) + == 100.0 + ) + + +def test_scorecard_message_distinguishes_interrogation_state(simple_df): + # Interrogated but nothing scorable (all steps inactive) -> "no validation steps" message + v_inter = ( + Validate(data=simple_df, tbl_name="t") + .col_vals_gt(columns="b", value=0, active=False) + .interrogate() + ) + html_inter = v_inter.get_scorecard().as_raw_html() + + assert "No Interrogation Performed" not in html_inter + assert "NO VALIDATION STEPS" in html_inter + + # Not interrogated -> "no interrogation performed" message + v_noninter = Validate(data=simple_df, tbl_name="t").col_vals_gt(columns="b", value=0) + + assert "No Interrogation Performed" in v_noninter.get_scorecard().as_raw_html() + + +# --------------------------------------------------------------------------- +# Dimension survives step expansion (column selectors, segments) +# --------------------------------------------------------------------------- + + +def test_dimension_preserved_on_column_selector_expansion(): + pl = pytest.importorskip("polars") + df = pl.DataFrame({"a1": [1, 2, 3], "a2": [4, 5, 6], "b": [7, 8, 9]}) + # `matches("a")` expands into two steps at interrogation time + v = pb.Validate(df).col_vals_gt(columns=pb.matches("a"), value=0).interrogate() + assert len(v.validation_info) == 2 + assert all(s.dimension == "validity" for s in v.validation_info) + + +def test_dimension_preserved_on_segment_expansion(): + pl = pytest.importorskip("polars") + df = pl.DataFrame({"grp": ["x", "x", "y", "y"], "v": [1, 2, 3, 4]}) + v = pb.Validate(df).col_vals_gt(columns="v", value=0, segments="grp").interrogate() + assert len(v.validation_info) == 2 + assert all(s.dimension == "validity" for s in v.validation_info) + + +def test_dimension_override_preserved_on_segment_expansion(): + pl = pytest.importorskip("polars") + df = pl.DataFrame({"grp": ["x", "x", "y", "y"], "v": [1, 2, 3, 4]}) + v = ( + pb.Validate(df) + .col_vals_gt(columns="v", value=0, segments="grp", dimension="consistency") + .interrogate() + ) + assert [s.dimension for s in v.validation_info] == ["consistency", "consistency"] diff --git a/tests/test_validate.py b/tests/test_validate.py index 6ff03cdfaa..4898c43232 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -841,6 +841,7 @@ def test_validation_plan_and_interrogation(request, tbl_fixture) -> None: "label", "brief", "autobrief", + "dimension", "active", "eval_error", "all_passed", @@ -924,6 +925,7 @@ def test_validation_plan_and_interrogation(request, tbl_fixture) -> None: "label", "brief", "autobrief", + "dimension", "active", "eval_error", "all_passed", @@ -11467,10 +11469,14 @@ def test_pointblank_config_class() -> None: assert config.report_incl_header is True assert config.report_incl_footer is True assert config.preview_incl_header is True + assert config.report_incl_dimensions is False + assert config.dimension_map is None + assert config.dimension_weights is None + assert config.dimension_thresholds is None assert ( str(config) - == "PointblankConfig(report_incl_header=True, report_incl_footer=True, report_incl_footer_timings=True, report_incl_footer_notes=True, preview_incl_header=True)" + == "PointblankConfig(report_incl_header=True, report_incl_footer=True, report_incl_footer_timings=True, report_incl_footer_notes=True, report_incl_dimensions=False, preview_incl_header=True, dimension_map=None, dimension_weights=None, dimension_thresholds=None)" ) diff --git a/user_guide/04-post-interrogation/.gitignore b/user_guide/04-post-interrogation/.gitignore new file mode 100644 index 0000000000..ad293093b0 --- /dev/null +++ b/user_guide/04-post-interrogation/.gitignore @@ -0,0 +1,2 @@ +/.quarto/ +**/*.quarto_ipynb diff --git a/user_guide/04-post-interrogation/05-quality-dimensions-and-scoring.qmd b/user_guide/04-post-interrogation/05-quality-dimensions-and-scoring.qmd new file mode 100644 index 0000000000..4de36f9dcd --- /dev/null +++ b/user_guide/04-post-interrogation/05-quality-dimensions-and-scoring.qmd @@ -0,0 +1,239 @@ +--- +title: "Quality Dimensions & Scoring" +jupyter: python3 +toc-expand: 2 +html-table-processing: none +bread-crumbs: true +--- + +```{python} +#| echo: false +#| output: false +import pointblank as pb +``` + +Beyond knowing *which* validation steps passed or failed, it's often useful to understand data +quality along broad, well-understood **dimensions** (completeness, validity, uniqueness, and so +on) and to roll everything up into a single **health score** that a governance stakeholder can +track over time. + +Pointblank does this automatically. Every validation step is tagged with a data quality dimension +(inferred from what the step checks), and after interrogation you can obtain per-dimension scores +and an overall health score. Nothing about how checks *run* changes; this is a labeling and +aggregation layer over results you already have. + +## The Six Dimensions + +Each validation step belongs to one of six data quality dimensions: + +| Dimension | What it measures | Example methods | +|---|---|---| +| **Completeness** | Presence of required values | `col_vals_not_null()`, `col_pct_null()`, `rows_complete()` | +| **Validity** | Values conform to rules, ranges, formats, or schema | `col_vals_gt()`, `col_vals_regex()`, `col_vals_in_set()`, `col_schema_match()` | +| **Uniqueness** | Absence of duplicate rows | `rows_distinct()` | +| **Consistency** | Internal agreement across columns, rows, or tables | `conjointly()`, `col_missing_consistent()`, `tbl_match()` | +| **Timeliness** | Data recency / freshness | `data_freshness()` | +| **Volume** | Expected row and column counts | `row_count_match()`, `col_count_match()` | + +The dimension for each step is inferred from its assertion type, so existing validations gain +dimensions with no changes on your part. + +## Dimensions in the Validation Report + +The dimension display in the validation report is **opt-in**: pass `incl_dimensions=True` to +[`get_tabular_report()`](`pointblank.Validate.get_tabular_report`) (or set it globally with +`pb.config(report_incl_dimensions=True)`). Consider a validation of some sales data that touches +several dimensions: + +```{python} +import pointblank as pb +import polars as pl + +sales = pl.DataFrame({ + "order_id": [1, 2, 3, 4, 5, 6, 7, 8, 8, 10], # one duplicate (8) + "amount": [120.0, -5.0, 47.5, 0.0, 30.0, 155.0, 175.0, 95.0, 205.0, None], + "email": ["a@ex.com", "invalid", "c@ex.com", "d@ex.com", "nope", + "f@ex.com", "g@ex.com", "h@ex.com", "i@ex.com", "j@ex.com"], + "status": ["paid", "paid", "refund", "paid", "pending", + "paid", "paid", "refund", "paid", "paid"], +}) + +validation = ( + pb.Validate(data=sales, tbl_name="sales", label="Sales data quality") + .col_vals_not_null(columns="amount") # Completeness + .col_vals_gt(columns="amount", value=0) # Validity + .col_vals_regex(columns="email", pattern=r"^[^@]+@[^@]+\.[^@]+$") # Validity + .col_vals_in_set(columns="status", set=["paid", "refund", "pending"]) # Validity + .rows_distinct(columns_subset=["order_id"]) # Uniqueness + .row_count_match(count=10) # Volume + .interrogate() +) + +validation.get_tabular_report(incl_dimensions=True) +``` + +Two things in the report now relate to dimensions: + +- **A dimension badge on each step number.** Each step's number carries a small, color-coded + two-letter badge in its top-left corner (`CM` completeness, `VA` validity, `UQ` uniqueness, + `CS` consistency, `TM` timeliness, `VO` volume). Hover over a badge to see the full dimension + name. The badge is compact by design, so it doesn't widen the report. +- **A health-score summary in the footer.** Below the table you'll find the overall **Health + Score** followed by a per-dimension breakdown, with each dimension's color reinforcing the badges + above. + +The scores themselves (below) are always available programmatically, whether or not the display is +enabled. + +## Overriding a Step's Dimension + +Automatic inference covers the common cases, but you can set a dimension explicitly with the +`dimension=` parameter on any validation method. This is useful for multi-faceted checks. For +example, treating a particular range check as a *consistency* rule rather than plain validity: + +```{python} +validation_override = ( + pb.Validate(data=sales, tbl_name="sales") + .col_vals_gt(columns="amount", value=0, dimension="consistency") + .interrogate() +) + +validation_override.get_dimension_scores() +``` + +You can also remap dimensions globally (for every step of a given type) with +[`config()`](`pointblank.config`): + +```{python} +pb.config(dimension_map={"col_vals_gt": "consistency"}) + +# Now `col_vals_gt` steps are categorized as "consistency" everywhere +remapped = ( + pb.Validate(data=sales) + .col_vals_gt(columns="amount", value=0) + .interrogate() +) +print(remapped.validation_info[0].dimension) + +pb.config() # reset back to defaults +``` + +An explicit `dimension=` on a step always takes precedence over the global map. + +## Health Scores + +After interrogation, two methods surface the scores: + +```{python} +validation.get_dimension_scores() +``` + +```{python} +validation.get_health_score() +``` + +Scores are **test-unit weighted**: a dimension's score is the total number of passing test units +divided by the total number of test units across its steps, expressed as a percentage. The overall +health score is the same calculation across every step. Because it's weighted by test units (not by +step count), the score reflects data volume. A failing check over a large table pulls the score down +more than one over a small table. + +::: {.callout-note} +Only steps that produced a pass/fail result contribute to scoring. Steps that haven't been +interrogated, inactive steps (`active=False`), and steps that could not be evaluated (for example, +a check that references a nonexistent column) are all excluded, so a broken check doesn't distort +the score. +::: + +### Weighting Dimensions + +Some organizations consider certain dimensions more critical than others. Provide per-dimension +weights with `config(dimension_weights=...)` to scale each dimension's contribution to the overall +score (a dimension not listed keeps a weight of `1.0`): + +```{python} +pb.config(dimension_weights={"completeness": 3.0}) + +validation_weighted = ( + pb.Validate(data=sales) + .col_vals_not_null(columns="amount") # Completeness, weighted 3x + .col_vals_gt(columns="amount", value=0) # Validity + .interrogate() +) +print(validation_weighted.get_health_score()) + +pb.config() # reset back to defaults +``` + +## The Scorecard + +For a compact, standalone summary (well suited to dashboards or an executive overview) use +[`get_scorecard()`](`pointblank.Validate.get_scorecard`). It shows the overall health score +prominently along with a per-dimension breakdown (a color-coded bar, the score, and passing/total +test units): + +```{python} +validation.get_scorecard() +``` + +The scorecard is a [Great Tables](https://github.com/posit-dev/great-tables) object, so you can +display it directly, export it to HTML with `.as_raw_html()`, or save it to an image file with +`.gtsave()`. + +## Enforcing Minimum Scores + +In automated pipelines and CI you often want to *fail the run* when a dimension slips below an +acceptable level. Call [`assert_dimension_scores()`](`pointblank.Validate.assert_dimension_scores`) +with per-dimension minimums; it raises an `AssertionError` if any dimension is below its minimum +(here, uniqueness is `80%`): + +```{python} +try: + validation.assert_dimension_scores(thresholds={"uniqueness": 95}) +except AssertionError as e: + print(e) +``` + +Dimensions present in the thresholds but absent from the validation are ignored, and a call where +every dimension meets its minimum simply returns without raising. You can also set the minimums +globally with `config(dimension_thresholds=...)`, and then call the method without arguments: + +```{python} +pb.config(dimension_thresholds={"completeness": 95}) + +try: + validation.assert_dimension_scores() +except AssertionError as e: + print(e) + +pb.config() # reset back to defaults +``` + +## Accessing Scores Programmatically + +The scores are also included in the summary available to +[`FinalActions`](`pointblank.FinalActions`) via +[`get_validation_summary()`](`pointblank.get_validation_summary`), under the `dimension_scores` and +`overall_health_score` keys. This makes it easy to log or trend the health score over time: + +```{python} +def log_health(): + summary = pb.get_validation_summary() + print(f"Overall health score: {summary['overall_health_score']}") + print(f"By dimension: {summary['dimension_scores']}") + +( + pb.Validate(data=sales, final_actions=pb.FinalActions(log_health)) + .col_vals_not_null(columns="amount") + .col_vals_gt(columns="amount", value=0) + .interrogate() +) +``` + +## Localized Dimensions + +Like the rest of the validation report, dimension labels and the health-score summary are +localized. When you set a reporting language (e.g., `Validate(..., lang="fr")`), the dimension +names in badge tooltips, the footer summary, and the scorecard are translated automatically. The +two-letter badge codes stay language-neutral (matching the report's other short codes such as +`TBL`, `EVAL`, and `W`/`E`/`C`), with the full localized name available on hover.