diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 33eae64..d5ea713 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.1.0 +current_version = 2.2.0 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P(rc|dev))(?P\d+))? diff --git a/focus_validator/config_objects/focus_to_duckdb_converter.py b/focus_validator/config_objects/focus_to_duckdb_converter.py index 231f0c6..069db76 100644 --- a/focus_validator/config_objects/focus_to_duckdb_converter.py +++ b/focus_validator/config_objects/focus_to_duckdb_converter.py @@ -548,7 +548,9 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = f"{col} IS NOT NULL AND typeof({col}) = 'VARCHAR'" + predicate_sql = self._apply_condition( + f"{col} IS NOT NULL AND typeof({col}) = 'VARCHAR'" + ) return SQLQuery( requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql @@ -558,6 +560,42 @@ def getCheckType(self) -> str: return "type_string" +class TypeJSONCheckGenerator(DuckDBCheckGenerator): + REQUIRED_KEYS = {"ColumnName"} + + def generateSql(self) -> SQLQuery: + col = self.params.ColumnName + keyword = self._get_validation_keyword() + message = self.errorMessage or f"{col} {keyword} be of type JSON." + msg_sql = message.replace("'", "''") + + condition = f"{col} IS NOT NULL AND NOT json_valid(CAST({col} AS VARCHAR))" + condition = self._apply_condition(condition) + + requirement_sql = f""" + WITH invalid AS ( + SELECT 1 + FROM {{table_name}} + WHERE {condition} + ) + SELECT + COUNT(*) AS violations, + CASE WHEN COUNT(*) > 0 THEN '{msg_sql}' END AS error_message + FROM invalid + """ + + predicate_sql = self._apply_condition( + f"{col} IS NOT NULL AND typeof({col}) = 'JSON'" + ) + + return SQLQuery( + requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql + ) + + def getCheckType(self) -> str: + return "type_json" + + class TypeDecimalCheckGenerator(DuckDBCheckGenerator): REQUIRED_KEYS = {"ColumnName"} @@ -590,7 +628,7 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = ( + predicate_sql = self._apply_condition( f"{col} IS NOT NULL AND typeof({col}) IN ('DECIMAL', 'DOUBLE', 'FLOAT')" ) @@ -639,7 +677,7 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = ( + predicate_sql = self._apply_condition( f"{col} IS NOT NULL " f"AND (typeof({col}) IN ('TIMESTAMP', 'TIMESTAMP_NS', 'TIMESTAMP WITH TIME ZONE', 'DATE') " f"OR ({col}::TEXT ~ '^[0-9]{{4}}-[0-1][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z$'))" @@ -684,7 +722,9 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = f"{col} IS NOT NULL AND (TRIM({col}::TEXT) ~ '^[+-]?([0-9]*[.])?[0-9]+([eE][+-]?[0-9]+)?$')" + predicate_sql = self._apply_condition( + f"{col} IS NOT NULL AND (TRIM({col}::TEXT) ~ '^[+-]?([0-9]*[.])?[0-9]+([eE][+-]?[0-9]+)?$')" + ) return SQLQuery( requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql @@ -741,7 +781,9 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = f"{col} IS NOT NULL AND ({col}::TEXT ~ '^[\\x00-\\x7F]*$')" + predicate_sql = self._apply_condition( + f"{col} IS NOT NULL AND ({col}::TEXT ~ '^[\\x00-\\x7F]*$')" + ) return SQLQuery( requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql @@ -806,7 +848,7 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = ( + predicate_sql = self._apply_condition( f"{col} IS NOT NULL " f"AND (typeof({col}) IN ('TIMESTAMP', 'TIMESTAMP_NS', 'TIMESTAMP WITH TIME ZONE', 'DATE') " f"OR (typeof({col}) = 'VARCHAR' AND {col}::TEXT ~ '^[0-9]{{4}}-[0-1][0-9]-[0-3][0-9]T[0-2][0-9]:[0-5][0-9]:[0-5][0-9]Z?$' " @@ -857,7 +899,9 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = f"{col} IS NOT NULL AND TRIM({col}::TEXT) IN ('{codes_list}')" + predicate_sql = self._apply_condition( + f"{col} IS NOT NULL AND TRIM({col}::TEXT) IN ('{codes_list}')" + ) return SQLQuery( requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql @@ -897,7 +941,9 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = f"{col} IS NOT NULL AND (TRIM({col}::TEXT) ~ '^[A-Z]{{3}}$')" + predicate_sql = self._apply_condition( + f"{col} IS NOT NULL AND (TRIM({col}::TEXT) ~ '^[A-Z]{{3}}$')" + ) return SQLQuery( requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql @@ -1035,7 +1081,7 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = ( + predicate_sql = self._apply_condition( f"{col} IS NOT NULL AND regexp_matches({col}, '{combined_pattern}')" ) @@ -1081,16 +1127,10 @@ def generateSql(self) -> SQLQuery: message = self.errorMessage or f"{col} {keyword} be valid JSON format" msg_sql = message.replace("'", "''") - # Requirement SQL (finds violations) - # Check if column is not null and either: - # 1. Cannot be cast to JSON, or - # 2. Is not a valid JSON string when treated as text - condition = ( - f"{col} IS NOT NULL " - f"AND (TRY_CAST({col} AS JSON) IS NULL " - f"OR (typeof({col}) = 'VARCHAR' AND NOT json_valid({col}::TEXT)))" + invalid_predicate = ( + f"{col} IS NOT NULL AND NOT json_valid(CAST({col} AS VARCHAR))" ) - condition = self._apply_condition(condition) + condition = self._apply_condition(invalid_predicate) requirement_sql = f""" WITH invalid AS ( @@ -1104,11 +1144,8 @@ def generateSql(self) -> SQLQuery: FROM invalid """ - # Predicate SQL (for condition mode) - predicate_sql = ( - f"{col} IS NOT NULL " - f"AND (TRY_CAST({col} AS JSON) IS NOT NULL " - f"OR (typeof({col}) = 'VARCHAR' AND json_valid({col}::TEXT)))" + predicate_sql = self._apply_condition( + f"{col} IS NOT NULL AND json_valid(CAST({col} AS VARCHAR))" ) return SQLQuery( @@ -1119,6 +1156,178 @@ def getCheckType(self) -> str: return "format_json" +class CheckJSONSchemaGenerator(DuckDBCheckGenerator): + REQUIRED_KEYS = {"ColumnName", "SchemaId"} + DEFAULTS = {"Path": "$"} + + def getCheckType(self) -> str: + return "json_schema" + + def generateSql(self) -> SQLQuery: + col = self.params.ColumnName + schema_id = self.params.SchemaId + keyword = self._get_validation_keyword() + self.errorMessage = ( + self.errorMessage or f"{col} {keyword} conform to JSON Schema '{schema_id}'" + ) + return SQLQuery(requirement_sql="SELECT 0 AS violations") + + def _extract_path_value(self, payload: Any, path: str) -> Any: + """Extract a value from a JSON payload using a limited JSONPath subset. + + Supported: '$', '$.key', '$.key.nested', '$.key[0]'. + Not supported: chained indices ('$.foo[0][1]'), bracket-key access + ('$["foo bar"]'), wildcards, or filters. + """ + if path == "$": + return payload + + if not path.startswith("$."): + raise InvalidRuleException( + f"Unsupported JSON path '{path}' for CheckJSONSchema in rule {self.rule_id}" + ) + + current = payload + for segment in path[2:].split("."): + if current is None: + return None + + token = segment + while token: + array_match = re.match( + r"^([A-Za-z_][A-Za-z0-9_]*)(\[(\d+)\])?(.*)$", token + ) + if not array_match: + raise InvalidRuleException( + f"Unsupported JSON path segment '{segment}' for CheckJSONSchema in rule {self.rule_id}" + ) + + key_name, _, array_idx, remainder = array_match.groups() + if not isinstance(current, dict): + return None + current = current.get(key_name) + + if array_idx is not None: + if not isinstance(current, list): + return None + idx = int(array_idx) + if idx >= len(current): + return None + current = current[idx] + + token = remainder or "" + + return current + + def generateCheck(self) -> DuckDBColumnCheck: + chk = super().generateCheck() + + schema_map = getattr(self.params, "schemas", None) or {} + schema_id = self.params.SchemaId + schema_entry = schema_map.get(schema_id) + + if not isinstance(schema_entry, dict) or "Schema" not in schema_entry: + raise InvalidRuleException( + f"SchemaId '{schema_id}' referenced by rule {self.rule_id} was not found in model Schemas" + ) + + schema = schema_entry["Schema"] + path = getattr(self.params, "Path", "$") + col = self.params.ColumnName + where_clauses = [f"{col} IS NOT NULL"] + row_condition = (self.row_condition_sql or "").strip() + if row_condition: + where_clauses.append(f"({row_condition})") + + query = f"SELECT {col} FROM {{table_name}} WHERE " + " AND ".join(where_clauses) + + def _exec_json_schema(conn): + try: + from jsonschema import ( # type: ignore[import-untyped] + Draft202012Validator, + ) + except ModuleNotFoundError as exc: + raise RuntimeError( + "CheckJSONSchema requires the 'jsonschema' package to be installed" + ) from exc + + Draft202012Validator.check_schema(schema) + validator = Draft202012Validator(schema) + table_name = getattr(self.params, "table_name", "focus_data") + sql = query.replace("{table_name}", table_name) + sql = sql.replace("{table_name}", table_name) + try: + rows = conn.execute(sql).fetchall() + except (duckdb.BinderException, duckdb.CatalogException) as exc: + msg = str(exc) + missing = [] + patterns = [ + r'Column with name ([A-Za-z0-9_"]+) does not exist', + r'Referenced column "([A-Za-z0-9_]+)" not found', + r'Binder Error: .*? column ([A-Za-z0-9_"]+)', + r'"([A-Za-z0-9_]+)" not found', + ] + for pattern in patterns: + for match in re.finditer(pattern, msg): + col_name = match.group(1).strip('"') + if col_name and col_name not in missing: + missing.append(col_name) + + missing_msg = ( + f"Missing columns: {', '.join(missing)}" + if missing + else "Missing required column(s)" + ) + return False, { + "violations": 1, + "schema_id": schema_id, + "message": f"{self.errorMessage}. {missing_msg}", + "failure_reason": missing_msg, + "error_type": "missing_columns", + } + + failure_messages: list[str] = [] + violations = 0 + for row_num, row in enumerate(rows, start=1): + raw_value = row[0] if isinstance(row, (tuple, list)) else row + try: + payload = ( + json.loads(raw_value) + if isinstance(raw_value, str) + else raw_value + ) + except Exception as exc: + violations += 1 + failure_messages.append(f"row {row_num}: invalid JSON ({exc})") + continue + + instance = self._extract_path_value(payload, path) + errors = sorted( + validator.iter_errors(instance), key=lambda err: list(err.path) + ) + if errors: + violations += 1 + failure_messages.append(f"row {row_num}: {errors[0].message}") + + ok = violations == 0 + details = { + "violations": violations, + "schema_id": schema_id, + "message": ( + self.errorMessage + if ok + else f"{self.errorMessage}. First error: {failure_messages[0]}" + ), + } + if failure_messages: + details["failure_messages"] = failure_messages[:5] + return ok, details + + chk.special_executor = _exec_json_schema + chk.meta["special_executor_kind"] = "json_schema" + return chk + + class CheckValueGenerator(DuckDBCheckGenerator): REQUIRED_KEYS = {"ColumnName", "Value"} @@ -1157,7 +1366,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate), ) def get_sample_sql(self) -> str: @@ -1246,7 +1456,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate), ) def get_sample_sql(self) -> str: @@ -1286,6 +1497,118 @@ def generatePredicate(self) -> str | None: return sql_query.get_predicate_sql() +class CheckRegexMatchGenerator(DuckDBCheckGenerator): + REQUIRED_KEYS = {"ColumnName", "Pattern"} + + def generateSql(self) -> SQLQuery: + col = self.params.ColumnName + pattern = self.params.Pattern + keyword = self._get_validation_keyword() + pattern_sql = str(pattern).replace("'", "''") + message = self.errorMessage or f"{col} {keyword} match regex '{pattern}'." + msg_sql = message.replace("'", "''") + + condition = f"{col} IS NOT NULL AND NOT regexp_matches(CAST({col} AS VARCHAR), '{pattern_sql}')" + condition = self._apply_condition(condition) + + requirement_sql = f""" + WITH invalid AS ( + SELECT 1 + FROM {{table_name}} + WHERE {condition} + ) + SELECT + COUNT(*) AS violations, + CASE WHEN COUNT(*) > 0 THEN '{msg_sql}' END AS error_message + FROM invalid + """ + + predicate_sql = self._apply_condition( + f"{col} IS NOT NULL AND regexp_matches(CAST({col} AS VARCHAR), '{pattern_sql}')" + ) + + return SQLQuery( + requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql + ) + + def get_sample_sql(self) -> str: + col = self.params.ColumnName + pattern = self.params.Pattern + pattern_sql = str(pattern).replace("'", "''") + + condition = f"{col} IS NOT NULL AND NOT regexp_matches(CAST({col} AS VARCHAR), '{pattern_sql}')" + condition = self._apply_condition(condition) + + return f""" + SELECT {col} + FROM {{table_name}} + WHERE {condition} + """ + + @property + def sample_sql(self) -> str: + return self.get_sample_sql() + + def getCheckType(self) -> str: + return "check_regex_match" + + +class CheckStringEndsWithGenerator(DuckDBCheckGenerator): + REQUIRED_KEYS = {"ColumnName", "Value"} + + def generateSql(self) -> SQLQuery: + col = self.params.ColumnName + value = self.params.Value + keyword = self._get_validation_keyword() + value_sql = str(value).replace("'", "''") + message = self.errorMessage or f"{col} {keyword} end with '{value}'." + msg_sql = message.replace("'", "''") + + condition = f"{col} IS NOT NULL AND NOT ends_with(CAST({col} AS VARCHAR), '{value_sql}')" + condition = self._apply_condition(condition) + + requirement_sql = f""" + WITH invalid AS ( + SELECT 1 + FROM {{table_name}} + WHERE {condition} + ) + SELECT + COUNT(*) AS violations, + CASE WHEN COUNT(*) > 0 THEN '{msg_sql}' END AS error_message + FROM invalid + """ + + predicate_sql = self._apply_condition( + f"{col} IS NOT NULL AND ends_with(CAST({col} AS VARCHAR), '{value_sql}')" + ) + + return SQLQuery( + requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql + ) + + def get_sample_sql(self) -> str: + col = self.params.ColumnName + value = self.params.Value + value_sql = str(value).replace("'", "''") + + condition = f"{col} IS NOT NULL AND NOT ends_with(CAST({col} AS VARCHAR), '{value_sql}')" + condition = self._apply_condition(condition) + + return f""" + SELECT {col} + FROM {{table_name}} + WHERE {condition} + """ + + @property + def sample_sql(self) -> str: + return self.get_sample_sql() + + def getCheckType(self) -> str: + return "check_string_ends_with" + + class CheckSameValueGenerator(DuckDBCheckGenerator): REQUIRED_KEYS = {"ColumnAName", "ColumnBName"} @@ -1317,7 +1640,7 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = ( + predicate_sql = self._apply_condition( f"{col_a} IS NOT NULL AND {col_b} IS NOT NULL AND {col_a} = {col_b}" ) @@ -1395,7 +1718,7 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = ( + predicate_sql = self._apply_condition( f"{col_a} IS NOT NULL AND {col_b} IS NOT NULL AND {col_a} <> {col_b}" ) @@ -1469,7 +1792,9 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = f"{a} IS NOT NULL AND {b} IS NOT NULL AND {r} IS NOT NULL AND ({a} * {b}) = {r}" + predicate_sql = self._apply_condition( + f"{a} IS NOT NULL AND {b} IS NOT NULL AND {r} IS NOT NULL AND ({a} * {b}) = {r}" + ) return SQLQuery( requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql @@ -1486,21 +1811,38 @@ def generatePredicate(self) -> str | None: return sql_query.get_predicate_sql() -class CheckGreaterOrEqualGenerator(DuckDBCheckGenerator): +class _CheckScalarComparisonGenerator(DuckDBCheckGenerator): + """Base for single-column scalar comparison checks (>=, >, <=, ...). + + Subclasses differ only by operator and wording, so they set: + - PASS_OPERATOR: operator a valid value satisfies (e.g. ">=") + - VIOLATION_OPERATOR: its negation, used to find violating rows (e.g. "<") + - MESSAGE_PHRASE: human-readable phrase (e.g. "greater than or equal to") + - CHECK_TYPE: value returned by getCheckType() + """ + REQUIRED_KEYS = {"ColumnName", "Value"} + PASS_OPERATOR: ClassVar[str] + VIOLATION_OPERATOR: ClassVar[str] + MESSAGE_PHRASE: ClassVar[str] + CHECK_TYPE: ClassVar[str] + + def _violation_condition(self) -> str: + col = self.params.ColumnName + val = self.params.Value + return f"{col} IS NOT NULL AND {col} {self.VIOLATION_OPERATOR} {self._lit(val)}" def generateSql(self) -> SQLQuery: col = self.params.ColumnName val = self.params.Value keyword = self._get_validation_keyword() message = ( - self.errorMessage or f"{col} {keyword} be greater than or equal to {val}." + self.errorMessage or f"{col} {keyword} be {self.MESSAGE_PHRASE} {val}." ) msg_sql = message.replace("'", "''") # Requirement SQL (finds violations) - condition = f"{col} IS NOT NULL AND {col} < {val}" - condition = self._apply_condition(condition) + condition = self._apply_condition(self._violation_condition()) requirement_sql = f""" WITH invalid AS ( @@ -1515,7 +1857,9 @@ def generateSql(self) -> SQLQuery: """ # Predicate SQL (for condition mode) - predicate_sql = f"{col} IS NOT NULL AND {col} >= {self._lit(val)}" + predicate_sql = self._apply_condition( + f"{col} IS NOT NULL AND {col} {self.PASS_OPERATOR} {self._lit(val)}" + ) return SQLQuery( requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql @@ -1524,11 +1868,7 @@ def generateSql(self) -> SQLQuery: def get_sample_sql(self) -> str: """Return SQL to fetch sample violating rows for display""" col = self.params.ColumnName - val = self.params.Value - - # Build condition to find violating rows (values less than the required minimum) - condition = f"{col} IS NOT NULL AND {col} < {val}" - condition = self._apply_condition(condition) + condition = self._apply_condition(self._violation_condition()) return f""" SELECT {col} @@ -1542,7 +1882,7 @@ def sample_sql(self) -> str: return self.get_sample_sql() def getCheckType(self) -> str: - return "check_greater_equal" + return self.CHECK_TYPE def generatePredicate(self) -> str | None: """Backward compatibility wrapper""" @@ -1552,6 +1892,88 @@ def generatePredicate(self) -> str | None: return sql_query.get_predicate_sql() +class CheckGreaterOrEqualGenerator(_CheckScalarComparisonGenerator): + PASS_OPERATOR = ">=" + VIOLATION_OPERATOR = "<" + MESSAGE_PHRASE = "greater than or equal to" + CHECK_TYPE = "check_greater_equal" + + +class CheckGreaterThanGenerator(_CheckScalarComparisonGenerator): + PASS_OPERATOR = ">" + VIOLATION_OPERATOR = "<=" + MESSAGE_PHRASE = "greater than" + CHECK_TYPE = "check_greater_than" + + +class CheckLessOrEqualGenerator(_CheckScalarComparisonGenerator): + PASS_OPERATOR = "<=" + VIOLATION_OPERATOR = ">" + MESSAGE_PHRASE = "less than or equal to" + CHECK_TYPE = "check_less_or_equal" + + +class CheckColumnComparisonGenerator(DuckDBCheckGenerator): + REQUIRED_KEYS = {"ColumnAName", "ColumnBName", "Comparator"} + + _VALID_COMPARATORS: ClassVar[Set[str]] = {"=", "!=", "<>", ">", ">=", "<", "<="} + + def generateSql(self) -> SQLQuery: + col_a = self.params.ColumnAName + col_b = self.params.ColumnBName + comparator = self.params.Comparator + keyword = self._get_validation_keyword() + + if comparator not in self._VALID_COMPARATORS: + raise InvalidRuleException( + f"Unsupported comparator for {self.rule_id}: {comparator}" + ) + + message = self.errorMessage or f"{col_a} {keyword} be {comparator} {col_b}." + msg_sql = message.replace("'", "''") + + pass_predicate = f"{col_a} IS NOT NULL AND {col_b} IS NOT NULL AND {col_a} {comparator} {col_b}" + condition = f"{col_a} IS NOT NULL AND {col_b} IS NOT NULL AND NOT ({col_a} {comparator} {col_b})" + condition = self._apply_condition(condition) + + requirement_sql = f""" + WITH invalid AS ( + SELECT 1 + FROM {{table_name}} + WHERE {condition} + ) + SELECT + COUNT(*) AS violations, + CASE WHEN COUNT(*) > 0 THEN '{msg_sql}' END AS error_message + FROM invalid + """ + + return SQLQuery( + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(pass_predicate), + ) + + def get_sample_sql(self) -> str: + col_a = self.params.ColumnAName + col_b = self.params.ColumnBName + comparator = self.params.Comparator + condition = f"{col_a} IS NOT NULL AND {col_b} IS NOT NULL AND NOT ({col_a} {comparator} {col_b})" + condition = self._apply_condition(condition) + + return f""" + SELECT {col_a}, {col_b} + FROM {{table_name}} + WHERE {condition} + """ + + @property + def sample_sql(self) -> str: + return self.get_sample_sql() + + def getCheckType(self) -> str: + return "check_column_comparison" + + class CheckDistinctCountGenerator(DuckDBCheckGenerator): REQUIRED_KEYS = {"ColumnAName", "ColumnBName", "ExpectedCount"} @@ -1605,6 +2027,66 @@ def getCheckType(self) -> str: return "distinct_count" +class CheckNoDuplicatesGenerator(DuckDBCheckGenerator): + REQUIRED_KEYS = {"ColumnName"} + + def generateSql(self) -> SQLQuery: + col = self.params.ColumnName + keyword = self._get_validation_keyword() + message = self.errorMessage or f"{col} {keyword} contain no duplicate values." + msg_sql = message.replace("'", "''") + + where_clause = f"WHERE {col} IS NOT NULL" + if self.row_condition_sql and self.row_condition_sql.strip(): + where_clause = f"WHERE ({col} IS NOT NULL) AND ({self.row_condition_sql})" + + requirement_sql = f""" + WITH counts AS ( + SELECT {col} AS value, COUNT(*) AS occurrences + FROM {{table_name}} + {where_clause} + GROUP BY {col} + ), + invalid AS ( + SELECT value, occurrences + FROM counts + WHERE occurrences > 1 + ) + SELECT + COUNT(*) AS violations, + CASE WHEN COUNT(*) > 0 THEN '{msg_sql}' END AS error_message + FROM invalid + """ + + return SQLQuery(requirement_sql=requirement_sql.strip(), predicate_sql=None) + + def get_sample_sql(self) -> str: + col = self.params.ColumnName + where_clause = f"WHERE {col} IS NOT NULL" + if self.row_condition_sql and self.row_condition_sql.strip(): + where_clause = f"WHERE ({col} IS NOT NULL) AND ({self.row_condition_sql})" + + return f""" + WITH dupes AS ( + SELECT {col} AS value + FROM {{table_name}} + {where_clause} + GROUP BY {col} + HAVING COUNT(*) > 1 + ) + SELECT t.{col} + FROM {{table_name}} t + JOIN dupes d ON t.{col} = d.value + """ + + @property + def sample_sql(self) -> str: + return self.get_sample_sql() + + def getCheckType(self) -> str: + return "check_no_duplicates" + + class CheckModelRuleGenerator(DuckDBCheckGenerator): REQUIRED_KEYS = {"ModelRuleId"} @@ -1825,7 +2307,8 @@ def type_check_expr(value_expr: str) -> str: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -1947,7 +2430,8 @@ def keyvalue_check(value_expr: str) -> str: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -2080,7 +2564,8 @@ def keys_check(value_expr: str) -> str: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -2213,7 +2698,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -2384,7 +2870,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -2548,7 +3035,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -2843,7 +3331,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -2948,7 +3437,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -3127,7 +3617,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -3225,7 +3716,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -3281,7 +3773,7 @@ def generateSql(self) -> SQLQuery: return SQLQuery( requirement_sql=requirement_sql.strip(), - predicate_sql=predicate_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) # Path provided - validate elements at that path @@ -3360,7 +3852,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -3465,7 +3958,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -3652,7 +4146,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -3764,7 +4259,8 @@ def generateSql(self) -> SQLQuery: """ return SQLQuery( - requirement_sql=requirement_sql.strip(), predicate_sql=predicate_sql.strip() + requirement_sql=requirement_sql.strip(), + predicate_sql=self._apply_condition(predicate_sql.strip()), ) def getCheckType(self) -> str: @@ -4343,6 +4839,10 @@ class FocusToDuckDBSchemaConverter: "generator": TypeStringCheckGenerator, "factory": lambda args: "ColumnName", }, + "TypeJSON": { + "generator": TypeJSONCheckGenerator, + "factory": lambda args: "ColumnName", + }, "TypeDecimal": { "generator": TypeDecimalCheckGenerator, "factory": lambda args: "ColumnName", @@ -4367,6 +4867,10 @@ class FocusToDuckDBSchemaConverter: "generator": FormatBillingCurrencyCodeGenerator, "factory": lambda args: "ColumnName", }, + "FormatJSON": { + "generator": FormatJSONGenerator, + "factory": lambda args: "ColumnName", + }, "FormatKeyValue": { "generator": FormatJSONGenerator, "factory": lambda args: "ColumnName", @@ -4375,10 +4879,18 @@ class FocusToDuckDBSchemaConverter: "generator": FormatCurrencyGenerator, "factory": lambda args: "ColumnName", }, + "CheckColumnComparison": { + "generator": CheckColumnComparisonGenerator, + "factory": lambda args: "ColumnAName", + }, "CheckNationalCurrency": { "generator": FormatBillingCurrencyCodeGenerator, "factory": lambda args: "ColumnName", }, + "CheckGreaterThanValue": { + "generator": CheckGreaterThanGenerator, + "factory": lambda args: "ColumnName", + }, "FormatUnit": { "generator": FormatUnitGenerator, "factory": lambda args: "ColumnName", @@ -4387,10 +4899,26 @@ class FocusToDuckDBSchemaConverter: "generator": CheckValueGenerator, "factory": lambda args: "ColumnName", }, + "CheckLessOrEqualThanValue": { + "generator": CheckLessOrEqualGenerator, + "factory": lambda args: "ColumnName", + }, "CheckNotValue": { "generator": CheckNotValueGenerator, "factory": lambda args: "ColumnName", }, + "CheckNoDuplicates": { + "generator": CheckNoDuplicatesGenerator, + "factory": lambda args: "ColumnName", + }, + "CheckRegexMatch": { + "generator": CheckRegexMatchGenerator, + "factory": lambda args: "ColumnName", + }, + "CheckStringEndsWith": { + "generator": CheckStringEndsWithGenerator, + "factory": lambda args: "ColumnName", + }, "CheckSameValue": { "generator": CheckSameValueGenerator, "factory": lambda args: "ColumnAName", @@ -4415,6 +4943,10 @@ class FocusToDuckDBSchemaConverter: "generator": CheckModelRuleGenerator, "factory": lambda args: "ModelRuleId", }, + "CheckJSONSchema": { + "generator": CheckJSONSchemaGenerator, + "factory": lambda args: "ColumnName", + }, "AND": { "generator": CompositeANDRuleGenerator, "factory": lambda args: "Items", @@ -4595,6 +5127,7 @@ def __init__( transpile_dialect: Optional[str] = None, show_violations: bool = False, rules_version: Optional[str] = None, + schemas: Optional[Dict[str, Any]] = None, ) -> None: self.log = logging.getLogger(f"{__name__}.{self.__class__.__qualname__}") self.conn: duckdb.DuckDBPyConnection | None = None @@ -4608,6 +5141,7 @@ def __init__( ) self.show_violations = show_violations self.rules_version = rules_version + self.schemas = schemas or {} # Build the effective CHECK_GENERATORS mapping for this version self.CHECK_GENERATORS = self._build_check_generators_for_version(rules_version) @@ -5304,7 +5838,7 @@ def __make_generator__( gen_cls = reg["generator"] # Strip reserved + 'CheckFunction' and pass as-is (no aliasing) - reserved = getattr(DuckDBCheckGenerator, "RESERVED", set()) or set() + reserved: set = getattr(DuckDBCheckGenerator, "RESERVED", set()) or set() params = { k: v for k, v in requirement.items() @@ -5374,6 +5908,8 @@ def __make_generator__( rule_id=rule_id, plan=self.plan, conn=self.conn, + schemas=self.schemas, + table_name=self.table_name, parent_results_by_idx=parent_results_by_idx or {}, parent_edges=parent_edges or (), row_condition_sql=row_condition_sql, @@ -5683,7 +6219,7 @@ def _compile_condition_with_generators( gen_cls = reg["generator"] # Basic required-key validation (optional) - required = getattr(gen_cls, "REQUIRED_KEYS", set()) or set() + required: set = getattr(gen_cls, "REQUIRED_KEYS", set()) or set() missing = [k for k in required if k not in spec] if missing: # For conditions, you can choose to return None or raise @@ -5911,15 +6447,20 @@ def _explain_check_sql(self, check) -> dict: # Conformance reference / special executor (no SQL) special = getattr(check, "special_executor", None) if callable(special): + special_kind = meta.get("special_executor_kind") return { "rule_id": rid, - "type": "reference", + "type": "special", "check_type": ctype, "generator": meta.get("generator"), "row_condition_sql": meta.get("row_condition_sql"), "referenced": getattr(check, "referenced_rule_id", None), "sql": None, # executed by reference, not SQL - "note": "mirrors referenced rule outcome (no SQL)", + "note": ( + "mirrors referenced rule outcome (no SQL)" + if special_kind == "reference" + else "executed via special executor (no SQL)" + ), "must_satisfy": must_satisfy, } diff --git a/focus_validator/data_loaders/parquet_data_loader.py b/focus_validator/data_loaders/parquet_data_loader.py index 22e59cb..9d0eb8f 100644 --- a/focus_validator/data_loaders/parquet_data_loader.py +++ b/focus_validator/data_loaders/parquet_data_loader.py @@ -58,14 +58,23 @@ def _smart_datetime_conversion( # Try multiple datetime parsing strategies converted = None + # A strategy succeeds only if parsing introduces no nulls + # beyond those already present (so nullable columns convert), + # and the column has at least one real value (an all-null + # column carries no evidence of being a datetime, so it is + # left as a string rather than coerced). + original_null_count = series.null_count() + has_values = original_null_count < series.len() + # Strategy 1: Try ISO format with timezone try: candidate = series.str.to_datetime( format="%Y-%m-%dT%H:%M:%S%z", # ISO with timezone like -05:00 strict=False, ) - # Check if conversion was successful (all values converted) - if candidate.null_count() == 0: + # Accept if parsing added no new nulls (nullable columns) + # and at least one value actually parsed + if has_values and candidate.null_count() == original_null_count: converted = candidate except Exception: pass @@ -77,8 +86,12 @@ def _smart_datetime_conversion( format="%Y-%m-%dT%H:%M:%SZ", # ISO with Z timezone strict=False, ) - # Check if conversion was successful (all values converted) - if candidate.null_count() == 0: + # Accept if parsing added no new nulls (nullable + # columns) and at least one value actually parsed + if ( + has_values + and candidate.null_count() == original_null_count + ): converted = candidate except Exception: pass @@ -90,8 +103,12 @@ def _smart_datetime_conversion( format="%Y-%m-%d %H:%M:%S", # Space-separated format strict=False, ) - # Check if conversion was successful (all values converted) - if candidate.null_count() == 0: + # Accept if parsing added no new nulls (nullable + # columns) and at least one value actually parsed + if ( + has_values + and candidate.null_count() == original_null_count + ): converted = candidate except Exception: pass @@ -102,8 +119,12 @@ def _smart_datetime_conversion( candidate = series.str.to_datetime( format="%Y-%m-%d", strict=False # Simple date format ) - # Check if conversion was successful (all values converted) - if candidate.null_count() == 0: + # Accept if parsing added no new nulls (nullable + # columns) and at least one value actually parsed + if ( + has_values + and candidate.null_count() == original_null_count + ): converted = candidate except Exception: pass @@ -147,14 +168,21 @@ def _smart_datetime_conversion( series.name, converted_values, dtype=pl.Datetime("us") ) - # Check if we successfully converted all values - if candidate.null_count() == 0: + # Accept if parsing added no new nulls (nullable + # columns) and at least one value actually parsed + if ( + has_values + and candidate.null_count() == original_null_count + ): converted = candidate except Exception: pass - # Strategy 6: Let Polars infer format (for fallback cases) + # Strategy 6: Let Polars infer the format (fallback for any + # single format strategies 1-5 did not match). Format inference + # cannot parse timezone-qualified ISO (trailing 'Z' or offsets), + # but strategies 1-2 already cover those. if converted is None: try: candidate = series.str.to_datetime( @@ -163,8 +191,12 @@ def _smart_datetime_conversion( exact=False, # Allow partial matches cache=True, # Cache format inference ) - # For auto-inference, allow some nulls but require most values to convert - if candidate.null_count() < len(candidate): + # Accept if parsing added no new nulls (nullable columns) + # and at least one value actually parsed + if ( + has_values + and candidate.null_count() == original_null_count + ): converted = candidate except Exception: pass diff --git a/focus_validator/rules/spec_rules.py b/focus_validator/rules/spec_rules.py index 3822105..d44a347 100644 --- a/focus_validator/rules/spec_rules.py +++ b/focus_validator/rules/spec_rules.py @@ -442,6 +442,7 @@ def load_rules(self) -> ValidationPlan: self.plan = val_plan self.column_types = column_types + self.model_data = model_data self._meta = { "json_rule_file": self.json_rule_file, "focus_dataset": self.focus_dataset, @@ -482,6 +483,7 @@ def validate( transpile_dialect=self.transpile_dialect, show_violations=show_violations, rules_version=self.rules_version, + schemas=getattr(self, "model_data", {}).get("Schemas", {}), ) # 1) Let the converter prepare schemas, UDFs, temp views, etc. if connection is None: @@ -620,6 +622,7 @@ def explain(self) -> Dict[str, Dict[str, Any]]: transpile_dialect=self.transpile_dialect, show_violations=False, # Not relevant for explain mode rules_version=self.rules_version, + schemas=getattr(self, "model_data", {}).get("Schemas", {}), ) # Create a minimal connection for explain mode (converter needs it for initialization) diff --git a/pyproject.toml b/pyproject.toml index 2490811..03bf649 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "focus_validator" -version = "2.1.0" +version = "2.2.0" description = "FOCUS spec validator." authors = [] readme = "README.md" @@ -26,6 +26,7 @@ requests = "*" pandera = { version = "^0.26.1" } multimethod = ">=2.0,<2.1" sqlglot = "^27.28.1" +jsonschema = "^4.25.1" numpy = { version = "^1.26"} pytz = "^2025.2" pandasql = "^0.7.3" diff --git a/tests/config_objects/test_focus_to_duckdb_generators.py b/tests/config_objects/test_focus_to_duckdb_generators.py index 1526d89..ae2be50 100644 --- a/tests/config_objects/test_focus_to_duckdb_generators.py +++ b/tests/config_objects/test_focus_to_duckdb_generators.py @@ -22,6 +22,7 @@ # Type generators TypeDecimalCheckGenerator, TypeStringCheckGenerator, + TypeJSONCheckGenerator, TypeDateTimeGenerator, # Format generators @@ -31,13 +32,21 @@ FormatBillingCurrencyCodeGenerator, FormatJSONGenerator, FormatCurrencyGenerator, + CheckJSONSchemaGenerator, + FocusToDuckDBSchemaConverter, # Value check generators CheckValueGenerator, CheckNotValueGenerator, + CheckRegexMatchGenerator, + CheckStringEndsWithGenerator, CheckSameValueGenerator, CheckNotSameValueGenerator, CheckGreaterOrEqualGenerator, + CheckGreaterThanGenerator, + CheckLessOrEqualGenerator, + CheckColumnComparisonGenerator, + CheckNoDuplicatesGenerator, # Column comparison generators ColumnByColumnEqualsColumnValueGenerator, @@ -223,6 +232,36 @@ def test_type_string_check_type(self): self.assertEqual(check_type, "type_string") +class TestTypeJSONGenerator(unittest.TestCase): + """Test TypeJSON SQL generation.""" + + def setUp(self): + """Set up TypeJSON generator.""" + mock_rule = Mock(spec=ModelRule) + mock_rule.rule_id = "Tags-TYPE-JSON-001" + + self.generator = TypeJSONCheckGenerator( + rule=mock_rule, + rule_id="Tags-TYPE-JSON-001", + ColumnName="Tags" + ) + + def test_type_json_sql_generation(self): + """Test SQL generation for TypeJSON check.""" + sql_result = self.generator.generateSql() + sql = _extract_sql(sql_result) + + self.assertIn("WITH invalid AS", sql) + self.assertIn("Tags IS NOT NULL", sql) + self.assertIn("NOT json_valid(CAST(Tags AS VARCHAR))", sql) + self.assertIn("Tags MUST be of type JSON", sql) + + def test_type_json_check_type(self): + """Test check type identification.""" + check_type = self.generator.getCheckType() + self.assertEqual(check_type, "type_json") + + class TestCheckValueGenerator(unittest.TestCase): """Test CheckValue SQL generation for exact value matching.""" @@ -293,6 +332,86 @@ def test_check_value_sql_injection_prevention(self): self.assertIn("O''Reilly", sql) +class TestCheckRegexMatchGenerator(unittest.TestCase): + """Test CheckRegexMatch SQL generation.""" + + def setUp(self): + """Set up CheckRegexMatch generator.""" + self.mock_rule = Mock(spec=ModelRule) + self.mock_rule.rule_id = "TEST-CHECK-REGEX" + + def test_check_regex_match_sql_generation(self): + """Test CheckRegexMatch emits regex validation SQL.""" + generator = CheckRegexMatchGenerator( + rule=self.mock_rule, + rule_id="TEST-CHECK-REGEX", + ColumnName="ContractCommitmentDurationType", + Pattern="^[1-9][0-9]*\\s+(Day|Days)$" + ) + + sql_result = generator.generateSql() + sql = _extract_sql(sql_result) + + self.assertIn("WITH invalid AS", sql) + self.assertIn("ContractCommitmentDurationType IS NOT NULL", sql) + self.assertIn( + "NOT regexp_matches(CAST(ContractCommitmentDurationType AS VARCHAR), '^[1-9][0-9]*\\s+(Day|Days)$')", + sql, + ) + self.assertIn("ContractCommitmentDurationType MUST match regex", sql) + + def test_check_regex_match_check_type(self): + """Test CheckRegexMatch check type identification.""" + generator = CheckRegexMatchGenerator( + rule=self.mock_rule, + rule_id="TEST-CHECK-REGEX", + ColumnName="ContractCommitmentDurationType", + Pattern="^[1-9][0-9]*\\s+(Day|Days)$" + ) + + self.assertEqual(generator.getCheckType(), "check_regex_match") + + +class TestCheckStringEndsWithGenerator(unittest.TestCase): + """Test CheckStringEndsWith SQL generation.""" + + def setUp(self): + """Set up CheckStringEndsWith generator.""" + self.mock_rule = Mock(spec=ModelRule) + self.mock_rule.rule_id = "TEST-CHECK-ENDSWITH" + + def test_check_string_endswith_sql_generation(self): + """Test CheckStringEndsWith emits suffix validation SQL.""" + generator = CheckStringEndsWithGenerator( + rule=self.mock_rule, + rule_id="TEST-CHECK-ENDSWITH", + ColumnName="ContractCommitmentDurationType", + Value="Years" + ) + + sql_result = generator.generateSql() + sql = _extract_sql(sql_result) + + self.assertIn("WITH invalid AS", sql) + self.assertIn("ContractCommitmentDurationType IS NOT NULL", sql) + self.assertIn( + "NOT ends_with(CAST(ContractCommitmentDurationType AS VARCHAR), 'Years')", + sql, + ) + self.assertIn("ContractCommitmentDurationType MUST end with ''Years''", sql) + + def test_check_string_endswith_check_type(self): + """Test CheckStringEndsWith check type identification.""" + generator = CheckStringEndsWithGenerator( + rule=self.mock_rule, + rule_id="TEST-CHECK-ENDSWITH", + ColumnName="ContractCommitmentDurationType", + Value="Years" + ) + + self.assertEqual(generator.getCheckType(), "check_string_ends_with") + + class TestFormatGenerators(unittest.TestCase): """Test various format validation generators.""" @@ -350,6 +469,200 @@ def test_format_currency_code_generator(self): self.assertIn("BillingCurrency IS NOT NULL", sql) # Should validate against ISO 4217 currency codes + def test_format_json_generator_returns_sql_query(self): + """Test FormatJSON SQL generation and predicate support.""" + mock_rule = Mock(spec=ModelRule) + mock_rule.rule_id = "FORMAT-JSON-001" + + generator = FormatJSONGenerator( + rule=mock_rule, + rule_id="FORMAT-JSON-001", + ColumnName="Tags", + ) + + sql_result = generator.generateSql() + + self.assertIsInstance(sql_result, SQLQuery) + self.assertIn("WITH invalid AS", sql_result.get_requirement_sql()) + self.assertIn("Tags IS NOT NULL", sql_result.get_requirement_sql()) + self.assertIn("json_valid(CAST(Tags AS VARCHAR))", sql_result.get_requirement_sql()) + self.assertIn("json_valid(CAST(Tags AS VARCHAR))", sql_result.get_predicate_sql()) + + def test_format_json_generator_applies_row_condition_to_predicate(self): + """Test FormatJSON predicate SQL includes row-level conditions.""" + mock_rule = Mock(spec=ModelRule) + mock_rule.rule_id = "FORMAT-JSON-002" + + generator = FormatJSONGenerator( + rule=mock_rule, + rule_id="FORMAT-JSON-002", + ColumnName="Tags", + row_condition_sql="ProviderName = 'AWS'", + ) + + predicate = generator.generatePredicate() + + self.assertIsNotNone(predicate) + self.assertIn("ProviderName = 'AWS'", predicate) + + def test_leaf_generators_apply_row_condition_to_predicate(self): + """All leaf generators bake the row condition into their predicate SQL. + + FormatJSON established this pattern; it is now uniform across leaf + generators so a check used in condition mode only matches rows within + its declared scope. See the matching FormatJSON test above. + """ + mock_rule = Mock(spec=ModelRule) + mock_rule.rule_id = "ROW-COND-PREDICATE" + row_condition = "ProviderName = 'AWS'" + + generators = [ + CheckSameValueGenerator( + rule=mock_rule, + rule_id="ROW-COND-PREDICATE", + ColumnAName="ColA", + ColumnBName="ColB", + exec_mode="condition", + row_condition_sql=row_condition, + ), + CheckGreaterOrEqualGenerator( + rule=mock_rule, + rule_id="ROW-COND-PREDICATE", + ColumnName="Amount", + Value=100, + exec_mode="condition", + row_condition_sql=row_condition, + ), + ColumnByColumnEqualsColumnValueGenerator( + rule=mock_rule, + rule_id="ROW-COND-PREDICATE", + ColumnAName="EffectiveCost", + ColumnBName="BilledCost", + ResultColumnName="ExpectedCost", + exec_mode="condition", + row_condition_sql=row_condition, + ), + ] + + for generator in generators: + predicate = generator.generatePredicate() + self.assertIsNotNone( + predicate, + f"{type(generator).__name__} should produce a predicate", + ) + self.assertIn( + row_condition, + predicate, + f"{type(generator).__name__} should apply the row condition " + f"to its predicate", + ) + + def test_check_json_schema_generator_validates_rows(self): + """Test CheckJSONSchema validates JSON values against model schemas.""" + mock_rule = Mock(spec=ModelRule) + mock_rule.rule_id = "CHECK-JSON-SCHEMA-001" + + generator = CheckJSONSchemaGenerator( + rule=mock_rule, + rule_id="CHECK-JSON-SCHEMA-001", + ColumnName="Payload", + Path="$", + SchemaId="TEST-SCHEMA", + schemas={ + "TEST-SCHEMA": { + "Schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["foo"], + "properties": {"foo": {"type": "string"}}, + "additionalProperties": False, + } + } + }, + row_condition_sql="ProviderName = 'AWS'", + ) + + check = generator.generateCheck() + + class FakeResult: + def __init__(self, rows): + self._rows = rows + + def fetchall(self): + return self._rows + + class FakeConn: + def __init__(self, rows): + self.rows = rows + self.last_sql = None + + def execute(self, sql): + self.last_sql = sql + return FakeResult(self.rows) + + fake_conn = FakeConn([ + ('{"foo": "ok"}',), + ('{"foo": 1}',), + ('not json',), + ]) + + ok, details = check.special_executor(fake_conn) + + self.assertFalse(ok) + self.assertEqual(details["violations"], 2) + self.assertEqual(details["schema_id"], "TEST-SCHEMA") + self.assertIn("ProviderName = 'AWS'", fake_conn.last_sql) + self.assertIn("row 2", details["failure_messages"][0]) + + def test_converter_build_check_threads_schemas_to_check_json_schema(self): + """Test converter build path passes Schemas data into CheckJSONSchema.""" + validation_criteria = ValidationCriteria( + MustSatisfy="Payload MUST conform to schema.", + Keyword="MUST", + Requirement={ + "CheckFunction": "CheckJSONSchema", + "ColumnName": "Payload", + "Path": "$", + "SchemaId": "TEST-SCHEMA", + }, + Condition={}, + Dependencies=[], + ) + + mock_rule = Mock(spec=ModelRule) + mock_rule.rule_id = "CHECK-JSON-SCHEMA-002" + mock_rule.validation_criteria = validation_criteria + mock_rule.is_dynamic.return_value = False + mock_rule.is_optional.return_value = False + + converter = FocusToDuckDBSchemaConverter( + focus_data=None, + explain_mode=True, + schemas={ + "TEST-SCHEMA": { + "Schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["foo"], + "properties": {"foo": {"type": "string"}}, + } + } + }, + ) + converter.conn = MagicMock() + converter.plan = SimpleNamespace(nodes=[]) + + check = converter.build_check( + rule=mock_rule, + parent_results_by_idx={}, + parent_edges=(), + rule_id="CHECK-JSON-SCHEMA-002", + node_idx=0, + ) + + self.assertEqual(check.checkType, "json_schema") + self.assertTrue(callable(check.special_executor)) + class TestComparisonGenerators(unittest.TestCase): """Test comparison and relational check generators.""" @@ -372,6 +685,45 @@ def test_check_greater_or_equal_generator(self): self.assertIn("WITH invalid AS", sql) self.assertIn("UsageQuantity < 0", sql) self.assertIn("UsageQuantity MUST be greater than or equal to 0", sql) + + def test_check_greater_than_generator(self): + """Test CheckGreaterThan SQL generation.""" + mock_rule = Mock(spec=ModelRule) + mock_rule.rule_id = "CHECK-GT-001" + + generator = CheckGreaterThanGenerator( + rule=mock_rule, + rule_id="CHECK-GT-001", + ColumnName="PaymentCurrencyBilledCost", + Value=0, + ) + + sql_result = generator.generateSql() + sql = _extract_sql(sql_result) + + self.assertIn("PaymentCurrencyBilledCost <= 0", sql) + self.assertIn("PaymentCurrencyBilledCost MUST be greater than 0", sql) + + def test_check_less_or_equal_generator(self): + """Test CheckLessOrEqual SQL generation.""" + mock_rule = Mock(spec=ModelRule) + mock_rule.rule_id = "CHECK-LTE-001" + + generator = CheckLessOrEqualGenerator( + rule=mock_rule, + rule_id="CHECK-LTE-001", + ColumnName="ContractCommitmentDiscountPercentage", + Value=1.0, + ) + + sql_result = generator.generateSql() + sql = _extract_sql(sql_result) + + self.assertIn("ContractCommitmentDiscountPercentage > 1.0", sql) + self.assertIn( + "ContractCommitmentDiscountPercentage MUST be less than or equal to 1.0", + sql, + ) def test_check_not_value_generator(self): """Test CheckNotValue SQL generation.""" @@ -434,6 +786,25 @@ def test_column_comparison_generator(self): sql = _extract_sql(sql_result) self.assertIn("(EffectiveCost * BilledCost)", sql) + + def test_check_column_comparison_generator(self): + """Test CheckColumnComparison SQL generation.""" + mock_rule = Mock(spec=ModelRule) + mock_rule.rule_id = "COLUMN-COMPARE-002" + + generator = CheckColumnComparisonGenerator( + rule=mock_rule, + rule_id="COLUMN-COMPARE-002", + ColumnAName="BillingPeriodLastUpdated", + ColumnBName="BillingPeriodCreated", + Comparator=">=", + ) + + sql_result = generator.generateSql() + sql = _extract_sql(sql_result) + + self.assertIn("BillingPeriodLastUpdated >= BillingPeriodCreated", sql) + self.assertIn("BillingPeriodLastUpdated MUST be >= BillingPeriodCreated", sql) class TestAdvancedGenerators(unittest.TestCase): @@ -459,6 +830,24 @@ def test_check_distinct_count_generator(self): self.assertIn("COUNT(DISTINCT BillingAccountName)", sql) self.assertIn("<> 1", sql) + def test_check_no_duplicates_generator(self): + """Test CheckNoDuplicates SQL generation.""" + mock_rule = Mock(spec=ModelRule) + mock_rule.rule_id = "NO-DUPES-001" + + generator = CheckNoDuplicatesGenerator( + rule=mock_rule, + rule_id="NO-DUPES-001", + ColumnName="ContractCommitmentId", + ) + + sql_result = generator.generateSql() + sql = _extract_sql(sql_result) + + self.assertIn("GROUP BY ContractCommitmentId", sql) + self.assertIn("WHERE occurrences > 1", sql) + self.assertIn("ContractCommitmentId MUST contain no duplicate values", sql) + class TestSQLGenerationPatterns(unittest.TestCase): """Test common SQL generation patterns and utilities.""" @@ -484,8 +873,14 @@ def test_sql_template_structure(self): generators = [ TypeDecimalCheckGenerator(rule=mock_rule, rule_id="TEMPLATE-TEST", ColumnName="TestColumn"), - TypeStringCheckGenerator(rule=mock_rule, rule_id="TEMPLATE-TEST", ColumnName="TestColumn"), + TypeStringCheckGenerator(rule=mock_rule, rule_id="TEMPLATE-TEST", ColumnName="TestColumn"), + TypeJSONCheckGenerator(rule=mock_rule, rule_id="TEMPLATE-TEST", ColumnName="TestColumn"), CheckValueGenerator(rule=mock_rule, rule_id="TEMPLATE-TEST", ColumnName="TestColumn", Value="TestValue"), + CheckRegexMatchGenerator(rule=mock_rule, rule_id="TEMPLATE-TEST", ColumnName="TestColumn", Pattern="^ok$"), + CheckStringEndsWithGenerator(rule=mock_rule, rule_id="TEMPLATE-TEST", ColumnName="TestColumn", Value="ok"), + CheckGreaterThanGenerator(rule=mock_rule, rule_id="TEMPLATE-TEST", ColumnName="TestColumn", Value=0), + CheckLessOrEqualGenerator(rule=mock_rule, rule_id="TEMPLATE-TEST", ColumnName="TestColumn", Value=1), + CheckColumnComparisonGenerator(rule=mock_rule, rule_id="TEMPLATE-TEST", ColumnAName="TestColumn", ColumnBName="OtherColumn", Comparator=">="), FormatNumericGenerator(rule=mock_rule, rule_id="TEMPLATE-TEST", ColumnName="TestColumn") ] diff --git a/tests/config_objects/test_generator_edge_cases.py b/tests/config_objects/test_generator_edge_cases.py new file mode 100644 index 0000000..955724e --- /dev/null +++ b/tests/config_objects/test_generator_edge_cases.py @@ -0,0 +1,241 @@ +"""Edge-case coverage for DuckDB check generators. + +Unlike ``test_focus_to_duckdb_generators.py`` (which mocks ``duckdb`` and +asserts on generated SQL strings), this module executes the generated SQL +against a real in-memory DuckDB so the behavioural edge cases the happy-path +tests miss are actually exercised: + + * ``CheckColumnComparison`` with one or both columns NULL (null handling) + * ``CheckStringEndsWith`` with multi-byte suffixes + * ``CheckJSONSchema`` with malformed paths and with ``jsonschema`` missing + * ``CheckNoDuplicates`` combined with a row-filter condition +""" + +import builtins +import sys +import unittest +from unittest.mock import Mock, patch + +# A sibling test module (test_focus_to_duckdb_generators.py) replaces ``duckdb`` +# in sys.modules with a MagicMock at import time. These tests execute SQL, so we +# need the real engine: drop any installed mock and import the genuine package. +# Binding it to this module's global means later sys.modules changes can't affect +# us, regardless of test collection order. +sys.modules.pop("duckdb", None) +import duckdb # noqa: E402 + +from focus_validator.config_objects.focus_to_duckdb_converter import ( # noqa: E402 + CheckColumnComparisonGenerator, + CheckJSONSchemaGenerator, + CheckNoDuplicatesGenerator, + CheckStringEndsWithGenerator, +) +from focus_validator.config_objects.rule import ModelRule # noqa: E402 +from focus_validator.exceptions import InvalidRuleException # noqa: E402 + + +def _rule(rule_id="EDGE-CASE"): + """A ModelRule mock. + + ``Mock(spec=ModelRule)`` does not expose pydantic fields, so + ``_get_validation_keyword`` falls back to its "MUST" default. That keeps the + generated error-message SQL literal free of Mock reprs, which matters + because we execute the SQL here. + """ + rule = Mock(spec=ModelRule) + rule.rule_id = rule_id + return rule + + +def _violations(conn, generator, table="focus_data"): + """Execute a generator's requirement SQL and return the violation count.""" + requirement_sql = generator.generateSql().requirement_sql + sql = requirement_sql.replace("{table_name}", table) + return conn.execute(sql).fetchone()[0] + + +class TestCheckColumnComparisonNullHandling(unittest.TestCase): + """CheckColumnComparison must skip rows where either column is NULL.""" + + def test_null_rows_are_not_flagged(self): + conn = duckdb.connect() + conn.execute("CREATE TABLE focus_data (EffectiveCost DOUBLE, BilledCost DOUBLE)") + conn.executemany( + "INSERT INTO focus_data VALUES (?, ?)", + [ + (10.0, 10.0), # equal, both present -> passes + (10.0, 20.0), # not equal, both present -> the only violation + (None, 10.0), # A null -> skipped + (10.0, None), # B null -> skipped + (None, None), # both null -> skipped + ], + ) + + generator = CheckColumnComparisonGenerator( + rule=_rule("CMP-001"), + rule_id="CMP-001", + ColumnAName="EffectiveCost", + ColumnBName="BilledCost", + Comparator="=", + ) + + # Only the (10, 20) row violates. With the old + # ``NOT (A IS NOT NULL AND B IS NOT NULL AND A = B)`` form the three + # NULL rows would also be flagged (4 violations). + self.assertEqual(_violations(conn, generator), 1) + + def test_null_handling_matches_check_same_value_convention(self): + """Both generators skip NULL rows, so they agree on the same data.""" + conn = duckdb.connect() + conn.execute("CREATE TABLE focus_data (ColA VARCHAR, ColB VARCHAR)") + conn.executemany( + "INSERT INTO focus_data VALUES (?, ?)", + [("x", "x"), ("x", "y"), (None, "y"), ("x", None), (None, None)], + ) + + comparison = CheckColumnComparisonGenerator( + rule=_rule(), + rule_id="CMP-002", + ColumnAName="ColA", + ColumnBName="ColB", + Comparator="=", + ) + self.assertEqual(_violations(conn, comparison), 1) + + +class TestCheckStringEndsWithMultiByte(unittest.TestCase): + """ends_with must handle multi-byte (non-ASCII) suffixes correctly.""" + + def setUp(self): + self.conn = duckdb.connect() + self.conn.execute("CREATE TABLE focus_data (Name VARCHAR)") + self.conn.executemany( + "INSERT INTO focus_data VALUES (?)", + [("café",), ("nope",), ("日本語",), (None,)], + ) + + def test_accented_suffix(self): + generator = CheckStringEndsWithGenerator( + rule=_rule("EW-1"), rule_id="EW-1", ColumnName="Name", Value="é" + ) + # "café" ends with "é"; "nope" and "日本語" do not; NULL is skipped. + self.assertEqual(_violations(self.conn, generator), 2) + + def test_cjk_suffix(self): + generator = CheckStringEndsWithGenerator( + rule=_rule("EW-2"), rule_id="EW-2", ColumnName="Name", Value="語" + ) + # "日本語" ends with "語"; "café" and "nope" do not; NULL is skipped. + self.assertEqual(_violations(self.conn, generator), 2) + + def test_multibyte_suffix_appears_in_sql(self): + generator = CheckStringEndsWithGenerator( + rule=_rule("EW-3"), rule_id="EW-3", ColumnName="Name", Value="café" + ) + sql = generator.generateSql().requirement_sql + self.assertIn("NOT ends_with(CAST(Name AS VARCHAR), 'café')", sql) + + +class TestCheckJSONSchemaEdgeCases(unittest.TestCase): + """Malformed JSON paths and a missing jsonschema dependency.""" + + def _generator(self): + return CheckJSONSchemaGenerator( + rule=_rule("JS-1"), + rule_id="JS-1", + ColumnName="Payload", + SchemaId="TEST-SCHEMA", + Path="$", + schemas={"TEST-SCHEMA": {"Schema": {"type": "object"}}}, + ) + + def test_path_without_dollar_prefix_raises(self): + generator = self._generator() + with self.assertRaises(InvalidRuleException): + generator._extract_path_value({"a": 1}, "no-dollar") + + def test_malformed_segment_raises(self): + generator = self._generator() + # A segment must start with a letter or underscore. + with self.assertRaises(InvalidRuleException): + generator._extract_path_value({"1bad": 1}, "$.1bad") + + def test_valid_paths_still_resolve(self): + generator = self._generator() + payload = {"a": {"b": 5}, "items": [{"x": 1}, {"x": 2}]} + self.assertEqual(generator._extract_path_value(payload, "$"), payload) + self.assertEqual(generator._extract_path_value(payload, "$.a.b"), 5) + self.assertEqual(generator._extract_path_value(payload, "$.items[1]"), {"x": 2}) + # Missing keys resolve to None rather than raising. + self.assertIsNone(generator._extract_path_value(payload, "$.a.missing")) + + def test_missing_jsonschema_dependency_raises_runtime_error(self): + generator = self._generator() + check = generator.generateCheck() + + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == "jsonschema": + raise ModuleNotFoundError("No module named 'jsonschema'") + return real_import(name, *args, **kwargs) + + conn = duckdb.connect() + with patch("builtins.__import__", side_effect=fake_import): + with self.assertRaises(RuntimeError) as ctx: + check.special_executor(conn) + + self.assertIn("jsonschema", str(ctx.exception)) + + +class TestCheckNoDuplicatesWithRowFilter(unittest.TestCase): + """CheckNoDuplicates must apply the row-filter before counting duplicates.""" + + def setUp(self): + self.conn = duckdb.connect() + self.conn.execute( + "CREATE TABLE focus_data (ResourceId VARCHAR, ProviderName VARCHAR)" + ) + self.conn.executemany( + "INSERT INTO focus_data VALUES (?, ?)", + [ + ("r1", "AWS"), + ("r1", "AWS"), # duplicate within the AWS filter + ("r2", "Azure"), + ("r2", "Azure"), # duplicate, but excluded by the filter + (None, "AWS"), # NULL never counts + ], + ) + + def test_duplicates_outside_filter_are_ignored(self): + generator = CheckNoDuplicatesGenerator( + rule=_rule("DUP-1"), + rule_id="DUP-1", + ColumnName="ResourceId", + row_condition_sql="ProviderName = 'AWS'", + ) + # Only the duplicated AWS "r1" counts; the Azure "r2" pair is filtered out. + self.assertEqual(_violations(self.conn, generator), 1) + + def test_without_filter_all_duplicates_counted(self): + generator = CheckNoDuplicatesGenerator( + rule=_rule("DUP-2"), + rule_id="DUP-2", + ColumnName="ResourceId", + ) + # Both "r1" and "r2" are duplicated when no filter is applied. + self.assertEqual(_violations(self.conn, generator), 2) + + def test_row_filter_appears_in_sql(self): + generator = CheckNoDuplicatesGenerator( + rule=_rule("DUP-3"), + rule_id="DUP-3", + ColumnName="ResourceId", + row_condition_sql="ProviderName = 'AWS'", + ) + sql = generator.generateSql().requirement_sql + self.assertIn("(ResourceId IS NOT NULL) AND (ProviderName = 'AWS')", sql) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/data_loaders/test_timezone_handling.py b/tests/data_loaders/test_timezone_handling.py index 129041a..40b8bce 100644 --- a/tests/data_loaders/test_timezone_handling.py +++ b/tests/data_loaders/test_timezone_handling.py @@ -215,6 +215,62 @@ def test_parquet_single_timezone_converted_to_utc(self): finally: safe_delete_file(temp_path) + def test_parquet_nullable_datetime_columns_are_converted(self): + """Nullable datetime columns convert instead of being dropped to string. + + Regression: every parse strategy used to require a zero null_count, so a + single null cell forced the column to stay a string. The strategies now + accept a result that introduces no new nulls beyond those already present. + """ + cases = { + "IsoZ": ["2023-01-01T10:00:00Z", None, "2023-01-03T08:45:00Z"], + "IsoOffset": ["2023-01-01T10:00:00-05:00", None, "2023-01-03T08:45:00-05:00"], + "DateOnly": ["2023-01-01", None, "2023-01-03"], + "SpaceSep": ["2023-01-01 10:00:00", None, "2023-01-03 08:45:00"], + } + for name, values in cases.items(): + with self.subTest(column=name): + df = pl.DataFrame({name: values, "Value": [1, 2, 3]}) + fd, temp_path = tempfile.mkstemp(suffix=".parquet") + try: + os.close(fd) + df.write_parquet(temp_path) + + column_types = {name: pl.Datetime("us", "UTC")} + loader = ParquetDataLoader(temp_path, column_types=column_types) + result_df = loader.load() + + self.assertIsNotNone(result_df) + self.assertNotIn(name, loader.failed_columns) + self.assertIsInstance(result_df[name].dtype, pl.Datetime) + self.assertEqual(result_df[name].dtype.time_zone, "UTC") + # The originally-null cell is preserved as null, not dropped. + self.assertEqual(result_df[name].null_count(), 1) + finally: + safe_delete_file(temp_path) + + def test_parquet_mixed_format_column_is_dropped(self): + """A genuinely mixed/unparseable column is still rejected (left as string).""" + df = pl.DataFrame( + {"BadDate": ["2023-01-01", "not-a-date", "01/31/2023"], "Value": [1, 2, 3]} + ) + fd, temp_path = tempfile.mkstemp(suffix=".parquet") + try: + os.close(fd) + df.write_parquet(temp_path) + + column_types = {"BadDate": pl.Datetime("us", "UTC")} + loader = ParquetDataLoader(temp_path, column_types=column_types) + result_df = loader.load() + + self.assertIsNotNone(result_df) + # Parsing introduces new nulls, so no strategy accepts it and the + # column is dropped rather than coerced. + self.assertIn("BadDate", loader.failed_columns) + self.assertNotIn("BadDate", result_df.columns) + finally: + safe_delete_file(temp_path) + def test_parquet_no_timezone_defaults_to_utc(self): """Test that Parquet columns without timezone default to UTC.""" # Create test DataFrame without timezone using Polars