diff --git a/.Rbuildignore b/.Rbuildignore deleted file mode 100644 index 4dac18c..0000000 --- a/.Rbuildignore +++ /dev/null @@ -1,8 +0,0 @@ -^renv$ -^renv\.lock$ -^a4d\.Rproj$ -^\.Rproj\.user$ -^LICENSE\.md$ -^\.github$ -^rsconnect$ -^data-raw$ diff --git a/.Rprofile b/.Rprofile deleted file mode 100644 index b3b3274..0000000 --- a/.Rprofile +++ /dev/null @@ -1,7 +0,0 @@ -source("renv/activate.R") - -if (interactive()) { - require("devtools", quietly = TRUE) - # automatically attaches usethis - devtools::load_all() -} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2044218 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.github +.pytest_cache +.ruff_cache +htmlcov +.coverage +profiling/*.prof +data/ +secrets/ diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f3165aa --- /dev/null +++ b/.env.example @@ -0,0 +1,26 @@ +# Environment Configuration +A4D_ENVIRONMENT=development + +# GCP Configuration +A4D_PROJECT_ID=a4dphase2 +A4D_DATASET=tracker +A4D_DOWNLOAD_BUCKET=a4dphase2_upload +A4D_UPLOAD_BUCKET=a4dphase2_output + +# GCP Authentication +# Use the a4d-pipeline service account key for local development — this gives identical +# auth to Cloud Run and is required for Drive API access (user ADC blocks Drive scope). +# Store the key in secrets/ (gitignored). In Cloud Run, the SA authenticates via metadata server. +# GOOGLE_APPLICATION_CREDENTIALS=/path/to/a4d-pipeline-key.json + +# Paths +A4D_DATA_ROOT=/path/to/tracker/files +A4D_OUTPUT_DIR=output + +# Processing Settings +A4D_MAX_WORKERS=4 + +# Error Values (matching R pipeline) +A4D_ERROR_VAL_NUMERIC=999999 +A4D_ERROR_VAL_CHARACTER=Undefined +A4D_ERROR_VAL_DATE=9999-12-31 diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml new file mode 100644 index 0000000..bb4c9bb --- /dev/null +++ b/.github/workflows/python-ci.yml @@ -0,0 +1,50 @@ +name: Python CI + +# No `paths:` filter on purpose: ruff (`check .` / `format --check .`) runs +# repo-wide, so a change under scripts/, docs/ or the justfile can turn CI red +# while never triggering it. The whole job takes ~30s. +on: + push: + branches: [migration, dev] + pull_request: + branches: [main, dev, migration] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v2 + with: + enable-cache: true + + - name: Set up Python + run: uv python install 3.14 + + - name: Install dependencies + run: uv sync --all-extras + + - name: Run ruff linting + run: uv run ruff check . + + - name: Run ruff formatting check + run: uv run ruff format --check . + + - name: Run type checking with ty + run: uv run ty check src/ + + - name: Run tests + run: uv run pytest -m "not slow and not integration" --cov --cov-report=xml + + - name: Enforce coverage floor for product pipeline code + run: | + uv run coverage report --include="src/a4d/extract/product.py,src/a4d/clean/product.py,src/a4d/clean/schema_product.py,src/a4d/extract/wide_format.py,src/a4d/pipeline/product.py,src/a4d/tables/product.py,src/a4d/validate/source_vs_output_product.py" --fail-under=85 + + - name: Upload coverage + uses: codecov/codecov-action@v3 + with: + files: ./coverage.xml + flags: python diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml deleted file mode 100644 index 68381d5..0000000 --- a/.github/workflows/test-coverage.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples -# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help -on: - push: - branches: [main, master, develop] - pull_request: - branches: [main, master, develop] - workflow_dispatch: - -name: test-coverage - -jobs: - test-coverage: - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up R 4.2.3 - uses: r-lib/actions/setup-r@v2 - with: - r-version: 4.2.3 - use-public-rspm: true - - - uses: r-lib/actions/setup-r-dependencies@v2 - with: - extra-packages: | - any::covr - needs: coverage - - - name: Test coverage - run: | - covr::codecov( - quiet = FALSE, - clean = FALSE, - install_path = file.path(Sys.getenv("RUNNER_TEMP"), "package") - ) - shell: Rscript {0} - - - name: Show testthat output - if: always() - run: | - ## -------------------------------------------------------------------- - find ${{ runner.temp }}/package -name 'testthat.Rout*' -exec cat '{}' \; || true - shell: bash - - - name: Upload test results - if: failure() - uses: actions/upload-artifact@v4 - with: - name: coverage-test-failures - path: ${{ runner.temp }}/package diff --git a/.gitignore b/.gitignore index 0791f1a..c9467a2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,77 @@ -.Rproj.user -.Rhistory -.RData -.Ruserdata -.Rhistory -.Rdata -.httr-oauth +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual environments +.venv/ +venv/ +ENV/ +env/ + +# uv +.uv/ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +.tox/ + +# Type checking +.mypy_cache/ +.dmypy.json +dmypy.json + +# IDEs +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Environment +.env +.env.local + +# Logs +*.log +logs/ + +# Data (sensitive) +data/ +output/ +*.parquet +*.xlsx +!reference_data/ +compare_report*.html + +# OS .DS_Store -.Renviron -rsconnect +Thumbs.db + +# Serena (MCP server state) +.serena/ -data/output -data/mapping_table.csv \ No newline at end of file +# Secrets (GCP service accounts, etc.) +secrets/ +.opencode +CLAUDE.local.md +coverage.xml diff --git a/.lintr b/.lintr deleted file mode 100644 index 6c4a172..0000000 --- a/.lintr +++ /dev/null @@ -1,9 +0,0 @@ -linters: linters_with_defaults( # see vignette("lintr") - line_length_linter = NULL, - object_usage_linter = NULL - ) -encoding: "UTF-8" -error_on_lint: FALSE -exclusions: list( - "tests/testthat" - ) diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..c1fe704 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,29 @@ +{ + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.cwd": "${workspaceFolder}/a4d-python", + "python.testing.pytestArgs": [ + "${workspaceFolder}/a4d-python/tests" + ], + "python.defaultInterpreterPath": "${workspaceFolder}/a4d-python/.venv/bin/python", + "workbench.colorCustomizations": { + "activityBar.activeBackground": "#ab307e", + "activityBar.background": "#ab307e", + "activityBar.foreground": "#e7e7e7", + "activityBar.inactiveForeground": "#e7e7e799", + "activityBarBadge.background": "#25320e", + "activityBarBadge.foreground": "#e7e7e7", + "commandCenter.border": "#e7e7e799", + "sash.hoverBorder": "#ab307e", + "statusBar.background": "#832561", + "statusBar.foreground": "#e7e7e7", + "statusBarItem.hoverBackground": "#ab307e", + "statusBarItem.remoteBackground": "#832561", + "statusBarItem.remoteForeground": "#e7e7e7", + "titleBar.activeBackground": "#832561", + "titleBar.activeForeground": "#e7e7e7", + "titleBar.inactiveBackground": "#83256199", + "titleBar.inactiveForeground": "#e7e7e799" + }, + "peacock.color": "#832561" +} \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..79395d4 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,52 @@ +# CLAUDE.md + +## Python Pipeline + +**Location**: Repo root +**Branch**: `migration` + +Python implementation of the A4D medical tracker data processing pipeline. + +**Key Directories**: +- `src/` - Python package source +- `tests/` - Test suite +- `docs/` - Documentation (see [docs/CLAUDE.md](docs/CLAUDE.md) for detailed guidance) +- `scripts/` - Utility scripts +- `reference_data/` - Shared YAML configs (synonyms, validation rules, provinces) + +**Quick Start**: +```bash +uv sync +uv run pytest +``` + +**Migration Guide**: [docs/archive/MIGRATION_GUIDE.md](docs/archive/MIGRATION_GUIDE.md) + +## The retired R implementation + +The legacy R pipeline used to live at `r-archive/`. It was deleted once the +migration was verified Python-only; nothing in this repo reads it any more. + +Recover it from git when you need to check a claim about R's old behaviour: + +```bash +git show r-archive-removed^:r-archive/R/script2_process_patient_data.R +git checkout r-archive-removed^ -- r-archive # whole tree, into the worktree +``` + +The `r-archive-removed` tag marks the commit that deleted it, so `^` is the +last commit that still contains it. Docstrings citing R files by name are +being removed as part of the documentation overhaul; until that lands, this is +how those citations are checked. + +## Shared Resources + +- `reference_data/synonyms/` - Column name mappings +- `reference_data/data_cleaning.yaml` - Validation rules +- `reference_data/provinces/` - Allowed provinces + +**Do not modify these** without testing the Python pipeline. +- Limit comments to explain why a design was made or give important context; + do not use comments for obvious code, and do not justify a design by what the + retired R pipeline did — say what the code does now and why, with real + examples from the tracker data diff --git a/DESCRIPTION b/DESCRIPTION deleted file mode 100644 index 73a25e5..0000000 --- a/DESCRIPTION +++ /dev/null @@ -1,33 +0,0 @@ -Package: a4d -Title: Process tracker data for the DataCross A4D project -Version: 0.0.0.9000 -Authors@R: - person("Michael", "Aydinbas", , "michael.aydinbas@gmail.com", role = c("aut", "cre")) -Description: Functionality to process, clean and ingest tracker data (excel files). -License: MIT + file LICENSE -Encoding: UTF-8 -Roxygen: list(markdown = TRUE) -RoxygenNote: 7.3.3 -Depends: - ParallelLogger -Imports: - data.table, - digest, - dplyr, - here, - lubridate, - readr, - readxl, - stringr, - styler, - testit, - tidyr, - yaml, - zoo, - arrow, - openxlsx, - glue, - jsonlite -Suggests: - testthat (>= 3.0.0) -Config/testthat/edition: 3 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a6075cf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +FROM python:3.14-slim + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + g++ \ + && rm -rf /var/lib/apt/lists/* + +# Install uv from the official image +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +# Use the system Python from the base image; do not let uv download its own +ENV UV_PYTHON_DOWNLOADS=never + +WORKDIR /app + +# Install dependencies first (without the project) for better layer caching. +# --no-install-project skips the editable install of a4d itself, which requires +# src/ to be present. Dependencies rarely change so this layer stays cached. +COPY pyproject.toml uv.lock README.md ./ +RUN uv sync --frozen --no-dev --no-install-project + +# Copy application code and reference data +COPY src/ src/ +COPY reference_data/ reference_data/ + +# Install the project itself now that src/ exists +RUN uv sync --frozen --no-dev + +# Set environment +ENV PYTHONPATH=/app/src +ENV PYTHONUNBUFFERED=1 +ENV A4D_DATA_ROOT=/workspace/data +ENV A4D_REFERENCE_DATA=/app/reference_data + +# Run the full pipeline: download → process → upload to GCS → ingest into BigQuery +CMD ["uv", "run", "a4d", "run"] diff --git a/NAMESPACE b/NAMESPACE deleted file mode 100644 index 11fda9c..0000000 --- a/NAMESPACE +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by roxygen2: do not edit by hand - -export(calculate_most_frequent) -export(check_patterns_in_column) -export(convert_to) -export(correct_decimal_sign) -export(cut_numeric_value) -export(extract_patient_data) -export(extract_unit_capacity) -export(extract_year_from_age) -export(fix_age) -export(fix_bmi) -export(fix_fbg) -export(fix_id) -export(fix_sex) -export(fix_t1d_diagnosis_age) -export(fix_testing_frequency) -export(get_tracker_year) -export(harmonize_patient_data_columns) -export(process_raw_patient_file) -export(process_raw_product_file) -export(process_tracker_file) -export(process_tracker_patient_data) -export(process_tracker_product_data) -export(read_cleaned_patient_data) -export(read_column_synonyms) -export(reading_patient_data) -export(report_empty_intersections) -export(sanitize_str) -export(setup_logger) -export(with_file_logger) -importFrom(data.table,"%like%") diff --git a/R/a4d-package.R b/R/a4d-package.R deleted file mode 100644 index ad285c8..0000000 --- a/R/a4d-package.R +++ /dev/null @@ -1,7 +0,0 @@ -#' @keywords internal -"_PACKAGE" - -## usethis namespace: start -#' @importFrom data.table %like% -## usethis namespace: end -NULL diff --git a/R/helper_main.R b/R/helper_main.R deleted file mode 100644 index c0d80b8..0000000 --- a/R/helper_main.R +++ /dev/null @@ -1,152 +0,0 @@ -#' @title initialize all necessary paths -#' -#' @description -#' Create necessary output folder for main script under tracker_root_path. -#' This script creates a new output folder next to the data files, and -#' deletes all old files in it. -#' The script asks the user to set the tracker_root_path, either by -#' selecting a folder or reading from the A4D_DATA_ROOT env var. -#' -#' @param names Folder names that will be created under the output folder. -#' @param delete If TRUE, delete all files under output. -#' -#' @return A list with tracker_root_path and output_root path -init_paths <- function(names, delete = FALSE) { - config <- config::get() - paths <- list() - data_dir <- config$data_root - output_dir <- config$output_dir - - paths$tracker_root <- data_dir - paths$output_root <- file.path(data_dir, output_dir) - - for (name in names) { - subdir <- file.path( - data_dir, - output_dir, - name - ) - - if (fs::dir_exists(subdir)) { - if (delete) { - fs::dir_delete(subdir) - } - } - - fs::dir_create(subdir) - - paths[[name]] <- subdir - } - - paths -} - - -#' @title Find all files matching a search pattern in a given directory. -#' -#' @description -#' Searches recursively for files matching a search pattern inside the root dir. -#' Only returns the file names without the paths. -#' -#' -#' @param tracker_root The root directory to search in. -#' @param pattern The search pattern to filter files. -#' -#' @return A vector with file names. -get_files <- function(tracker_root, pattern = "\\.xlsx$") { - tracker_files <- list.files(path = tracker_root, recursive = T, pattern = pattern) - tracker_files <- - tracker_files[stringr::str_detect(tracker_files, "~", negate = T)] -} - - -#' @title Read synonyms from the synonyms YAML files. -#' -#' @description -#' Read in all defined synonyms from the YAML files inside the synonyms folder. -#' -#' @return A list with both patient and product data synonyms as tibble. -get_synonyms <- function() { - ## Extract synonyms for products and patients - ## If you encounter new columns, just add the synonyms to these YAML files - synonyms_patient <- - read_column_synonyms(synonym_file = "synonyms_patient.yaml") - synonyms_product <- - read_column_synonyms(synonym_file = "synonyms_product.yaml") - - list(patient = synonyms_patient, product = synonyms_product) -} - - -#' @title Get all synonyms for all variable names -#' -#' @description -#' This function reads the synonyms from a YAML file -#' and generates a tibble containing unique column names and their synonyms. -#' -#' @param synonym_file A YAML file containing the synonyms -#' @param path_prefixes Path prefixes for searching for the yaml file. Usually -#' this does not need to be set unless for testing purpouses. -#' -#' @return A tibble containing unique column names and their synonyms. -#' @export -#' -#' @examples -#' \dontrun{ -#' read_column_synonyms(synonym_file = "synonyms_patient.yaml") -#' read_column_synonyms(synonym_file = "synonyms_product.yaml") -#' } -read_column_synonyms <- function(synonym_file, path_prefixes = c("reference_data", "synonyms")) { - path <- do.call(file.path, as.list(c(path_prefixes, synonym_file))) - columns_synonyms <- - yaml::read_yaml(path) %>% - unlist() %>% - as.data.frame() %>% - tibble::rownames_to_column() %>% - # remove digits that were created when converting to data frame - dplyr::mutate( - rowname = stringr::str_replace(rowname, pattern = "[:digit:]+$", "") - ) %>% - dplyr::rename( - "variable_name" = "rowname", - "tracker_name" = "." - ) %>% - dplyr::as_tibble() -} - - -#' @title Export data as parquet to a given destination. -#' -#' @param data Data frame to export as parquet file. -#' @param filename Output file name. -#' @param output_root Root output directory. -#' @param suffix Suffix will be appended to the original file name (e.g. "patient_data"). -#' -#' @examples -#' \dontrun{ -#' export_data( -#' data = df_raw_product, -#' filename = tracker_name, -#' output_root = output_root, -#' suffix = "_product_data" -#' ) -#' } -export_data_as_parquet <- function(data, filename, output_root, suffix) { - data %>% - arrow::write_parquet( - sink = file.path(output_root, paste0(filename, suffix, ".parquet")), - ) -} - - -#' @title Read allowed provinces from a YAML file. -#' -#' @description -#' Read in all provinces from a YAML file inside the provinces folder. -#' -#' @return A named character vector with all allowed provinces. -get_allowed_provinces <- function() { - ## Should new countries and provinces be added, update the YAML file - provinces <- yaml::read_yaml("reference_data/provinces/allowed_provinces.yaml") %>% unlist() - return(provinces) -} diff --git a/R/helper_product_data.R b/R/helper_product_data.R deleted file mode 100644 index a77521b..0000000 --- a/R/helper_product_data.R +++ /dev/null @@ -1,601 +0,0 @@ -#### Define functions ##### - -`%notin%` <- Negate(`%in%`) - -#' @title set_second_row_as_headers_and_remove_first_row -#' -#' @description -#' Clean df where first row is empty -set_second_row_as_headers_and_remove_first_row <- function(df) { - colnames(df) <- df[1, ] - df <- df[-1, ] - return(df) -} - -#### 1. Identify xlsx cutoff #### -# ============================================================================== - -# Test whether both helper functions work for all years -# and if word searches need to be done for years separately -# Conditions to cut of at start msd data -helper_is_msd_start_row <- function(df, i) { - condition <- dplyr::case_when( - # 2021 - # any(grepl("MEDICAL", df[i, ])) & - # any(grepl("Product", df[i + 1, ])) & - # any(grepl("Entry Date", df[i + 1, ])) ~ TRUE, - - # 2024 - any(grepl("Product", df[i, ])) & - any(grepl("Date", df[i, ])) & - any(grepl("Received", df[i, ])) ~ TRUE, - - # 2019 & 2020 & 2021 - any(grepl("Product", df[i, ])) & - any(grepl("Date", df[i, ])) & - any(grepl("Units Received|Units \r\nReceived", df[i, ])) ~ TRUE, - - # 2017 & 2018 - any(grepl("Description of Support", df[i, ])) & - any(grepl("Date", df[i, ])) & - any(grepl("Units Received", df[i, ])) ~ TRUE, - TRUE ~ FALSE - ) -} - -# Conditions to cut of at start patient data -helper_is_msd_end_row <- function(df, i) { - includes_patient_name <- any(grepl("Patient Name", df[i + 1, ])) - includes_patient_recruitment <- any(grepl("patient recruitment", tolower(df[i, ])) | - grepl("patient data summary", tolower(df[i - 1, ]))) # second check for 2022 data onwards - - condition <- dplyr::case_when( - !includes_patient_name ~ FALSE, - !includes_patient_recruitment ~ FALSE, - !(any(grepl("patient id", tolower(df[i + 1, ]))) | any(grepl("ID", df[i + 1, ]))) ~ FALSE, - # !any(grepl("No", df[i + 1, ])) ~ FALSE, - # !any(grepl("ID", df[i + 1,])) ~ FALSE, - TRUE ~ TRUE - ) - - return(condition) -} - - -get_msd_start <- function(df, j) { - if (helper_is_msd_start_row(df, j) == TRUE) { - start_df_msd <- j + 1 - return(start_df_msd) - } -} - -get_msd_end <- function(df, j) { - if (helper_is_msd_end_row(df, j)) { - end_df_msd <- j - 1 - return(end_df_msd) - } -} - -get_patient_start <- function(df, j) { - if (helper_is_msd_end_row(df, j)) { - start_df_patient <- j + 1 - } -} - -get_patient_end <- function(df, j) { - if (helper_is_msd_end_row(df, j)) { - end_df_patient <- nrow(df) - } -} - -#' @title extract_product_data -#' -#' @description -#' Reads product data from a monthly file based on extraction logic -extract_product_data <- function(monthly_tracker_df) { - start_df_msd <- NULL - end_df_msd <- NULL - - for (i in 1:nrow(monthly_tracker_df)) { - start <- get_msd_start(monthly_tracker_df, i) - end <- get_msd_end(monthly_tracker_df, i) - - if (!purrr::is_empty(start) & purrr::is_empty(start_df_msd)) { - start_df_msd <- start - 1 - } - if (!purrr::is_empty(end) & purrr::is_empty(end_df_msd)) { - end_df_msd <- end - } - } - - if (is.null(start_df_msd)) { - logWarn( - log_to_json( - message = "Cannot find the initial row for the product data.", - script = "script1", - file = "helper_product_data.R", - functionName = "extract_product_data", - warningCode = "invalid_tracker" - ) - ) - } - - if (is.null(end_df_msd)) { - logWarn( - log_to_json( - message = "Cannot find the final row for the product data.", - script = "script1", - file = "helper_product_data.R", - functionName = "extract_product_data", - warningCode = "invalid_tracker" - ) - ) - } - - product_data_df <- monthly_tracker_df[start_df_msd:end_df_msd, ] - - # Clean empty remaining first row - product_data_df <- set_second_row_as_headers_and_remove_first_row(product_data_df) - - return(product_data_df) -} - -#### 2. Match Product data columns #### -# ============================================================================== - -#' @title harmonize_input_data_columns -#' -#' @description -#' Imports the product df, cleans it and matches it against -# column synonyms to unify column names -# -#' @param product_df: The product data to be cleaned and matched against column synonyms -#' @param columns_synonyms: Long format output of read_column_synonyms to match columns -#' @param sheet: The current sheet name of the tracker file -harmonize_input_data_columns <- function(product_df, columns_synonyms, sheet) { - # In case that there is additional data in strange columns, keep only relevant columns - # keep.cols <- names(product_df) %in% c("") - - product_df <- product_df %>% purrr::discard(~ all(is.na(.) | . == "")) - product_df <- product_df[!is.na(names(product_df))] - - colnames(product_df) <- sanitize_column_name(colnames(product_df)) - synonym_headers <- sanitize_column_name(columns_synonyms$name_to_be_matched) - - ## report all column names which have not been found - unknown_column_names <- colnames(product_df)[!colnames(product_df) %in% synonym_headers] - if (length(unknown_column_names) > 0) { - logWarn( - log_to_json( - message = "Sheet {values['sheet']}: Unknown column names: {values['col_names']}.", - values = list(sheet = sheet, col_names = unknown_column_names), - script = "script1", - file = "helper_product_data.R", - functionName = "harmonize_input_data_columns", - warningCode = "invalid_tracker" - ) - ) - } - - # replacing var codes - colnames_found <- match(colnames(product_df), synonym_headers, nomatch = 0) - colnames(product_df)[colnames(product_df) %in% synonym_headers] <- columns_synonyms$name_clean[colnames_found] - - if (sum(colnames_found == 0) != 0) { - # SK: remove non matching column names - relcols <- which(colnames(product_df) %in% columns_synonyms$name_clean) - product_df <- product_df[, relcols] - } - - return(product_df) -} - -#' @title read_column_synonyms_product -#' -#' @description -#' Imports the codebook, cleans, removes duplicates and transforms it -#' into long df format -read_column_synonyms_product <- function(codebook_data_file) { - columns_synonyms <- codebook_data_file %>% - readxl::read_xlsx(sheet = "synonyms_ProductData") %>% - dplyr::as_tibble() %>% - tidyr::pivot_longer( - cols = dplyr::everything(), - names_to = "name_clean", - values_to = "name_to_be_matched" - ) %>% - # subset(!is.na(name_to_be_matched)) %>% # commented out by SK, as this should also include final column names - # lapply(sanitize_column_name) %>% - dplyr::as_tibble() %>% - dplyr::group_by(name_to_be_matched) %>% - dplyr::slice(1) %>% - dplyr::ungroup() - - return(columns_synonyms) -} - -sanitize_column_name <- function(column_name) { - column_name <- column_name %>% - stringr::str_to_lower() %>% - stringr::str_replace_all(stringr::fixed(" "), "") %>% - stringr::str_replace_all("[^[:alnum:]]", "") - - column_name_clean <- column_name - - return(column_name_clean) -} - - -#### 3. Change Product data Date format and standardize columns #### - -# ============================================================================== -#' @title format_date_excelnum -#' -#' @description -#' Reformates dates entered into excel in wrong format (e.g., 44042) to final date format (yyyy-mm-dd) -format_date_excelnum <- function(product_df) { - rel_rows <- which(!grepl("-", product_df$product_entry_date) & !grepl("\\.", product_df$product_entry_date)) - product_df[rel_rows, "product_entry_date"] <- suppressWarnings(as.character(openxlsx::convertToDate(as.numeric(unlist(product_df[rel_rows, "product_entry_date"]))))) - - return(product_df) -} - -#' @title format_date_exceldate -#' -#' @description -#' Reformates dates entered into excel in wrong format (e.g., dd-mm-yyyy) to final date format (yyyy-mm-dd) -format_date_exceldate <- function(product_df) { - rel_rows <- which(grepl("-", product_df$product_entry_date) | grepl("\\.", product_df$product_entry_date)) - product_df[rel_rows, "product_entry_date"] <- suppressWarnings(as.character(lubridate::dmy(unlist(product_df[rel_rows, "product_entry_date"])))) - - return(product_df) -} - -#' @title format_date -#' -#' @description -#' Combines all date reformatting functions into one overall function -format_date <- function(product_df) { - product_df <- format_date_exceldate(product_df) - product_df <- format_date_excelnum(product_df) - - return(product_df) -} - -#' @title extract_month -#' -#' @description -#' Extract month as digit (e.g., 12 for december) based on excel tab name -extract_month <- function(sheetname) { - output <- dplyr::case_when( - grepl("Jan", sheetname) ~ "01", - grepl("Feb", sheetname) ~ "02", - grepl("Mar", sheetname) ~ "03", - grepl("Apr", sheetname) ~ "04", - grepl("May", sheetname) ~ "05", - grepl("Jun", sheetname) ~ "06", - grepl("Jul", sheetname) ~ "07", - grepl("Aug", sheetname) ~ "08", - grepl("Sep", sheetname) ~ "09", - grepl("Oct", sheetname) ~ "10", - grepl("Nov", sheetname) ~ "11", - grepl("Dec", sheetname) ~ "12" - ) - - return(output) -} - -#' @title recode_unitcolumnstozero -#' -#' @description -#' Recode NAs to 0 in all "unit columns" -recode_unitcolumnstozero <- function(product_df) { - product_df <- product_df %>% - dplyr::mutate_at(dplyr::vars(c("product_units_received", "product_units_released", "product_units_returned")), ~ replace(., is.na(.), 0)) - - return(product_df) -} - -#' @title clean_unitsreceived -#' -#' @description -#' Clean column "product_units_received" from character values and rows with values but no further information -clean_unitsreceived <- function(product_df) { - # Clean column "product_units_received from" from character values - drop_rows <- product_df %>% - dplyr::filter(grepl("START|END|BALANCE", product_units_received, ignore.case = TRUE)) %>% - dplyr::select(index) %>% - unlist() %>% - as.numeric() - product_df <- product_df %>% - dplyr::mutate(product_units_received = ifelse(index %in% drop_rows, - 0, - suppressWarnings(as.numeric(product_units_received)) - )) - - return(product_df) -} - - -#' @title Transfers in trackers with specific format which have "START BALANCE" in product data the start balance to the correct column. -#' -#' @description -#' Run before clean_receivedfrom and recode_unitcolumnstozero. Necessary for specific format such as in 2019_PKH and 2020_STH. When start balance in tracker next to text "start balance" one need to extract the balance either from the "received from" column or the "units released" column. -#' -#' @param product_df Input product data. -#' -#' @return Dataframe with corrected product_received_from column extracting the starting balance from trackers where necessary. -update_receivedfrom <- function(product_df) { - if (any(grepl("Balance", product_df[["product_units_received"]], ignore.case = TRUE)) & any(is.na(product_df$product_received_from))) { - product_df <- product_df %>% - dplyr::mutate(product_received_from = dplyr::case_when( - grepl("Balance", product_units_received, ignore.case = TRUE) & !is.na(product_units_released) ~ product_units_released, - grepl("Balance", product_units_received, ignore.case = TRUE) & !is.na(product_received_from) ~ product_received_from - )) %>% - dplyr::mutate(product_units_released = ifelse(!is.na(product_received_from), NA, product_units_released)) - - logInfo( - log_to_json( - message = "The rule for the case was applied successfully- Released (product_units_released) column also includes values for Start/End Balance", - script = "script1", - file = "helper_product_data.R", - functionName = "update_receivedfrom" - ) - ) - } - - return(product_df) -} - -#' @title clean_receivedfrom -#' -#' @description -#' Clean column "product_received_from" from character values -clean_receivedfrom <- function(product_df) { - # If there is notion of a balance trackkeeping in received column, which would align with a "START" statement in product_received column: - if (any(grepl("START", product_df[["product_units_received"]], ignore.case = TRUE))) { - # Copy balance start values from this column to product_balance column. Balance end values are not kept since these seem to represent final accounting values. - product_df <- product_df %>% - dplyr::mutate(product_balance = dplyr::case_when( - grepl("START", product_units_received, ignore.case = TRUE) ~ product_received_from - )) - } - - # Clean column - drop_rows <- product_df %>% - dplyr::filter(!grepl("[[:alpha:]]", product_received_from) & is.na(product_received_from) == FALSE) %>% - dplyr::select(index) %>% - unlist() %>% - as.numeric() - product_df <- product_df %>% - dplyr::mutate(product_received_from = ifelse(index %in% drop_rows, - NA, - product_received_from - )) - - return(product_df) -} - -#' @title compute_balance_cleanrows -#' -#' @description -#' Preparation function for compute_balance and compute_balance_status. -#' -#' Remove rows without any meaningful information regarding units (release, return, received) or entry status -compute_balance_cleanrows <- function(product_df) { - delete_rows <- as.vector(0) - for (i in 1:nrow(product_df)) { - if ((is.na(product_df$product_units_received[i]) == TRUE) & (is.na(product_df$product_units_released[i]) == TRUE) & (is.na(product_df$product_units_returned[i]) == TRUE) & - (is.na(product_df$product_released_to[i]) == TRUE) & (is.na(product_df$product_entry_date[i]) == TRUE) & (is.na(product_df$product_balance[i]) == TRUE)) { - delete_rows[length(delete_rows) + 1] <- i - } - } - - if (delete_rows > 0) { - product_df <- product_df[-delete_rows, ] - } - - return(product_df) -} - -#' @title compute_balance_status -#' -#' @description -#' Computate balance status (start vs. change vs. end) per product -compute_balance_status <- function(product_df) { - # Define change as default - indices_start <- product_df %>% - dplyr::group_by(product) %>% - dplyr::slice_head(n = 1) %>% - dplyr::as_tibble() %>% - dplyr::select(index) %>% - unlist() %>% - as.numeric() - - indices_end <- product_df %>% - dplyr::group_by(product) %>% - dplyr::slice_tail(n = 1) %>% - dplyr::as_tibble() %>% - dplyr::select(index) %>% - unlist() %>% - as.numeric() - - product_df <- product_df %>% - dplyr::mutate(product_balance_status = dplyr::case_when( - index %in% indices_start ~ "start", - index %in% indices_end ~ "end", - TRUE ~ "change" - )) - - return(product_df) -} - -#' @title compute_balance -#' -#' @description -#' Computes balance based on start balance value, units released, units returned, units received.- -compute_balance <- function(product_df, year) { - # Change variable type - product_df["product_balance"] <- as.numeric(unlist(product_df["product_balance"])) - product_df["product_units_released"] <- as.numeric(unlist(product_df["product_units_released"])) - product_df["product_units_returned"] <- as.numeric(unlist(product_df["product_units_returned"])) - - # Need to put units received and released to 0 since here only summarising end of month - if (year >= 2021) { - product_df <- product_df %>% - dplyr::mutate(product_units_received = dplyr::if_else(product_balance_status == "end", 0, product_units_received)) %>% - dplyr::mutate(product_units_released = dplyr::if_else(product_balance_status == "end", 0, product_units_released)) - } - - # Calculate start balance: take units_received value if there is no balance value present - product_df <- product_df %>% - dplyr::group_by(product) %>% - dplyr::mutate( - product_balance = dplyr::case_when( - (product_balance_status == "start") & (is.na(product_balance) == TRUE) ~ (product_units_received - product_units_released), - TRUE ~ as.numeric(product_balance) - ) - ) - - # Calculate change and end of balance values: take last balance + received - release - for (i in 1:nrow(product_df)) { - if (product_df$product_balance_status[i] %in% c("change", "end")) { - release <- NA - reception <- NA - if (is.na(as.numeric(product_df[i, "product_units_released"])) == TRUE) { - release <- as.numeric(0) - } else { - release <- as.numeric(product_df[i, "product_units_released"]) - } - if (is.na(as.numeric(product_df[i, "product_units_received"])) == TRUE) { - reception <- as.numeric(0) - } else { - reception <- as.numeric(product_df[i, "product_units_received"]) - } - product_df[i, "product_balance"] <- as.numeric(product_df[i - 1, "product_balance"]) - release + reception - } - } - - # ToDo LOW (KW): Ungroup statement can possibly be moved a further up limiting - # the group scope for clarity. I added the ungroup as it was missing from the - # function, but did not look for the best position so far. - return(product_df %>% dplyr::ungroup()) -} - -#' @title adjust_column_classes -#' -#' @description -#' Adjust classes of dataframe names -adjust_column_classes <- function(product_df) { - list_date <- c("product_entry_date") - list_character <- c( - "product", "product_received_from", "product_released_to", "product_returned_by", "product_balance_status", - "product_sheet_name", "file_name", "product_units_notes" - ) - list_numeric <- c( - "product_units_received", "product_units_released", "product_balance", "product_units_returned", - "product_table_month", "product_table_year", "index" - ) - - product_df <- product_df %>% - dplyr::mutate_at(list_date, as.Date) %>% - dplyr::mutate_at(list_character, as.character) %>% - dplyr::mutate_at(list_numeric, as.numeric) - - return(product_df) -} - -#' @title testit -#' -#' @description -#' Wait for x seconds in script -testit <- function(x) { - p1 <- proc.time() - Sys.sleep(x) - proc.time() - p1 -} - - -#### 4. Extract concatenated product name information #### - -#' @title extract_product_multiple -#' -#' @description -#' Splits product name cells with multiple values into separate rows. Uses numeric product content in cell for columns units_received or units_released -extract_product_multiple <- function(product_df) { - # Split multiple-product cell into several rows - df_product_test_x <- product_df %>% - tidyr::separate_rows(., product, convert = TRUE, sep = "\\; ") %>% - tidyr::separate_rows(., product, convert = TRUE, sep = " and ") - - - # Extract product unit information from product cell and add to release or received cell, depending on entries in the corresponding column - df_product_test_x$product_units_notes <- as.character(NA) - - # If units columns not present yet, add - if ("product_units_received" %notin% colnames(df_product_test_x)) { - df_product_test_x$product_units_received <- as.numeric(NA) - } # else{ - # df_product_test_x$product_units_received <- as.numeric(df_product_test_x$product_units_received)} - if ("product_units_released" %notin% colnames(df_product_test_x)) { - df_product_test_x$product_units_released <- as.numeric(NA) - } # else{ - # df_product_test_x$product_units_released <- as.numeric(df_product_test_x$product_units_released)} - - # Create column "notes", containing input from brackets if word "box" or "unit" is present - df_product_test_x <- df_product_test_x %>% - dplyr::rowwise() %>% - dplyr::mutate( - product_units_notes = dplyr::case_when( # e.g., "1 box" or "1 unit". Other units (e.g., 2ml x5) are not extracted. - (grepl("\\(", product) & grepl("\\)", product) & (grepl("box", product) | grepl("unit", product))) ~ as.character(substring(stringr::str_extract_all(product, "\\([^()]+\\)")[[1]], 2, nchar(stringr::str_extract_all(product, "\\([^()]+\\)")[[1]]) - 1))[1], - TRUE ~ product_units_notes - ) - ) - - # Adjust columns units_receceived and units_released with box/unit content in brackets - for (row in 1:nrow(df_product_test_x)) { - product <- df_product_test_x[row, "product"] - if ("product_received_from" %in% colnames(df_product_test_x)) { - product_received_from <- df_product_test_x[row, "product_received_from"] - } else { - product_received_from <- NA - } - if ("product_released_to" %in% colnames(df_product_test_x)) { - product_released_to <- df_product_test_x[row, "product_released_to"] - } else { - product_released_to <- NA - } - - # recode product_units_received - if ((grepl("\\(", product) & grepl("\\)", product) & (grepl("box", product) | grepl("unit", product)) & is.na(product_received_from) == FALSE) & (!grepl("\\/", product))) { - df_product_test_x[row, "product_units_received"] <- as.character( - as.numeric( - stringi::stri_extract_all_regex( - substring( - stringr::str_extract_all(product, "\\([^()]+\\)")[[1]], - 2, - nchar(stringr::str_extract_all(product, "\\([^()]+\\)")[[1]]) - 1 - ), - "[1-9]+" - )[[1]][1] - ) - ) - } - - # recode product_units_released - if ((grepl("\\(", product) & grepl("\\)", product) & (grepl("box", product) | grepl("unit", product)) & is.na(product_released_to) == FALSE) & (!grepl("\\/", product))) { - df_product_test_x[row, "product_units_released"] <- as.numeric( - stringi::stri_extract_all_regex( - substring( - stringr::str_extract_all(product, "\\([^()]+\\)")[[1]], - 2, - nchar(stringr::str_extract_all(product, "\\([^()]+\\)")[[1]]) - 1 - ), - "[1-9]+" - )[[1]][1] - ) - } - } - - return(df_product_test_x) -} diff --git a/R/logger.R b/R/logger.R deleted file mode 100644 index 47b2072..0000000 --- a/R/logger.R +++ /dev/null @@ -1,89 +0,0 @@ -#' Configure basic logger -#' -#' @param output_dir: Output directory of the main script -#' @export -setup_logger <- function(output_dir, log_name) { - logFileName <- file.path(output_dir, "logs", paste0("main_", log_name, ".log")) - - if (file.exists(logFileName)) { - file.remove(logFileName) - } - - logger <- createLogger( - name = "MAIN", - threshold = "TRACE", - appenders = list( - createFileAppender( - layout = layoutParallel, - fileName = logFileName - ) - ) - ) - registerLogger(logger) - - log_dir <- file.path(output_dir, "logs") - - fs::dir_create(log_dir) -} - - -#' @title Add a basic default file logger for a newly processed file. -#' -#' @param logfile The name of the log file. -#' @param output_root Output root directory for the current process. -#' -#' @return returns the loggers that where previously set for usage with with_ -setup_file_logger <- function(logfile, output_root) { - loggers <- getLoggers() - clearLoggers() - logFileName <- file.path(output_root, "logs", paste0(logfile, ".log")) - if (file.exists(logFileName)) { - file.remove(logFileName) - } - addDefaultFileLogger(logFileName, name = logfile) - loggers -} - - -#' Temporary File Logger -#' -#' Temporarily change logging to exclusively log to specific log file -#' -#' @param new New filename for the logfile -#' @param code Code to be executed -#' @param output_root Root output folder from where to construct the log path - -#' @export -with_file_logger <- withr::with_( - setup_file_logger, - function(loggers) { - clearLoggers() - for (logger in loggers) { - registerLogger(logger) - } - } -) - - -#' @title Log-To-Json Formatter -#' -#' @param message log message with glue syntax {values['X']}. -#' @param values A named list with the glue referenced values as single strings. -#' @param script script name where the log message was created. -#' @param file file name where the log message was created. -#' @param errorCode A unique error code for easy aggregation of error messages. -#' @param warningCode A unique warning code for easy aggregation of warning messages. -#' @param functionName The name of the function where the log message was created. -#' @return stringified JSON to log with logX functions -log_to_json <- function(message, values = NA, script = NA, file = NA, errorCode = NA, warningCode = NA, functionName = NA) { - logObject <- list( - message = glue::glue(message), - values = values, - script = script, - file = file, - errorCode = errorCode, - warningCode = warningCode, - functionName = functionName - ) - return(jsonlite::toJSON(logObject, flatten = TRUE, auto_unbox = TRUE)) -} diff --git a/R/read_product_data.R b/R/read_product_data.R deleted file mode 100644 index 41adc5d..0000000 --- a/R/read_product_data.R +++ /dev/null @@ -1,682 +0,0 @@ -# function to process product data in step 1 -# extract product data with existing functions -# rename columns and remove empty rows/columns -# function based on parts from run_a4d_product_data.R and helper functions -reading_product_data_step1 <- - function(tracker_data_file, columns_synonyms) { - # rename column names to match - colnames(columns_synonyms) <- c("name_clean", "name_to_be_matched") - - # get all month sheets with product data, get year - sheet_list <- readxl::excel_sheets(tracker_data_file) - month_list <- sheet_list[na.omit(pmatch(month.abb, sheet_list))] - year <- get_tracker_year(tracker_data_file, month_list) - - logInfo( - log_to_json( - message = "Found {values['len']} sheets: {values['sheets']}.", - values = list(len = length(sheet_list), sheets = sheet_list), - script = "script1", - file = "read_product_data.R", - functionName = "reading_product_data_step1" - ) - ) - logInfo( - log_to_json( - message = "Found {values['len']} month sheets: {values['months']}.", - values = list(len = length(month_list), months = month_list), - script = "script1", - file = "read_product_data.R", - functionName = "reading_product_data_step1" - ) - ) - - # loop through all months - for (curr_sheet in month_list) { - # open tracker data - tracker_data <- data.frame( - readxl::read_xlsx( - path = tracker_data_file, - sheet = curr_sheet, - col_types = c("text"), - .name_repair = "unique_quiet" - ) - ) - - # Jump to next tab sheet if there are no product data - if ( - !any((grepl("Product", tracker_data[, ]) | - grepl("Description of Support", tracker_data[, ]))) - ) { - # go to next month - logWarn( - log_to_json( - message = "Sheet {values['sheet']}: Could not find product data in tracker data. Skipping.", - values = list(sheet = curr_sheet), - script = "script1", - file = "read_product_data.R", - functionName = "read_product_data_step1", - warningCode = "invalid_tracker" - ) - ) - - next - } - - # Extract relevant data and renamn columns - product_df <- extract_product_data(tracker_data) - - if (check_wide_format_columns(product_df)) { - logWarn( - log_to_json( - message = "Sheet {values['sheet']} has Mandalay Children's Hospital data (2019-2021) wide-format columns. The wide-format area should be changed to long-format!", - values = list(sheet = curr_sheet), - script = "script1", - file = "read_product_data.R", - functionName = "read_product_data_step1", - warningCode = "invalid_tracker" - ) - ) - product_df <- create_new_rows(product_df) - } - - if (any(grepl("2017_Mandalay|2018_Mandalay|2019_Mandalay", basename(tracker_data_file))) & - (check_patterns_in_column(product_df, (find_string_cols(product_df, "Released To"))))) { - logWarn( - log_to_json( - message = "Sheet {values['sheet']} Tracker data file contains Mandalay Children's Hospital data (2017-2019), may contain wide-format cells. The wide-format cells should be changed to standart long-format!", - values = list(sheet = curr_sheet), - script = "script1", - file = "read_product_data.R", - functionName = "read_product_data_step1", - warningCode = "invalid_tracker" - ) - ) - product_df <- wide_cells_2_rows(product_df) - } - - # If after extraction, dataframe is empty, this iteration is also skipped. - if (all(is.na(product_df))) { - logWarn( - log_to_json( - message = "Sheet {values['sheet']}: Product data is empty. Skipping.", - values = list(sheet = curr_sheet), - script = "script1", - file = "read_product_data.R", - functionName = "read_product_data_step1", - warningCode = "invalid_tracker" - ) - ) - - next - } - - # harmonize to names - product_df <- product_df %>% harmonize_input_data_columns(columns_synonyms, curr_sheet) - - # Remove meaningless rows with no information - del_rows <- apply(product_df, MARGIN = 1, function(x) sum(!is.na(x))) # count per row how many cols are not NA. If only 0, there is not enough information and row is dropped. - del_rows <- as.numeric(which(del_rows < 1)) - - product_df <- product_df %>% - dplyr::slice(., -del_rows) %>% - dplyr::filter_all(dplyr::any_vars(complete.cases(.))) %>% # Remove empty rows - dplyr::filter((product != "Product" | is.na(product)) & (product != "PATIENT DATA SUMMARY" | is.na(product))) # remove new headers from data 2022 onwards - - # Checking if the patient's name is missing next to the released units - col_released <- "product_units_released" - col_released_to <- "product_released_to" - num_na_rows <- count_na_rows(product_df, col_released, col_released_to) - tryCatch( - { - if (num_na_rows > 0) { - logWarn( - log_to_json( - message = "Sheet {values['sheet']} has {values['num_na_rows']} rows where the patient's name is missing next to the released units.", - values = list(sheet = curr_sheet, num_na_rows = num_na_rows), - script = "script1", - file = "read_product_data.R", - functionName = "read_product_data_step1", - warningCode = "invalid_tracker" - ) - ) - } - }, - error = function(e) { - logError( - log_to_json( - message = "Sheet {values['sheet']}: Error when testing num_na_rows > 0: {values['e']}", - values = list(sheet = curr_sheet, e = e$message), - script = "script1", - file = "read_product_data.R", - functionName = "read_product_data_step1", - errorCode = "tryCatch" - ) - ) - } - ) - - # Checking non-processed dates in product_entry_date - non_processed_dates <- product_df %>% - dplyr::filter(!is.na(product_entry_date) & !grepl("^[0-9]+$", product_entry_date)) # filter if: no NA and the string is not only numeric. - tryCatch( - { - if (nrow(non_processed_dates) > 0) { - logWarn( - log_to_json( - message = "Sheet {values['sheet']}: The number of rows with non-processed dates in product_entry_date is {values['num_rows']}: {values['dates']}.", - values = list(sheet = curr_sheet, num_rows = nrow(non_processed_dates), dates = non_processed_dates$product_entry_date), - script = "script1", - file = "read_product_data.R", - functionName = "read_product_data_step1", - warningCode = "invalid_value" - ) - ) - } - }, - error = function(e) { - logError( - log_to_json( - message = "Sheet {values['sheet']}: Error when testing nrow(non_processed_dates) > 0: {values['e']}", - values = list(sheet = curr_sheet, e = e$message), - script = "script1", - file = "read_product_data.R", - functionName = "read_product_data_step1", - errorCode = "tryCatch" - ) - ) - } - ) - - # Add country, hospital, month, year, tabname - product_df <- product_df %>% - dplyr::mutate( - product_table_month = extract_month(curr_sheet), - product_table_year = year, - product_sheet_name = curr_sheet - ) - - # Check if the entry dates for the balance match the month/year on the sheet - check_entry_dates(product_df, curr_sheet) - - # Remove leading spaces from the 'product_released_to' column - if ("product_released_to" %in% colnames(product_df)) { - product_df$product_released_to <- trimws(product_df$product_released_to, which = "left") - } - - # Replace Extra "Total" Values with NA - if ("product_units_released" %in% colnames(product_df)) { - product_df <- replace_extra_total_values_with_NA(product_df, "product_units_released") - } - - # combine all months - if (!exists("df_final")) { - df_final <- product_df - } else { - df_final <- dplyr::bind_rows(df_final, product_df) - } - } - if (exists("df_final")) { - return(df_final) - } else { - return(NULL) - } - } - -#' @title Count rows with missing patient's name next to the released units -#' -#' @description -#' This function checks if the patient's name is missing next to the released units in a given dataframe. It counts the number of such rows. -#' -#' @param df A dataframe containing patient data. -#' @param units_released_col The name of the column that contains information about the units released. -#' @param released_to_col The name of the column that contains information about who the units were released to (patient's name). -#' -#' @return The number of rows where the patient's name is missing next to the released units. -count_na_rows <- function(df, units_released_col, released_to_col) { - na_rows <- df[is.na(df[[released_to_col]]) & !is.na(df[[units_released_col]]), ] - nrow(na_rows) -} - - -#' @title Check entry dates in product data -#' -#' @description -#' This function checks if the entry dates for the balance match the month/year on the sheet. If any discrepancies are found, it logs a warning message with the number of mismatched dates and their corresponding 'product_entry_date' values. -#' -#' @param df A dataframe containing product data. -#' @param Sheet The name of the sheet where the product data is located. -#' -#' @return This function does not return a value. It logs a warning message if there are any dates in 'product_entry_date' that don't match the month/year on the sheet. -check_entry_dates <- function(df, Sheet) { - # Check if the entry dates for the balance match the month/year on the sheet - entry_dates_df <- df %>% dplyr::filter(grepl("^[0-9]+$", product_entry_date)) - - entry_dates_df$product_entry_date <- as.numeric(entry_dates_df$product_entry_date) - entry_dates_df$product_table_month <- as.numeric(entry_dates_df$product_table_month) - - # Extract month and year from product_entry_date column - entry_dates_df$ed_date <- as.Date(entry_dates_df$product_entry_date, origin = "1899-12-30") - entry_dates_df$ed_month <- as.numeric(format(entry_dates_df$ed_date, "%m")) - entry_dates_df$ed_year <- as.numeric(format(entry_dates_df$ed_date, "%Y")) - - # Compare month and year with product_table_month and product_table_year - not_same <- entry_dates_df[entry_dates_df$ed_month != entry_dates_df$product_table_month | - entry_dates_df$ed_year != entry_dates_df$product_table_year, ] - if (nrow(not_same) > 0) { - logWarn( - log_to_json( - message = "Sheet {values['sheet']}: There are {values['nrow']} dates in product_entry_date that don't match extracted month and year.", - values = list(sheet = Sheet, nrow = nrow(not_same), dates = not_same$ed_date), - script = "script1", - file = "read_product_data.R", - functionName = "check_entry_dates", - warningCode = "invalid_value" - ) - ) - } -} - -#' @title Remove Rows with NA Values in Specified Columns. -#' -#' @description -#' This function takes a data frame and a vector of column names as input. It removes rows from the data frame where all the specified columns contain only NA values. If any of the specified columns have at least one non-NA value, the corresponding row will be retained in the output data frame. -#' -#' @param df A data frame that contains the data from which rows need to be removed. -#' @param column_names A character vector specifying the column names to be checked for NA values. -#' -#' @return A new data frame with rows removed if all the specified columns in the row contain NA values. The returned data frame will have the same structure as the input data frame, but with rows that satisfy the condition removed. -remove_rows_with_na_columns <- - function(df, column_names) { - # Get the row indices where all specified columns are NA - na_rows <- apply(df[column_names], 1, function(x) all(is.na(x))) - - # log message - logInfo( - log_to_json( - message = "{values['na_rows']} out of {values['nrows']} rows with NA values in specified columns were removed.", - values = list(na_rows = length(na_rows[na_rows == T]), nrows = nrow(df)), - script = "script1" - ) - ) - - # Return the data frame without the NA rows - return(df[!na_rows, ]) - } - - -#' @title Check negative values in 'product_balance' column -#' -#' @description -#' This function checks for negative values in the 'product_balance' column of a given dataframe. If any negative values are found, it logs a warning message with the number of negative values and their corresponding 'product_balance' values. -#' -#' @param df A dataframe containing product data. -#' @param Sheet The name of the sheet where the product data is located. -#' -#' @return This function does not return a value. It logs a warning message if there are any negative values in the 'product_balance' column. -check_negative_balance <- function(df, Sheet) { - # Create a new data frame containing only rows with negative values in product_balance column - negative_df <- df[df$product_balance < 0, ] - - # Check if there are any rows in the new data frame - if (nrow(negative_df) > 0) { - # Log a warning message with the number of negative values and their corresponding product_balance values - logWarn( - log_to_json( - message = "Sheet {values['sheet']}: There are {values['nrow']} negative values in product_balance: {values['values']}.", - values = list(sheet = Sheet, nrow = nrow(negative_df), values = negative_df$product_balance), - script = "script1", - file = "read_product_data.R", - functionName = "check_negative_balance", - warningCode = "invalid_value" - ) - ) - } -} - - -#' @title Switch product_received_from and product_units_received column. -#' -#' @description -#' Renaming columns when string Remaining stock in wrong column. E.g. 2018_PNG for Nov and Dec. -#' -#' -#' @param df Dataframe. Output of tracker file from script 1 for product data. -#' -#' @return Dataframe with columns switched (renamed) -switch_columns_stock <- - function(df) { - if (sum(stringr::str_detect(df$product_units_received[!is.na(df$product_units_received)], "Remaining Stock")) > 0) { - df <- df %>% - dplyr::rename( - "product_units_received" = "product_received_from", - "product_received_from" = "product_units_received" - ) - logDebug( - log_to_json( - message = "Columns product_units_received and product_received_from were switched", - script = "script1", - file = "read_product_data.R", - functionName = "switch_columns_stock" - ) - ) - } - - return(df) - } - -#' @title Compare two lists and return unmatched strings -#' -#' @description -#' This function compares two lists and returns the strings that are present in the first list but not in the second list. -#' -#' @param list1 The first list of strings to be compared. -#' @param list2 The second list of strings to be compared. -#' -#' @return A vector of strings that are present in 'list1' but not in 'list2'. -compare_lists <- function(list1, list2) { - # Use the setdiff function to find strings in list1 that are not in list2 - unmatched_strings <- setdiff(list1, list2) - - return(unmatched_strings) -} - - -#' @title Report unknown products -#' -#' @description -#' This function reports unknown products. It compares the product list in a given dataframe with a stock list, and logs any products that are not found in the stock list. -#' -#' @param df A dataframe containing product data. -#' @param Sheet The name of the sheet where the product data is located. -#' @param stock_list_df A dataframe containing the stock list of products. -#' -#' @return This function does not return a value. It logs a warning message if there are any unknown products, and logs an info message if there are no unknown products. -report_unknown_products <- function(df, Sheet, stock_list_df) { - # Create lists containing only products names - products_list <- df$product - stock_products_list <- stock_list_df$product - - products_list <- tolower(products_list) - products_list <- products_list[!is.na(products_list)] - stock_products_list <- tolower(stock_products_list) - - # Create a new list containing only unknown products names - unmatched_products <- compare_lists(products_list, stock_products_list) - - # Check if there are any unknown products names - if (length(unmatched_products) > 0) { - # Log a warning message with the number of unknown products names - logWarn( - log_to_json( - message = "Sheet {values['sheet']} has {values['len_products']} unknown product names: {values['unknown_products']}.", - values = list(sheet = Sheet, len_products = length(unmatched_products), unknown_products = unmatched_products), - script = "script1", - file = "read_product_data.R", - functionName = "report_unknown_products", - warningCode = "invalid_value" - ) - ) - } -} - - -#' @title Load Product List from Stock Summary -#' -#' @description -#' This function loads the product list from 'Stock_Summary' sheet in an Excel file. -#' -#' @param stock_summary_xlsx A string that represents the path to the Excel file. Defaults to "reference_data/master_tracker_variables.xlsx". -#' -#' @return A data frame that contains the product names. If there is an error during the process, it logs the error message. -#' -#' @examples -#' \dontrun{ -#' product_list <- load_product_reference_data() -#' product_list <- load_product_reference_data("your_file.xlsx") -#' } -load_product_reference_data <- function(stock_summary_xlsx = "reference_data/master_tracker_variables.xlsx") { - tryCatch( - { - product_names_df <- readxl::read_excel(stock_summary_xlsx, "Stock_Summary") - colnames(product_names_df) <- tolower(colnames(product_names_df)) - logDebug( - log_to_json( - message = "{values['nrow']} product names were loaded from the Stock Summary.", - values = list(nrow = nrow(product_names_df)), - script = "script2", - file = "read_product_data.R", - functionName = "load_product_reference_data" - ) - ) - - return(product_names_df) - }, - error = function(e) { - logError( - log_to_json( - "Could not load stock product list. Error = {values['e']}.", - values = list(e = e$message), - script = "script2", - file = "read_product_data.R", - errorCode = "tryCatch", - functionName = "load_product_reference_data" - ) - ) - } - ) -} - -#' Inserts product categories if contained in a provided mapping -#' -#' @description Product categories are added in the sense of a left join, -#' meaning missing categories will result in NA values while products are never -#' dropped. -#' -#' @param inventory_data A product inventory dataframe containing a "product" column -#' @param product_category_mapping A dataframe mapping "product" to "category" -#' -#' @return A dataframe containing the inventory data with an additional product -#' category column. -add_product_categories <- function(inventory_data, product_category_mapping) { - inventory_data %>% - dplyr::left_join( - dplyr::rename(product_category_mapping, product_category = "category"), - c("product") - ) -} - -#' @title Extract Unit Capacity from a Specified Column -#' -#' @description -#' This function extracts the unit capacity from a specified column in a dataframe. -#' It assumes that the unit capacity is represented by numbers immediately before 's or s within parentheses. -#' All strings which contain the word 'singles' extracted as 1. -#' Non-numeric characters are removed and the extracted values are converted to numeric. -#' NA values are replaced with 1. -#' -#' @param df A dataframe that contains the column to extract unit capacity from. -#' @param column_name The name of the column to extract unit capacity from. -#' -#' @return A dataframe with an additional column 'product_unit_capacity' that contains the extracted unit capacity. -#' @export -#' -#' @examples -#' \dontrun{ -#' df <- data.frame(product = c("Product A (2s)", "Product B (3's)", "Product C")) -#' df <- extract_unit_capacity(df, "product") -#' } -extract_unit_capacity <- function(df, column_name) { - # Extract all symbols between parentheses - df$product_unit_capacity <- stringr::str_extract(df[[column_name]], "\\(([^)]+)\\)") - - # Recode all strings which contain the word 'singles' into '1s' - df$product_unit_capacity <- ifelse(grepl("singles", df$product_unit_capacity, ignore.case = TRUE), - "1s", df$product_unit_capacity - ) - - # Extract numbers that are immediately before 's or s - df$product_unit_capacity <- stringr::str_extract(df$product_unit_capacity, "\\d+(?=s|'s)") - - # Remove non-numeric characters - df$product_unit_capacity <- gsub("[^0-9]", "", df$product_unit_capacity) - - # Convert the new field to numeric - df$product_unit_capacity <- as.numeric(df$product_unit_capacity) - - # Add 1 to NA values - df$product_unit_capacity[is.na(df$product_unit_capacity)] <- 1 - - return(df) -} - -#' @title Process product data in script 2. -#' -#' @description -#' Processes data output from script 1. Data will be further cleaned and processed to fix data quality issues and to create a report. -#' -#' -#' @param df Dataframe. Output of tracker file from script 1 for product data. -#' @param columns_synonyms Tibble with synonyms for product data (columns: name_clean and name_to_be_matched) -#' -#' @return Cleaned product data for one specified tracker. -reading_product_data_step2 <- - function(df, columns_synonyms) { - # rename column names to match - colnames(columns_synonyms) <- c("name_clean", "name_to_be_matched") - - # save all results - df_final <- c() - - # get product list from Stock_Summary - product_reference_data <- load_product_reference_data() - known_products <- product_reference_data %>% dplyr::select(product) - product_category_mapping <- product_reference_data %>% dplyr::select(product, category) - - # loop through all months - for (sheet_month in unique(df$product_sheet_name)) { - # filter on month sheet - product_df <- df %>% - dplyr::filter(product_sheet_name == sheet_month) - - # Split product cells with various products and/or unit information - product_df <- extract_product_multiple(product_df) - - # Add columns that should be in final dataframe but are still missing - columns_missing <- columns_synonyms %>% - dplyr::group_by(name_clean) %>% - dplyr::distinct(., name_clean) %>% - unlist() %>% - as.character() - - missing_cols <- which(columns_missing %notin% colnames(product_df)) - missing_cols_names <- unique(columns_missing[missing_cols]) - product_df[missing_cols_names] <- NA - - # switch column names if applicable - product_df <- switch_columns_stock(product_df) - - # Remove rows which do not contain any new information - column_names_check <- c("product_entry_date", "product_units_received", "product_received_from", "product_units_released", "product_released_to", "product_units_returned", "product_returned_by") - product_df <- remove_rows_with_na_columns(product_df, column_names_check) - # jump to next sheet if dataframe empty from here - if (nrow(product_df) == 0) { - logWarn( - log_to_json( - message = "Sheet {values['sheet']} is empty after filtering. Skipping", - values = list(sheet = sheet_month), - script = "script2", - file = "read_product_data.R", - functionName = "reading_product_data_step2", - warningCode = "invalid_tracker" - ) - ) - - next - } - - # Add row index - product_df$index <- seq(1, nrow(product_df), 1) - - # Recode date - product_df <- format_date(product_df) - - # Extend product name and sort by product - # Keep first row and last row and order the rest by date - product_df <- product_df %>% - dplyr::ungroup() %>% - tidyr::fill(c(product), .direction = "down") %>% - dplyr::group_by(product) %>% - dplyr::mutate(rank = ifelse(dplyr::row_number() == 1, 1, - dplyr::if_else(dplyr::row_number() == dplyr::n(), dplyr::n() + 2, - dplyr::if_else(is.na(product_entry_date), dplyr::row_number(), dplyr::dense_rank(product_entry_date) + 1) - ) - )) %>% - dplyr::arrange(product, rank) %>% - dplyr::ungroup() %>% - dplyr::select(-rank) - - # extract start/end balance when they are in the sheet (e.g. 2019_PKH and 2020_STH examples) - product_df <- update_receivedfrom(product_df) - - # Recode all NAs in unit columns to 0 - product_df <- recode_unitcolumnstozero(product_df) - - # Clean columns "units received" and "received from" from unexpected character (units) vs. numeric (received from) values. - # In first function, received_from numeric information (Balance status at START BALANCE) is transferred to variable product_balance for later computation of this variable. - product_df <- clean_receivedfrom(product_df) - product_df <- clean_unitsreceived(product_df) - - # Recode all NAs in unit columns to 0, if NA coercion was applied earlier - product_df <- recode_unitcolumnstozero(product_df) - - ## Compute balance - # Remove rows without changes in units release or without information on balance - product_df <- compute_balance_cleanrows(product_df) - - # Compute balance_status (start vs. end vs. change) - product_df <- compute_balance_status(product_df) - - # Compute balance - product_df <- compute_balance(product_df, product_df$product_table_year[1]) - - # Adjust classes of variables - product_df <- adjust_column_classes(product_df) - - # remove index column - product_df <- subset(product_df, select = -index) - - # check negative values in product_balance column - check_negative_balance(product_df, sheet_month) - - # Report unknown product names - if (exists("known_products") && !is.null(known_products) && nrow(known_products) > 0) { - report_unknown_products(product_df, sheet_month, known_products) - } - - product_df <- product_df %>% add_product_categories(product_category_mapping) - - # extract Unit Capacity from product column - product_df <- extract_unit_capacity(product_df, "product") - - #### hospital and country information missing here!! - - # finish and combine - df_final <- df_final %>% - rbind(product_df) - } - - if (nrow(df_final) > 0) { - return(df_final) - } else { - logWarning( - log_to_json( - message = "No product data extracted for the following tracker: {values['name']}.", - values = list(name = df$file_name[1]), - file = "read_product_data.R", - functionName = "reading_product_data_step2", - warningCode = "script2_warning_product_data" - ) - ) - } - } diff --git a/R/script1_get_tracker_year.R b/R/script1_get_tracker_year.R deleted file mode 100644 index 3a5fd35..0000000 --- a/R/script1_get_tracker_year.R +++ /dev/null @@ -1,18 +0,0 @@ -#' @title Get the tracker year -#' -#' @description -#' This function parses the tracker year from either the monthly sheets -#' or the tracker file name if the parsing is not successful -#' -#' @param tracker_data_file The path to the tracker file -#' @param month_list The list of all the monthly sheets within a tracker -#' -#' @return The year of the tracker -#' @export -get_tracker_year <- function(tracker_data_file, month_list) { - year <- suppressWarnings(2000 + unique(readr::parse_number(month_list)[1])) - if (is.na(year)) { - year <- as.integer(stringr::str_match(tracker_data_file, "[:digit:]{4}")) - } - return(year) -} diff --git a/R/script1_helper_read_patient_data.R b/R/script1_helper_read_patient_data.R deleted file mode 100644 index fafa76f..0000000 --- a/R/script1_helper_read_patient_data.R +++ /dev/null @@ -1,179 +0,0 @@ -#' Extract the patient data from a month sheet of a tracker file. -#' -#' @description -#' Searches for first and last row that starts with country and clinic code -#' and extracts the data from this range as data.frame -#' -#' -#' @param tracker_data_file Excel tracker file. -#' @param sheet Excel sheet name. -#' @param year year of this tracker. -#' -#' @return data.frame with the patient data -#' @export -extract_patient_data <- function(tracker_data_file, sheet, year) { - tracker_data <- openxlsx::read.xlsx( - xlsxFile = tracker_data_file, - sheet = sheet, - colNames = F, - detectDates = F, - skipEmptyRows = F, - skipEmptyCols = F, - fillMergedCells = T - ) - # tracker_data <- readxl::read_excel( - # path = tracker_data_file, - # sheet = sheet, - # range = readxl::cell_limits(c(1, NA), c(NA, NA)), - # trim_ws = T, - # col_names = F, - # .name_repair = "unique_quiet" - # ) - - # Assumption: first column is always empty until patient data begins - patient_data_range <- which(!is.na(tracker_data[, 1])) - row_min <- min(patient_data_range) - row_max <- max(patient_data_range) - - testit::assert(row_min <= row_max) # <= because there could only be a single patient - - header_cols <- - stringr::str_replace_all(tracker_data[row_min - 1, ], "[\r\n]", "") - header_cols_2 <- - stringr::str_replace_all(tracker_data[row_min - 2, ], "[\r\n]", "") - - testit::assert(!(is.na(header_cols[2]) && is.na(header_cols_2[2])), fact = "Second header column must not be empty.") - - # tracker since 2022 used "Updated 2022" in the monthly sheets to note the date for Blood Pressure Update Date - # the same column name appears in the Patient List so we have to replace it here and make it unique - if ( - year >= 2022 && - any(stringr::str_detect(header_cols_2, stringr::regex("Level\\s?of\\s?Education", ignore_case = TRUE)), na.rm = TRUE) && - any(stringr::str_detect(header_cols_2, stringr::regex("Updated\\s?2022", ignore_case = TRUE)), na.rm = TRUE) - ) { - header_cols_2 <- stringr::str_replace(header_cols_2, stringr::regex("Updated\\s?2022", ignore_case = TRUE), "Level of Education Or Occupation") - } - if ( - year >= 2022 && - any(stringr::str_detect(header_cols_2, stringr::regex("Blood\\s?Pressure", ignore_case = TRUE)), na.rm = TRUE) && - any(stringr::str_detect(header_cols_2, stringr::regex("Updated\\s?2022", ignore_case = TRUE)), na.rm = TRUE) - ) { - header_cols_2 <- stringr::str_replace(header_cols_2, stringr::regex("Updated\\s?2022", ignore_case = TRUE), "Blood Pressure") - } - - - # trackers from 2022 and newer have an empty first row - # and openxlsx always skips empty rows at the start of the file - if ((year == 2022 && sheet != "Patient List") || year > 2022) { - row_min <- row_min + 1 - row_max <- row_max + 1 - } - - logInfo( - log_to_json( - message = "Sheet {values['sheet']}: Patient data found in rows {values['row_min']} to {values['row_max']}.", - values = list(sheet = sheet, row_min = row_min, row_max = row_max), - script = "script1", - file = "helper_read_patient_data.R", - functionName = "extract_patient_data" - ) - ) - - df_patient <- readxl::read_excel( - path = tracker_data_file, - sheet = sheet, - range = readxl::cell_limits(c(row_min, NA), c(row_max, length(header_cols))), - trim_ws = F, - col_names = F, - col_types = c("text"), - .name_repair = "unique_quiet" - ) - - if (header_cols[2] == header_cols_2[2]) { - # take into account that date info gets separated from the updated values (not in the same row, usually in the bottom row) - logInfo( - log_to_json( - message = "Multiline header found. Merging header rows.", - script = "script1", - file = "helper_read_patient_data.R", - functionName = "extract_patient_data" - ) - ) - - diff_colnames <- which((header_cols != header_cols_2)) - header_cols[diff_colnames] <- - paste(header_cols_2[diff_colnames], header_cols[diff_colnames]) - - empty_colnames <- which(is.na(header_cols)) - header_cols[empty_colnames] <- header_cols_2[empty_colnames] - } - - # make.names is necessary to avoid duplicate column names - # special chars are all removed by harmonize column names later - colnames(df_patient) <- make.names(header_cols, unique = T) - - # delete columns without a header (=NA) - df_patient <- df_patient[, !is.na(colnames(df_patient))] - - # removes duplicate columns that appear due to merged cells (e.g. insulin regimen) - # there is a really strange case when unmerging cells can lead to two insulin regime columns - # but with different data! For now just remove the second (hidden) column - insulin_regimen_cols <- grep("Insulin", colnames(df_patient)) - if (length(insulin_regimen_cols) > 1) { - # take all columns except the indices from grep beginning at second pos - df_patient <- df_patient[, -insulin_regimen_cols[-1]] - } - # df_patient <- df_patient %>% distinct() - # remove empty rows with only NA - df_patient <- - df_patient[rowSums(is.na(df_patient)) != ncol(df_patient), ] - - if ("NA" %in% names(df_patient)) { - df_patient <- df_patient %>% dplyr::select(-`NA`) - } - - df_patient -} - - -#' Harmonize patient data column names. -#' -#' @description -#' Imports the patient df, cleans it and matches it against -#' column synonyms to unify column names -#' -#' @param patient_df data.frame holding the patient data of the month sheet of a tracker. -#' @param columns_synonyms data.frame with synonyms for tracker variables. -#' -#' @return data.frame with harmonized column names. -#' @export -harmonize_patient_data_columns <- - function(patient_df, columns_synonyms) { - patient_df <- patient_df[!is.na(names(patient_df))] - - colnames(patient_df) <- sanitize_str(colnames(patient_df)) - synonym_headers <- sanitize_str(columns_synonyms$tracker_name) - - colnames_found <- - match(colnames(patient_df), synonym_headers, nomatch = 0) - colnames(patient_df)[colnames(patient_df) %in% synonym_headers] <- - columns_synonyms$variable_name[colnames_found] - - mismatching_column_ids <- which(colnames_found == 0) - if (length(mismatching_column_ids) > 0) { - logWarn( - log_to_json( - message = "Non-matching column names found: {values['col_names']}.", - values = list( - col_names = colnames(patient_df)[mismatching_column_ids] - ), - script = "script1", - file = "helper_read_patient_data.R", - functionName = "harmonize_patient_data_columns", - warningCode = "invalid_tracker" - ) - ) - } - - patient_df - } diff --git a/R/script1_process_patient_data.R b/R/script1_process_patient_data.R deleted file mode 100644 index ec5b6b1..0000000 --- a/R/script1_process_patient_data.R +++ /dev/null @@ -1,46 +0,0 @@ -#' @title Process tracker patient data -#' -#' @description -#' This function reads the patient data from a CSV file, adds a file name column, -#' and exports the data as a parquet file. -#' -#' @param tracker_name The name of the tracker. -#' @param tracker_data_file The path to the tracker data file. -#' @param output_root The root directory of the output folder. -#' @param synonyms_patient A named list of synonyms for the patient data columns. -#' -#' @export -process_tracker_patient_data <- - function(tracker_name, - tracker_data_file, - output_root, - synonyms_patient) { - df_patient_raw <- - reading_patient_data( - tracker_data_file = tracker_data_file, - columns_synonyms = synonyms_patient - ) - - df_patient_raw <- df_patient_raw %>% dplyr::mutate(file_name = tracker_name) - - # instead of clinic_code and country_code, we extract clinic_id from parent folder - # and join with static clinic data later in the database - df_patient_raw$clinic_id <- basename(dirname(tracker_data_file)) - - logInfo( - log_to_json( - message = "df patient data dim: {values['dim']}.", - values = list(dim = dim(df_patient_raw)), - script = "script1", - file = "script1_process_patient_data.R", - functionName = "process_tracker_patient_data" - ) - ) - - export_data_as_parquet( - data = df_patient_raw, - filename = tracker_name, - output_root = output_root, - suffix = paste0("_patient_raw") - ) - } diff --git a/R/script1_process_product_data.R b/R/script1_process_product_data.R deleted file mode 100644 index b0fe4cb..0000000 --- a/R/script1_process_product_data.R +++ /dev/null @@ -1,57 +0,0 @@ -#' @title Process tracker product data -#' -#' @description -#' This function reads the product data from an Excel file, -#' fills missing values, and exports the data as a parquet file. -#' -#' @param tracker_name The name of the tracker. -#' @param tracker_data_file The path to the tracker data file. -#' @param output_root The root directory of the output folder. -#' @param synonyms_product A named list of synonyms for the product data columns. -#' -#' @export -process_tracker_product_data <- - function(tracker_name, - tracker_data_file, - output_root, - synonyms_product) { - df_raw_product <- - reading_product_data_step1( - tracker_data_file = tracker_data_file, - columns_synonyms = synonyms_product - ) - - if (!is.null(df_raw_product)) { - df_raw_product <- df_raw_product %>% dplyr::mutate(file_name = tracker_name) - - df_raw_product$clinic_id <- basename(dirname(tracker_data_file)) - - logInfo( - log_to_json( - message = "df_raw_product dim: {values['dim']}.", - values = list(dim = dim(df_raw_product)), - script = "script1", - file = "script1_process_product_data.R", - functionName = "process_tracker_product_data" - ) - ) - - # product set sensitive column to NA and add tracker file name as a column - export_data_as_parquet( - data = df_raw_product, - filename = tracker_name, - output_root = output_root, - suffix = "_product_raw" - ) - } else { - logWarn( - log_to_json( - message = "Empty product data!", - script = "script1", - file = "script1_process_product_data.R", - functionName = "process_tracker_product_data", - warningCode = "empty_product_data" - ) - ) - } - } diff --git a/R/script1_process_tracker_file.R b/R/script1_process_tracker_file.R deleted file mode 100644 index 569fd1c..0000000 --- a/R/script1_process_tracker_file.R +++ /dev/null @@ -1,101 +0,0 @@ -#' Process tracker file. -#' -#' @param paths list of paths to directories. -#' @param tracker_file Name of the tracker file. -#' @param tracker_name Basename of the tracker file. -#' @param synonyms list of synonyms for patient and product data. -#' -#' @export -process_tracker_file <- function(paths, tracker_file, tracker_name, synonyms) { - tracker_data_file <- - file.path(paths$tracker_root, tracker_file) - - logInfo( - log_to_json( - "Current file: {values['file']}.", - values = list(file = tracker_name), - script = "script1", - file = "script1_process_tracker_file.R", - functionName = "process_tracker_file" - ) - ) - - logfile <- paste0(tracker_name, "_", "patient") - with_file_logger(logfile, - { - tryCatch( - process_tracker_patient_data( - tracker_name = tracker_name, - tracker_data_file = tracker_data_file, - output_root = paths$patient_data_raw, - synonyms_patient = synonyms$patient - ), - error = function(e) { - logError( - log_to_json( - "Could not process patient data. Error = {values['e']}.", - values = list(e = e$message), - script = "script1", - file = "script1_process_patient_data.R", - errorCode = "critical_abort", - functionName = "process_tracker_patient_data" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - "Could not process patient data. Warning = {values['w']}.", - values = list(w = w$message), - script = "script1", - file = "script1_process_patient_data.R", - warningCode = "critical_abort", - functionName = "process_tracker_patient_data" - ) - ) - } - ) - }, - output_root = paths$output_root - ) - - logfile <- paste0(tracker_name, "_", "product") - - with_file_logger(logfile, - { - tryCatch( - process_tracker_product_data( - tracker_name = tracker_name, - tracker_data_file = tracker_data_file, - output_root = paths$product_data_raw, - synonyms_product = synonyms$product - ), - error = function(e) { - logError( - log_to_json( - "Could not process product data. Error = {values['e']}.", - values = list(e = e$message), - script = "script1", - file = "script1_process_product_data.R", - errorCode = "critical_abort", - functionName = "process_tracker_product_data" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - "Could not process product data. Warning = {values['w']}.", - values = list(w = w$message), - script = "script1", - file = "script1_process_product_data.R", - warningCode = "critical_abort", - functionName = "process_tracker_product_data" - ) - ) - } - ) - }, - output_root = paths$output_root - ) -} diff --git a/R/script1_read_patient_data.R b/R/script1_read_patient_data.R deleted file mode 100644 index 4314f48..0000000 --- a/R/script1_read_patient_data.R +++ /dev/null @@ -1,163 +0,0 @@ -#' @title Reading patient data from the tracker data file -#' -#' @description -#' This function reads the patient data from the tracker data file, -#' harmonizes the columns, and returns the data as a data frame. -#' -#' @param tracker_data_file The path to the tracker data file. -#' @param columns_synonyms A named list of column synonyms. -#' -#' @return A data frame with the patient data. -#' -#' @export -reading_patient_data <- - function(tracker_data_file, columns_synonyms) { - sheet_list <- readxl::excel_sheets(tracker_data_file) - testit::assert(length(sheet_list) > 0) - - logInfo( - log_to_json( - message = "Found {values['len']} sheets: {values['sheets']}.", - values = list(len = length(sheet_list), sheets = sheet_list), - script = "script1", - file = "script1_read_patient_data.R", - functionName = "reading_patient_data" - ) - ) - - month_list <- - sheet_list[na.omit(pmatch(month.abb, sheet_list))] - testit::assert(length(month_list) > 0) - - logInfo( - log_to_json( - message = "Found {values['len']} month sheets: {values['months']}.", - values = list(len = length(month_list), months = month_list), - script = "script1", - file = "script1_read_patient_data.R", - functionName = "reading_patient_data" - ) - ) - - # Extract year - year <- get_tracker_year(tracker_data_file, month_list) - logInfo( - log_to_json( - message = "Tracker year = {values['year']}.", - values = list(year = year), - script = "script1", - file = "script1_read_patient_data.R", - functionName = "reading_patient_data" - ) - ) - - testit::assert(year >= 2017 & year <= 2026) - - tidy_tracker_list <- NULL - - for (curr_sheet in month_list) { - df_patient <- extract_patient_data(tracker_data_file, curr_sheet, year) - testit::assert(nrow(df_patient) > 0) - - df_patient <- - harmonize_patient_data_columns(df_patient, columns_synonyms) - testit::assert("patient_id" %in% colnames(df_patient)) - - # -- if we have duplicate columns, merge them - if (anyDuplicated(colnames(df_patient)) > 0) { - duplicated_cols <- colnames(df_patient) %>% - table() %>% - dplyr::as_tibble() %>% - dplyr::filter(n > 1) %>% - dplyr::select(1) %>% - dplyr::pull() - for (col in duplicated_cols) { - mask <- colnames(df_patient) == col - merged_col <- df_patient[mask] %>% - tidyr::unite(!!col, sep = ",") - df_patient <- df_patient[!mask] - df_patient <- df_patient %>% - tibble::add_column(!!col := dplyr::pull(merged_col), .name_repair = "minimal") - } - } - - # add the current sheet name, month name and year to the patient data frame - df_patient <- df_patient %>% - dplyr::mutate( - sheet_name = curr_sheet, - tracker_month = match(substr(curr_sheet, 1, 3), month.abb), - tracker_year = year - ) - - tidy_tracker_list[[curr_sheet]] <- df_patient - } - - df_raw <- dplyr::bind_rows(tidy_tracker_list) - - - # filter all rows with no patient id or patient name - df_raw <- df_raw %>% - dplyr::filter(!(is.na(patient_id) & is.na(name))) %>% - dplyr::filter(!(patient_id == "0" & name == "0")) - - if ("Patient List" %in% sheet_list) { - patient_list <- extract_patient_data( - tracker_data_file, - "Patient List", - year - ) - testit::assert(nrow(patient_list) > 0) - patient_list <- harmonize_patient_data_columns( - patient_list, - columns_synonyms - ) - testit::assert("patient_id" %in% colnames(patient_list)) - - patient_list <- patient_list %>% - dplyr::filter(!(is.na(patient_id) & is.na(name))) %>% - dplyr::filter(!(patient_id == "0" & name == "0")) - - df_raw <- dplyr::left_join( - df_raw %>% dplyr::select(-any_of(c("hba1c_baseline"))), - patient_list %>% - dplyr::select(-any_of(c( - "name" - ))), - by = "patient_id", - relationship = "many-to-one", - suffix = c(".monthly", ".static") - ) - } - - if ("Annual" %in% sheet_list) { - annual_data <- extract_patient_data( - tracker_data_file, - "Annual", - year - ) - testit::assert(nrow(annual_data) > 0) - annual_data <- harmonize_patient_data_columns( - annual_data, - columns_synonyms - ) - testit::assert("patient_id" %in% colnames(annual_data)) - - annual_data <- annual_data %>% - dplyr::filter(!(is.na(patient_id) & is.na(name))) %>% - dplyr::filter(!(patient_id == "0" & name == "0")) - - df_raw <- dplyr::left_join( - df_raw, - annual_data %>% - dplyr::select(-any_of(c( - "status", - "name" - ))), - by = "patient_id", - relationship = "many-to-one", - suffix = c(".monthly", ".annual") - ) - } - - df_raw - } diff --git a/R/script2_helper_dates.R b/R/script2_helper_dates.R deleted file mode 100644 index 853f318..0000000 --- a/R/script2_helper_dates.R +++ /dev/null @@ -1,53 +0,0 @@ -#' @title Try to handle invalid dates. -#' -#' @description -#' For each value of the character vector, this function uses the -#' lubridate parse functions to find common formats -#' -#' -#' @param df data frame with patient data. -#' @param colname column that has invalid dates. -parse_invalid_dates <- function(df, colname) { - orders <- c("dmy", "dmY", "by", "bY") - df <- df %>% dplyr::mutate(across(!!colname, function(x) { - lubridate::parse_date_time(x, orders) - })) -} - - -numbers_only <- function(x) !grepl("\\D", x) - - -parse_date_string <- function(text) { - parsed_date <- as.Date(lubridate::parse_date_time(text, orders = c("ymd", "dmy", "my"), quiet = T)) - - if (is.na(parsed_date) && numbers_only(text)) { - parsed_date <- - try(as.Date(as.numeric(text), origin = "1899-12-30"), silent = TRUE) - } - - if (class(parsed_date) == "try-error" || is.na(parsed_date)) parsed_date <- as.Date("9999-01-01") - - if (lubridate::year(parsed_date) < 17) { - lubridate::year(parsed_date) <- lubridate::year(parsed_date) + 2000 - } - - return(parsed_date) -} - - -fix_date_cols <- function(d) { - if (!is.na(d)) { - d <- gsub(".*[(]|[)].*", "", d) - - d <- parse_date_string(d) - - if (is.na(d)) d <- as.Date("9999-01-01") - } else { - # without this, so with NA instead, dplyr::mutate is throwing an error: - # "must return compatible vectors across groups". - d <- NA_Date_ - } - - return(d) -} diff --git a/R/script2_helper_patient_data_fix.R b/R/script2_helper_patient_data_fix.R deleted file mode 100644 index 278ab1c..0000000 --- a/R/script2_helper_patient_data_fix.R +++ /dev/null @@ -1,751 +0,0 @@ -# DESCRIPTION ------------------------------------------------------------- - -# This function cleans the output of the "tidy" function by reformatting certain columns. -# Basic functions are created and then applied to each piece of data available in one row. - -#### Base Functions #### - -#' @title Try to convert character x into a value of target type. -#' -#' @description -#' Use cast_fnc to cast x to target type. -#' If that cast fails, output error_val instead. -#' If x is already NA, output NA. -#' -#' @param x character value. -#' @param cast_fnc function used to convert character values into target type. -#' @param error_val error value to use when cast_fnc fails. -#' @param col_name column name if used with across. Optional, defaults to "". -#' @param id patient id, optional, defaults to "". -#' -#' @return converted value -#' @export -convert_to <- function(x, cast_fnc, error_val, col_name = "", id = "") { - x <- tryCatch( - cast_fnc(x), - error = function(e) { - logError( - log_to_json( - message = "Error while trying to convert value {values['x']} in column {values['col_name']} for patient: {values['id']}. Replacing with error val {values['err_val']}.", - values = list(x = x, col_name = col_name, id = id, err_val = error_val), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "convert_to", - errorCode = "function_call" - ) - ) - x <- error_val - }, - warning = function(w) { - logWarn( - log_to_json( - message = "Warning while trying to convert value {values['x']} in column {values['col_name']} for patient: {values['id']}. Replacing with error val {values['err_val']}.", - values = list(x = x, col_name = col_name, id = id, err_val = error_val), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "convert_to", - errorCode = "function_call" - ) - ) - x <- error_val - } - ) - - x -} - - -#' @title If x is outside its allowed range, return error value. -#' -#' @param x numeric value to check. -#' @param min lower bound. -#' @param max upper bound. -#' @param col_name column name if used with mutate/across. -#' -#' @return either x or error value. -#' @export -cut_numeric_value <- function(x, - min, - max, - col_name = "") { - if (is.na(x) || x == ERROR_VAL_NUMERIC) { - return(x) - } - - if (x < min || x > max) { - logWarn( - log_to_json( - message = "Found invalid value {values['x']} for column {values['col_name']} outside [{values['min']}, {values['max']}]. Value was replaced with {values['error_val']}.", - values = list(x = x, col_name = col_name, min = min, max = max, error_val = ERROR_VAL_NUMERIC), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "cut_numeric_value", - warningCode = "invalid_value" - ) - ) - x <- ERROR_VAL_NUMERIC - } - - x -} - - -#' @title Correct decimal sign. -#' -#' @param x character representation of a number. -#' -#' @return character representation with a correct decimal sign. -#' @export -correct_decimal_sign <- function(x) { - stringr::str_replace(x, ",", ".") -} - - -#' @title Split a column value into value and date. -#' -#' @description -#' Assumes that the measurement entry is in the format "8.53 (28/8/2017)". -#' Uses regex to split this string into the value part and the date part, -#' for example in "8.53 " and "28/8/2017". -#' -#' @param df data frame with patient data. -#' @param colname column to split into value and date. -#' -#' @return data frame with the original column and a new colname_date column. -extract_date_from_measurement <- - function(df, colname) { - df <- df %>% - tidyr::separate_wider_regex( - !!colname, - c( - magrittr::set_names(".*", colname), - "[(]", - magrittr::set_names(".*?", paste0(colname, "_date")), - "[)]?" - ), - too_few = "align_start" - ) - - if (colname %in% c("fbg_updated_mmol", "fbg_updated_mg")) { - df <- df %>% - dplyr::rename_with(~ stringr::str_replace(.x, "_mmol_date|_mg_date", "_date")) - } - - df - } - - -#' @title WIP: Convert dates that are stored as 5-digit number from Excel to date format -#' -#' @param date date as five number digit. -#' -#' @seealso https://stackoverflow.com/a/50274507: depending on whether read in -#' file was created on Mac or Windows, the origin date might need to be adjusted. -#' -#' @return Standardized date in a character format. -#' -#' @examples -#' \dontrun{ -#' fix_digit_date("44939") # should return "2023-01-13" -#' } -fix_digit_date <- - function(date) { - if (is.na(date)) { - return(lubridate::NA_Date_) - } - - if (stringr::str_detect(string = date, pattern = "^[[:digit:]]{5}")) { - date <- as.character(openxlsx::convertToDate(date)) - } - - date - } - - -#' @title Try to handle invalid dates. -#' -#' @description -#' For each value of the character vector, this function uses the -#' lubridate parse functions to find common formats -#' -#' @param date character date value. -#' -#' @return Either a correctly parsed date or NA. -parse_dates <- function(date) { - if (is.na(date)) { - return(lubridate::NA_Date_) - } - - parsed_date <- suppressWarnings(lubridate::as_date(date)) - - if (is.na(parsed_date)) { - orders <- c("dmy", "dmY", "dbY", "by", "bY", "mY", "my", "y") - logWarn( - log_to_json( - message = "Could not parse date value {values['date']}. Trying to parse with lubridate::parse_date_time and orders = {values['orders']}.", - values = list(date = date, orders = orders), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "parse_dates", - warningCode = "invalid_value" - ) - ) - - if (grepl("[[:alpha:]]{4}", date)) { - date <- sub("([[:alpha:]]{3})[[:alpha:]]", "\\1", date) - } - parsed_date <- lubridate::parse_date_time(date, orders) - parsed_date <- as.Date(parsed_date) - } - - parsed_date -} - - -#' @title Check value against list of allowed values. -#' -#' @description -#' Invalid values are logged and depending on the replace invalid flag -#' also replaced with an error value. When checking for the validity of values -#' the case of the string is not considered nor are spaces and non-alphanumeric -#' characters. If the value matches an entry in the allowed_values list -#' that match is returned as the result. -#' -#' @param x character value to check. -#' @param valid_values list of valid options. -#' @param id patient id. -#' @param replace_invalid A flag controlling whether to also replace invalid . -#' @param error_val value that replaces invalid values. -#' @param col column name when used with mutate/across. -#' -#' @return Either x or "Other". -check_allowed_values <- function(x, valid_values, id, replace_invalid = TRUE, error_val = "Undefined", col = "") { - if (is.na(x) || x == "") { - return(NA_character_) - } - - valid_value_mapping <- setNames(as.list(valid_values), sanitize_str(valid_values)) - - if (!sanitize_str(x) %in% names(valid_value_mapping)) { - logWarn( - log_to_json( - message = "Patient {values['id']}: Value {values['x']} for column {values['col']} is not in the list of allowed values. ", - values = list(id = id, x = x, col = col), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "check_allowed_values", - warningCode = "invalid_value" - ) - ) - - if (replace_invalid) { - logWarn( - log_to_json( - message = "Replacing {values['x']} with {values['error_val']}.", - values = list(x = x, error_val = error_val), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "check_allowed_values", - warningCode = "invalid_value" - ) - ) - - x <- error_val - } - - return(x) - } - - valid_value_mapping[[sanitize_str(x)]] -} - -parse_character_cleaning_pipeline <- function(column_name, column_config) { - pipeline <- rlang::expr( - !!rlang::parse_expr(column_name) - ) - for (step in column_config$steps) { - pipeline <- rlang::expr(!!pipeline %>% !!parse_step(column_name, step)) - } - - pipeline -} - -parse_step <- function(column_name, step) { - switch(step$type, - allowed_values = parse_allowed_value_check(column_name, step), - basic_function = rlang::expr((!!rlang::parse_expr(step$function_name))()) - ) -} - -parse_allowed_value_check <- function(column_name, check_details) { - args <- rlang::exprs( - valid_values = !!check_details$allowed_values, - id = patient_id, - replace_invalid = !!check_details$replace_invalid, - col = !!column_name - ) - if ("error_value" %in% check_details) { - args$error_val <- rlang::parse_expr(check_details$error_value) - } - rlang::expr(check_allowed_values(!!!args)) -} - -parse_character_cleaning_config <- function(config) { - allowed_value_expr <- list() - for (column in names(config)) { - allowed_value_expr[[column]] <- parse_character_cleaning_pipeline(column, config[[column]]) - } - - allowed_value_expr -} - - -#### age #### - -#' @title Fix age column. -#' -#' @description -#' Based on date of birth calculate the current age of the patient. -#' If this age is different than the given patient age, we take the calculated one. -#' -#' -#' @param age patient age. -#' @param dob patient date of birth. -#' @param tracker_year year the tracker was filled out. -#' @param tracker_month month the tracker was filled out. -#' @param id patient id. -#' -#' @return fixed age. -#' @export -fix_age <- function(age, dob, tracker_year, tracker_month, id) { - calc_age <- age - - if (!is.na(dob)) { - dob <- as.Date(dob) - calc_age <- tracker_year - lubridate::year(dob) - delta_month <- tracker_month - lubridate::month(dob) - if (delta_month < 0) { - calc_age <- calc_age - 1 - } - - if (is.na(age)) { - logWarn( - log_to_json( - message = "Patient {values['id']}: age is missing. Using calculated age {values['calc_age']} instead of original age.", - values = list(id = id, calc_age = calc_age), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "fix_age", - warningCode = "missing_value" - ) - ) - } else { - if (calc_age != age) { - logWarn( - log_to_json( - message = "Patient {values['id']}: age {values['age']} is different from calculated age {values['calc_age']}. Using calculated age instead of original age.", - values = list(id = id, age = age, calc_age = calc_age), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "fix_age", - warningCode = "invalid_value" - ) - ) - } - } - - if (calc_age < 0) { - logWarn( - log_to_json( - message = "Patient {values['id']}: calculated age is negative. Please check this manually. Using error value instead.", - values = list(id = id), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "fix_age", - warningCode = "invalid_value" - ) - ) - - calc_age <- ERROR_VAL_NUMERIC - } - } - - calc_age -} - - -#### bmi #### - -#' @title Calculate bmi value based on weight and height. -#' -#' @description -#' Formula: weight (kg) / height^2 (meters^2) -#' bmi is set to NA if either weight or height are NA. -#' bmi is set to error val if either weight or height are error val. -#' -#' @param weight numeric weight value. -#' @param height numeric height value. -#' @param id patient id. -#' -#' @return calculated bmi value. -#' @export -fix_bmi <- function(weight, height, id) { - bmi <- dplyr::case_when( - is.na(weight) || is.na(height) ~ NA_real_, - weight == ERROR_VAL_NUMERIC || height == ERROR_VAL_NUMERIC ~ ERROR_VAL_NUMERIC, - .default = weight / height^2 - ) - - - if (!is.na(weight) && weight == ERROR_VAL_CHARACTER) { - logWarn( - log_to_json( - message = "Patient {values['id']}: the weight is out of bounds.", - values = list(id = id), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "fix_bmi", - warningCode = "invalid_value" - ) - ) - } - - if (!is.na(height) && height == ERROR_VAL_CHARACTER) { - logWarn( - log_to_json( - message = "Patient {values['id']}: the height is out of bounds.", - values = list(id = id), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "fix_bmi", - warningCode = "invalid_value" - ) - ) - } - - bmi -} - - -#### sex #### - -#' @title Replace sex synonyms with either M, F or Other. -#' -#' @param sex patient gender. -#' @param id patient id. -#' -#' @return Either M, F or Other. -#' @export -fix_sex <- function(sex, id) { - synonyms_female <- c("female", "girl", "woman", "fem", "feminine", "f") - synonyms_male <- c("male", "boy", "man", "masculine", "m") - - fixed_sex <- dplyr::case_when( - is.na(sex) | sex == "" ~ NA_character_, - tolower(sex) %in% synonyms_female ~ "F", - tolower(sex) %in% synonyms_male ~ "M", - .default = ERROR_VAL_CHARACTER - ) - - if (!is.na(fixed_sex) && fixed_sex == ERROR_VAL_CHARACTER) { - logWarn( - log_to_json( - message = "Patient {values['id']}: sex {values['sex']} is not in the list of synonyms. Replacing it with {values['fixed_sex']}.", - values = list(id = id, sex = sex, fixed_sex = fixed_sex), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "fix_sex", - warningCode = "invalid_value" - ) - ) - } - - fixed_sex -} - - -#### t1d_diagnosis_age #### - -#' @title Try to replace text descriptions for age with proper numerical values. -#' -#' @param t1d_diagnosis_age patient age when diagnosed with T1D. -#' @param id patient id -#' -#' @return Corrected value with text replacement. -#' @export -fix_t1d_diagnosis_age <- function(t1d_diagnosis_age, id) { - if (is.na(t1d_diagnosis_age)) { - return(t1d_diagnosis_age) - } - - age_corrected <- dplyr::case_when( - grepl("birth|born|month", tolower(t1d_diagnosis_age)) ~ "0", - grepl("y", tolower(t1d_diagnosis_age)) ~ extract_year_from_age(t1d_diagnosis_age), - .default = t1d_diagnosis_age - ) - - age_corrected -} - - -#' @title Extract year from age descriptions like "10y5m" -#' -#' @description -#' Example: "1y5m" -> "1" -#' -#' @param age text representation of diagnosis age -#' -#' @return year of given age as character. -#' @export -extract_year_from_age <- function(age) { - if (is.na(age)) { - return(age) - } - - unlist(strsplit(age, "y", fixed = T))[1] -} - - -#### fbg #### - -# @Description: Makes sure FBG values are in mmol/L -# @fbg: Any fbg value in mg/dL or mmol/L -# @country_id: ID of the country where patient values were taken -# @hospital_id: ID of the hospital where patient values were taken -# @Output: FBG value in mmol/L or NA if not matched -transform_fbg_in_mmol <- function(fbg, country_id, hospital_id) { - factor_mmol_in_mg <- 18.02 - measure_unit <- - assign_fbg_unit_per_hospital( - country_id = country_id, - hospital_id = hospital_id - ) - - # If not unit "mmol/L" is assumed - fbg_mmol <- dplyr::case_when( - measure_unit == "mg/dL" ~ fbg / factor_mmol_in_mg, - measure_unit == "mmol/L" ~ fbg, - is.na(measure_unit) ~ fbg, - TRUE ~ NA_real_ - ) - - as.numeric(fbg_mmol) -} - - -#' @title Replace common textual descriptions with numeric values -#' -#' @param fbg character value. -#' -#' @return fbg transformed value. -#' @export -fix_fbg <- function(fbg) { - if (is.na(fbg) || fbg == "") { - return(NA_character_) - } - - # Source for levels: https://www.cdc.gov/diabetes/basics/getting-tested.html - fbg <- dplyr::case_when( - grepl("high|bad|hi", tolower(fbg)) ~ "200", - grepl("med|medium", tolower(fbg)) ~ "170", - grepl("low|good|okay", tolower(fbg)) ~ "140", - TRUE ~ fbg - ) %>% - gsub(pattern = "(DKA)", replacement = "", fixed = T) %>% - trimws() - - fbg -} - - -#### testing_fqr_pday #### - -#' @title Fix problems with ranges and decimal numbers in testing_frequency. -#' -#' @param test_frq character value for testing frequency -#' @param id patient id -#' -#' @return validated test_frq value -#' @export -fix_testing_frequency <- function(test_frq, id) { - if (is.na(test_frq) || test_frq == "") { - return(NA_character_) - } - - if (grepl("-", test_frq, fixed = TRUE)) { - logWarn( - log_to_json( - message = "Patient {values['id']}: Found a range for testing_frequency. Replacing it with the mean.", - values = list(id = id), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "fix_testing_frequency", - warningCode = "invalid_value" - ) - ) - - test_frq <- try(as.character(replace_range_with_mean(test_frq), silent = TRUE)) - } - - test_frq -} - - -#' @title Replace a range with mean. -#' -#' @description -#' A range like "0-2" will be replaced with the mean(c(0,2)). -#' -#' @param x range as text.. -#' -#' @return mean of that range -replace_range_with_mean <- function(x) { - mean(as.numeric(unlist(stringr::str_split(x, "-")))) -} - - -#### blood_pressure_mmhg #### - -#' @title Separate single blood pressure value into sys and dias values. -#' -#' @param df data frame with patient data. -#' -#' @return data frame with two new columns: blood_pressure_sys_mmhg and blood_pressure_dias_mmhg. -split_bp_in_sys_and_dias <- function(df) { - df <- df %>% - dplyr::mutate( - blood_pressure_mmhg = dplyr::case_when( - stringr::str_detect(blood_pressure_mmhg, "/", negate = T) ~ paste(ERROR_VAL_NUMERIC, ERROR_VAL_NUMERIC, sep = "/"), - TRUE ~ blood_pressure_mmhg - ) - ) - - if (paste(ERROR_VAL_NUMERIC, ERROR_VAL_NUMERIC, sep = "/") %in% df$blood_pressure_mmhg) { - logWarn( - log_to_json( - message = "Found invalid values for column blood_pressure_mmhg that do not follow the format X/Y. Values were replaced with {values['err_val']}.", - values = list(error_val = ERROR_VAL_NUMERIC), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "split_bp_in_sys_and_dias", - warningCode = "invalid_value" - ) - ) - } - - df <- df %>% - tidyr::separate_wider_delim( - cols = blood_pressure_mmhg, - delim = "/", - names = c("blood_pressure_sys_mmhg", "blood_pressure_dias_mmhg"), - ) - - - df -} - - -#### height #### - -transform_cm_to_m <- function(height) { - if (is.na(height) || height == ERROR_VAL_NUMERIC) { - return(height) - } - - height <- as.numeric(height) - height <- ifelse(height > 50, - height / 100, - height - ) - - height -} - - -#### id #### -#' @title check patient id and truncate it if necessary. -#' -#' @param id patient id. -#' -#' @return Either original id, truncated id, or error value. -#' @export -fix_id <- function(id) { - if (is.na(id)) { - return(NA_character_) - } - - id <- stringr::str_replace(id, "-", "_") - - if (!grepl("^[[:upper:]]{2}_[[:upper:]]{2}[[:digit:]]{3}$", id)) { - logWarn( - log_to_json( - message = "Patient {values['id']}: id cannot be matched to a 8 letter alpha numeric code like XX_YY001.", - values = list(id = id), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "fix_id", - warningCode = "invalid_value" - ) - ) - - if (stringr::str_length(id) > 8) { - logWarn( - log_to_json( - message = "Patient {values['id']}: id was truncated because it is longer than 8 characters.", - values = list(id = id), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "fix_id", - warningCode = "invalid_value" - ) - ) - - id <- stringr::str_sub(id, 1, 8) - } else { - logError( - log_to_json( - message = "Patient {values['id']}: id is not valid. Replacing with error value {values['err_val']}.", - values = list(id = id, err_val = ERROR_VAL_CHARACTER), - script = "script2", - file = "script2_helper_patient_data_fix.R", - functionName = "fix_id", - errorCode = "invalid_value" - ) - ) - - id <- ERROR_VAL_CHARACTER - } - } - - id -} - - -#### insulin regimen #### - -#' @title Extract the part of a string before a comma -#' -#' @param string The input string -#' -#' @return The extracted part. -extract_first_raw_regimen <- function(string) { - stringr::str_extract(string, "[^,]*") -} - -#' @title Maps Insulin regiment with allowed values. -#' -#' @description -#' The mapping is done using regular expressions. -#' -#' -#' @param raw_input The string that needs to be mapped -#' -#' @return The mapping result -extract_regimen <- function(raw_input) { - output <- sub("^.*basal.*$", "Basal-bolus (MDI)", raw_input, ignore.case = TRUE) - output <- sub("^.*premixed.*$", "Premixed 30/70 BD", output, ignore.case = TRUE) - output <- sub("^.*self-mixed.*$", "Self-mixed BD", output, ignore.case = TRUE) - output <- sub("^.*conventional.*$", "Modified conventional TID", output, ignore.case = TRUE) - - output -} diff --git a/R/script2_process_patient_data.R b/R/script2_process_patient_data.R deleted file mode 100644 index 7e44df3..0000000 --- a/R/script2_process_patient_data.R +++ /dev/null @@ -1,357 +0,0 @@ -#' @title Process raw patient data -#' -#' @description -#' This function reads the raw patient data from a parquet file, cleans the data, -#' and exports the cleaned data as a parquet file. -#' -#' @param paths list of paths to the input and output directories. -#' @param patient_file name of the parquet file with the raw patient data. -#' @param patient_file_name base name of the parquet file with the raw patient data. -#' @param output_root root directory of the output folder. -#' -#' @export -process_raw_patient_file <- function(paths, patient_file, patient_file_name, output_root) { - patient_file_path <- - file.path(paths$output_root, patient_file) - - allowed_provinces <- get_allowed_provinces() - - df_patient_raw <- arrow::read_parquet(patient_file_path) - - # --- TRANSFORMATIONS --- - # data before 2019 had only one column for updated hba1c and fbg - # with date as part of the value - if (!"hba1c_updated_date" %in% colnames(df_patient_raw) && "hba1c_updated" %in% colnames(df_patient_raw)) { - logWarn( - log_to_json( - message = "Column {values['target']} not found. Trying to parse from {values['src']}.", - values = list(src = "hba1c_updated", target = "updated_hba1c_date"), - script = "script2", - file = "script2_process_patient_data.R", - functionName = "process_raw_patient_file", - warningCode = "missing_column" - ) - ) - df_patient_raw <- - extract_date_from_measurement(df_patient_raw, "hba1c_updated") - } - - if (!"fbg_updated_date" %in% colnames(df_patient_raw) && "fbg_updated_mg" %in% colnames(df_patient_raw)) { - logWarn( - log_to_json( - message = "Column {values['target']} not found. Trying to parse from {values['src']}.", - values = list(src = "fbg_updated_mg.", target = "updated_fbg_date"), - script = "script2", - file = "script2_process_patient_data.R", - functionName = "process_patient_file", - warningCode = "missing_column" - ) - ) - df_patient_raw <- - extract_date_from_measurement(df_patient_raw, "fbg_updated_mg") - } - - if (!"fbg_updated_date" %in% colnames(df_patient_raw) && "fbg_updated_mmol" %in% colnames(df_patient_raw)) { - logWarn( - log_to_json( - message = "Column {values['target']} not found. Trying to parse from {values['src']}.", - values = list(src = "fbg_updated_mmol.", target = "updated_fbg_date"), - script = "script2", - file = "script2_process_patient_data.R", - functionName = "process_raw_patient_file", - warningCode = "missing_column" - ) - ) - df_patient_raw <- - extract_date_from_measurement(df_patient_raw, "fbg_updated_mmol") - } - - # blood pressure is given as sys/dias value pair before 2024, - # so we split this column in two separate columns - if ("blood_pressure_mmhg" %in% colnames(df_patient_raw)) { - df_patient_raw <- split_bp_in_sys_and_dias(df_patient_raw) - } - - # The maximum value available for hba1c will be around 14% - 18%, - # depending on the equipment being used. - # If the reading is above the maximum available value the > sign is used - - # we would prefer to retain this character in the database as it is important for data analysis. - df_patient_raw <- df_patient_raw %>% - dplyr::mutate( - hba1c_baseline_exceeds = ifelse(grepl(">|<", hba1c_baseline), TRUE, FALSE), - hba1c_updated_exceeds = ifelse(grepl(">|<", hba1c_updated), TRUE, FALSE) - ) - - # Since 2024 the tracker habe three columns for human insulin and two columns for analog insulin - # The three columns for human insulin are human_insulin_pre_mixed, human_insulin_short_acting, human_insulin_intermediate_acting - # The two columns for analog insulin are analog_insulin_rapid_acting, analog_insulin_long_acting - # One patient can have multiple insulin regimens at the same time, but only the human insulin type or the analog insulin type - # Whenever the patient has a "Y" in one of the human insulin columns, the insulin type is set to "human insulin", otherwise to "analog insulin" - # Whenever there is a "Y" in one of the columns, the name of that column is added as value to the insulin_subtype column - if ("human_insulin_pre_mixed" %in% colnames(df_patient_raw)) { - df_patient_raw <- df_patient_raw %>% - dplyr::mutate( - insulin_type = ifelse( - human_insulin_pre_mixed == "Y" | - human_insulin_short_acting == "Y" | - human_insulin_intermediate_acting == "Y", - "human insulin", - "analog insulin" - ), - insulin_subtype = - ifelse(human_insulin_pre_mixed == "Y", "pre-mixed", "") %>% - paste(ifelse(human_insulin_short_acting == "Y", "short-acting", ""), sep = ",") %>% - paste(ifelse(human_insulin_intermediate_acting == "Y", "intermediate-acting", ""), sep = ",") %>% - paste(ifelse(analog_insulin_rapid_acting == "Y", "rapic-acting", ""), sep = ",") %>% - paste(ifelse(analog_insulin_long_acting == "Y", "long-acting", ""), sep = ",") %>% - stringr::str_split(",") %>% - purrr::map(~ .x[.x != ""]) %>% - purrr::map_chr(~ paste(.x, collapse = ",")) - ) - } - - # --- META SCHEMA --- - # meta schema has all final columns for the database - # along with their corresponding data types - # short type string for read_csv: - # iiinDccDcnnDnncnlnlDncDccDDDccccDccccciDciiiDn - schema <- tibble::tibble( - # complication_screening_date = lubridate::as_date(1), - # est_strips_pmonth = integer(), - # family_support_scale = character(), # TODO - # inactive_reason = character(), - # insulin_dosage = character(), - # meter_received_date = lubridate::as_date(1), # TODO - # remarks = character(), - # additional_support = character(), - age = integer(), - analog_insulin_long_acting = character(), - analog_insulin_rapid_acting = character(), - blood_pressure_dias_mmhg = integer(), - blood_pressure_sys_mmhg = integer(), - blood_pressure_updated = lubridate::as_date(1), - bmi = numeric(), - bmi_date = lubridate::as_date(1), - # clinic_code = character(), - clinic_id = character(), - clinic_visit = character(), - complication_screening_eye_exam_date = lubridate::as_date(1), - complication_screening_eye_exam_value = character(), - complication_screening_foot_exam_date = lubridate::as_date(1), - complication_screening_foot_exam_value = character(), - complication_screening_kidney_test_date = lubridate::as_date(1), - complication_screening_kidney_test_value = character(), - complication_screening_lipid_profile_cholesterol_value = character(), - complication_screening_lipid_profile_date = lubridate::as_date(1), - complication_screening_lipid_profile_hdl_mmol_value = numeric(), - complication_screening_lipid_profile_hdl_mg_value = numeric(), - complication_screening_lipid_profile_ldl_mmol_value = numeric(), - complication_screening_lipid_profile_ldl_mg_value = numeric(), - complication_screening_lipid_profile_triglycerides_value = numeric(), - complication_screening_remarks = character(), - complication_screening_thyroid_test_date = lubridate::as_date(1), - complication_screening_thyroid_test_ft4_pmol_value = numeric(), - complication_screening_thyroid_test_ft4_ng_value = numeric(), - complication_screening_thyroid_test_tsh_value = numeric(), - # country_code = character(), - dm_complication_eye = character(), - dm_complication_kidney = character(), - dm_complication_others = character(), - dm_complication_remarks = character(), - dob = lubridate::as_date(1), - edu_occ = character(), - edu_occ_updated = lubridate::as_date(1), - family_history = character(), - fbg_baseline_mg = numeric(), - fbg_baseline_mmol = numeric(), - fbg_updated_date = lubridate::as_date(1), - fbg_updated_mg = numeric(), - fbg_updated_mmol = numeric(), - file_name = character(), - hba1c_baseline = numeric(), - hba1c_baseline_exceeds = logical(), - hba1c_updated = numeric(), - hba1c_updated_exceeds = logical(), - hba1c_updated_date = lubridate::as_date(1), - height = numeric(), - hospitalisation_cause = character(), - hospitalisation_date = lubridate::as_date(1), - human_insulin_intermediate_acting = character(), - human_insulin_pre_mixed = character(), - human_insulin_short_acting = character(), - insulin_injections = numeric(), - insulin_regimen = character(), - insulin_total_units = numeric(), - insulin_type = character(), - insulin_subtype = character(), - last_clinic_visit_date = lubridate::as_date(1), - last_remote_followup_date = lubridate::as_date(1), - lost_date = lubridate::as_date(1), - name = character(), - observations = character(), - observations_category = character(), - other_issues = character(), - patient_consent = character(), - patient_id = character(), - province = character(), - recruitment_date = lubridate::as_date(1), - remote_followup = character(), - sex = character(), - sheet_name = character(), - status = character(), - status_out = character(), - support_level = character(), - t1d_diagnosis_age = integer(), - t1d_diagnosis_date = lubridate::as_date(1), - t1d_diagnosis_with_dka = character(), - testing_frequency = integer(), - tracker_date = lubridate::as_date(1), - tracker_month = integer(), - tracker_year = integer(), - # updated_2022_date = lubridate::as_date(1), - weight = numeric() - ) - - extra_cols <- colnames(df_patient_raw)[!colnames(df_patient_raw) %in% colnames(schema)] - logWarn( - log_to_json( - message = "Extra columns in patient dynamic data: {values['extra_cols']}.", - values = list(extra_cols = paste(extra_cols, collapse = ", ")), - script = "script2", - file = "script2_process_patient_data.R", - functionName = "process_raw_patient_file", - warningCode = "invalid_tracker" - ) - ) - - missing_cols <- - colnames(schema)[!colnames(schema) %in% colnames(df_patient_raw)] - logWarn( - log_to_json( - message = "Missing columns in patient dynamic data: {values['missing_cols']}.", - values = list(missing_cols = paste(missing_cols, collapse = ",")), - script = "script2", - file = "script2_process_patient_data.R", - functionName = "process_raw_patient_file", - warningCode = "invalid_tracker" - ) - ) - - # add all columns of schema to df_patient_raw - # keep all rows, only append missing cols - df_patient <- merge.default(df_patient_raw, schema, all.x = T) - df_patient <- df_patient[colnames(schema)] - - # the cleaning, fixing and validating happens in three major steps: - df_patient <- - df_patient %>% - dplyr::rowwise() %>% - # 1. handle known problems before converting to target type - dplyr::mutate( - # t1d_diagnosis_age = fix_t1d_diagnosis_age(t1d_diagnosis_age, patient_id), - hba1c_baseline = stringr::str_replace(hba1c_baseline, "<|>", ""), - hba1c_updated = stringr::str_replace(hba1c_updated, "<|>", ""), - # fbg_baseline_mg = fix_fbg(fbg_baseline_mg), - # fbg_baseline_mmol = fix_fbg(fbg_baseline_mmol), - fbg_updated_mg = fix_fbg(fbg_updated_mg), - fbg_updated_mmol = fix_fbg(fbg_updated_mmol), - testing_frequency = fix_testing_frequency(testing_frequency, patient_id), - analog_insulin_long_acting = sub("-", "N", analog_insulin_long_acting, fixed = TRUE), - analog_insulin_rapid_acting = sub("-", "N", analog_insulin_rapid_acting, fixed = TRUE), - human_insulin_intermediate_acting = sub("-", "N", human_insulin_intermediate_acting, fixed = TRUE), - human_insulin_pre_mixed = sub("-", "N", human_insulin_pre_mixed, fixed = TRUE), - human_insulin_short_acting = sub("-", "N", human_insulin_short_acting, fixed = TRUE) - ) %>% - # 2. convert the refined character columns into the target data type - dplyr::mutate( - dplyr::across( - schema %>% dplyr::select(tidyselect::where(is.numeric)) %>% names(), - \(x) convert_to(correct_decimal_sign(x), as.numeric, ERROR_VAL_NUMERIC, dplyr::cur_column(), id = patient_id) - ), - dplyr::across( - schema %>% dplyr::select(tidyselect::where(is.logical)) %>% names(), - \(x) convert_to(x, as.logical, FALSE, dplyr::cur_column(), id = patient_id) - ), - dplyr::across( - schema %>% dplyr::select(tidyselect::where(lubridate::is.Date)) %>% names(), - \(x) convert_to(fix_digit_date(x), parse_dates, as.Date(ERROR_VAL_DATE), dplyr::cur_column(), id = patient_id) - ), - dplyr::across( - schema %>% dplyr::select(tidyselect::where(is.integer)) %>% names(), - \(x) convert_to(x, function(x) as.integer(round(as.double(x))), ERROR_VAL_NUMERIC, dplyr::cur_column(), id = patient_id) - ) - ) %>% - # 3. fix remaining problems in the target data type - dplyr::mutate( - # height and weight are needed to calculate bmi - height = transform_cm_to_m(height) %>% - cut_numeric_value(min = 0, max = 2.3, col_name = "height"), - weight = cut_numeric_value(weight, min = 0, max = 200, col_name = "weight"), - bmi = fix_bmi(weight, height, patient_id) %>% - cut_numeric_value(min = 10, max = 80, "bmi"), - age = fix_age(age, dob, tracker_year, tracker_month, patient_id) %>% - cut_numeric_value(min = 0, max = 100, "age"), - sex = fix_sex(sex, patient_id), - hba1c_baseline = cut_numeric_value(hba1c_baseline, min = 0, max = 25, "hba1c_baseline"), - # https://www.cleveland19.com/story/1425584/ohio-man-holds-world-record-of-highest-blood-sugar/ - # fbg_baseline_mmol = cut_numeric_value(fbg_baseline_mmol, min = 0, max = 136.5, "fbg_baseline_mmol"), - # https://www.cleveland19.com/story/1425584/ohio-man-holds-world-record-of-highest-blood-sugar/ - fbg_updated_mmol = cut_numeric_value(fbg_updated_mmol, min = 0, max = 150, "fbg_updated_mmol"), - # blood_pressure_sys_mmhg = cut_numeric_value(blood_pressure_sys_mmhg, min = 20, max = 250, "blood_pressure_sys_mmhg"), - # blood_pressure_dias_mmhg = cut_numeric_value(blood_pressure_dias_mmhg, min = 20, max = 220, "blood_pressure_dias_mmhg"), - tracker_date = lubridate::ym(paste(tracker_year, tracker_month, sep = "-")), - !!!parse_character_cleaning_config(a4d:::config$cleaning), - # should be fixed last as other fix functions use id to log invalid rows! - patient_id = fix_id(patient_id) - ) %>% - dplyr::ungroup() - - # Formula to calculate mmol/l from mg/dl: mmol/l = mg/dl / 18 - # if (all(is.na(df_patient$fbg_baseline_mmol))) { - # df_patient <- df_patient %>% - # dplyr::mutate(fbg_baseline_mmol = dplyr::case_when( - # fbg_baseline_mg != ERROR_VAL_NUMERIC ~ fbg_baseline_mg / 18 - # )) - # } - if (all(is.na(df_patient$fbg_updated_mmol))) { - df_patient <- df_patient %>% - dplyr::mutate(fbg_updated_mmol = dplyr::case_when( - fbg_updated_mg != ERROR_VAL_NUMERIC ~ fbg_updated_mg / 18 - )) - } - - # Formula to calculate mg/dl from mmol/l: mg/dl = 18 × mmol/l - # if (all(is.na(df_patient$fbg_baseline_mg))) { - # df_patient <- df_patient %>% - # dplyr::mutate(fbg_baseline_mg = dplyr::case_when( - # fbg_baseline_mmol != ERROR_VAL_NUMERIC ~ fbg_baseline_mmol * 18 - # )) - # } - if (all(is.na(df_patient$fbg_updated_mg))) { - df_patient <- df_patient %>% - dplyr::mutate(fbg_updated_mg = dplyr::case_when( - fbg_updated_mmol != ERROR_VAL_NUMERIC ~ fbg_updated_mmol * 18 - )) - } - - # sort by year and month like it is in the tracker files - df_patient <- df_patient %>% - dplyr::arrange(tracker_date, patient_id) - - logInfo( - log_to_json( - message = "df_patient dim: {values['dim']}.", - values = list(dim = dim(df_patient)), - script = "script2", - file = "script2_process_patient_data.R", - functionName = "process_raw_patient_file", - ) - ) - - export_data_as_parquet( - data = df_patient, - filename = stringr::str_replace(patient_file_name, "_patient_raw", ""), - output_root = output_root, - suffix = "_patient_cleaned" - ) -} diff --git a/R/script2_process_product_data.R b/R/script2_process_product_data.R deleted file mode 100644 index 1b74b03..0000000 --- a/R/script2_process_product_data.R +++ /dev/null @@ -1,37 +0,0 @@ -#' @title Process raw product data -#' -#' @description -#' This function reads the raw product data from a parquet file, cleans the data, and exports the cleaned data as a parquet file. -#' -#' @param paths list of paths to the input and output directories. -#' @param product_file name of the raw product data file. -#' @param product_file_name base name of the raw product data file. -#' @param synonyms_product list of synonyms for the product data. -#' @param output_root The root directory of the output folder. -#' -#' @export -process_raw_product_file <- function(paths, product_file, product_file_name, synonyms_product, output_root) { - product_file_path <- - file.path(paths$tracker_root, product_file) - - df_product_raw <- arrow::read_parquet(product_file_path) - - df_product_raw <- reading_product_data_step2(df_product_raw, synonyms_product) - - logDebug( - log_to_json( - message = "df_product_raw dim: {values['dim']}.", - values = list(dim = dim(df_product_raw)), - script = "script2", - file = "script2_process_product_data.R", - functionName = "process_raw_product_file" - ) - ) - - export_data_as_parquet( - data = df_product_raw, - filename = stringr::str_replace(product_file_name, "_product_raw", ""), - output_root = output_root, - suffix = "_product_cleaned" - ) -} diff --git a/R/script2_sanitize_str.R b/R/script2_sanitize_str.R deleted file mode 100644 index fba8c6f..0000000 --- a/R/script2_sanitize_str.R +++ /dev/null @@ -1,25 +0,0 @@ -#' @title Sanitize column name -#' -#' @description -#' This function takes in a string, -#' sanitizes it by converting all letters to lower case, -#' removing all spaces and special characters except alphanumeric characters, -#' and then returns the sanitized string. -#' -#' @param text A text to be sanitized. -#' -#' @return The sanitized text. -#' -#' @examples -#' sanitize_str("John Doe's Column") # should return "johndoescolumn" -#' sanitize_str("Date 2022") # should return "date2022" -#' sanitize_str("My Awesome 1st Column!!") # should return "myawesome1stcolumn" -#' -#' @export -sanitize_str <- function(text) { - text <- text %>% - stringr::str_to_lower() %>% - stringr::str_replace_all(stringr::fixed(" "), "") %>% - stringr::str_replace_all("[^[:alnum:]]", "") - text -} diff --git a/R/script3_create_table_clinic_static_data.R b/R/script3_create_table_clinic_static_data.R deleted file mode 100644 index 38bd840..0000000 --- a/R/script3_create_table_clinic_static_data.R +++ /dev/null @@ -1,34 +0,0 @@ -#' @title Create clinic static data table -#' -#' @description -#' This function reads the clinic data from an Excel file, fills missing values, and exports the data as a parquet file. -#' -#' @param output_root The root directory of the output folder. -# @export -create_table_clinic_static_data <- function(output_root) { - clinic_data <- readxl::read_excel( - path = here::here("reference_data", "clinic_data.xlsx"), sheet = 1, - col_types = c("text", "text", "text", "text", "text", "text", "text", "text", "text", "date", "date") - ) - clinic_data <- clinic_data %>% - tidyr::fill(country_code:clinic_id, .direction = "down") %>% - dplyr::select(2:11) - - - logInfo( - log_to_json( - message = "clinic_data dim: {values['dim']}.", - values = list(dim = dim(clinic_data)), - script = "script3", - file = "create_table_clinic_static_data.R", - functionName = "create_table_clinic_static_data" - ) - ) - - export_data_as_parquet( - data = clinic_data, - filename = "clinic_data_static", - output_root = output_root, - suffix = "" - ) -} diff --git a/R/script3_create_table_patient_data.R b/R/script3_create_table_patient_data.R deleted file mode 100644 index 543aeea..0000000 --- a/R/script3_create_table_patient_data.R +++ /dev/null @@ -1,69 +0,0 @@ -#' @title Create CSV with monthly patient data -#' -#' @description -#' Read in all cleaned patient data CSV and create a single data.frame. -#' Only take dynamic columns. -#' -#' -#' @param patient_data_files list of CSV files with cleaned patient data from step 2. -#' @param input_root root directory of the input CSV files. -#' @param output_root root directory of the output folder. -create_table_patient_data_monthly <- function(patient_data_files, input_root, output_root) { - # THERE MIGHT BE MONTHLY COLUMNS MISSING - PLEASE ADD THEM - dynamic_patient_columns <- - c( - "age", - "bmi", - "bmi_date", - "clinic_id", - "fbg_updated_date", - "fbg_updated_mg", - "fbg_updated_mmol", - "file_name", - "hba1c_updated", - "hba1c_updated_exceeds", - "hba1c_updated_date", - "height", - "hospitalisation_cause", - "hospitalisation_date", - "insulin_injections", - "insulin_regimen", - "insulin_total_units", - "insulin_type", - "insulin_subtype", - "last_clinic_visit_date", - "last_remote_followup_date", - "observations", - "observations_category", - "patient_id", - "sheet_name", - "status", - "support_level", - "testing_frequency", - "tracker_date", - "tracker_month", - "tracker_year", - "weight" - ) - - patient_data <- read_cleaned_patient_data(input_root, patient_data_files) %>% - dplyr::select(tidyselect::all_of(dynamic_patient_columns)) %>% - dplyr::arrange(tracker_year, tracker_month, patient_id) - - logInfo( - log_to_json( - message = "patient_data dim: {values['dim']}.", - values = list(dim = dim(patient_data)), - script = "script3", - file = "create_table_patient_data.R", - functionName = "create_table_patient_data_monthly" - ) - ) - - export_data_as_parquet( - data = patient_data, - filename = "patient_data_monthly", - output_root = output_root, - suffix = "" - ) -} diff --git a/R/script3_create_table_patient_data_annual.R b/R/script3_create_table_patient_data_annual.R deleted file mode 100644 index f7eee1f..0000000 --- a/R/script3_create_table_patient_data_annual.R +++ /dev/null @@ -1,79 +0,0 @@ -#' @title Create CSV with annual patient data -#' -#' @description -#' Read in all cleaned patient data CSV and create a single data.frame. -#' Group this data by id and take the latest available data (latest year and month). -#' -#' -#' @param patient_data_files list of CSV files with cleaned patient data from step 2. -#' @param input_root root directory of the input CSV files. -#' @param output_root root directory of the output folder. -create_table_patient_data_annual <- function(patient_data_files, input_root, output_root) { - # THERE MIGHT BE STATIC COLUMNS MISSING - PLEASE ADD THEM - annual_patient_columns <- - c( - "patient_id", - "status", - "edu_occ", - "edu_occ_updated", - "blood_pressure_updated", - "blood_pressure_sys_mmhg", - "blood_pressure_dias_mmhg", - "complication_screening_kidney_test_date", - "complication_screening_kidney_test_value", - "complication_screening_eye_exam_date", - "complication_screening_eye_exam_value", - "complication_screening_foot_exam_date", - "complication_screening_foot_exam_value", - "complication_screening_lipid_profile_date", - "complication_screening_lipid_profile_triglycerides_value", - "complication_screening_lipid_profile_cholesterol_value", - "complication_screening_lipid_profile_ldl_mg_value", - "complication_screening_lipid_profile_ldl_mmol_value", - "complication_screening_lipid_profile_hdl_mg_value", - "complication_screening_lipid_profile_hdl_mmol_value", - "complication_screening_thyroid_test_date", - "complication_screening_thyroid_test_ft4_ng_value", - "complication_screening_thyroid_test_ft4_pmol_value", - "complication_screening_thyroid_test_tsh_value", - "complication_screening_remarks", - "dm_complication_eye", - "dm_complication_kidney", - "dm_complication_others", - "dm_complication_remarks", - "family_history", - "other_issues", - "tracker_date", - "tracker_month", - "tracker_year" - ) - - - annual_patient_data <- read_cleaned_patient_data(input_root, patient_data_files) %>% - dplyr::select(tidyselect::all_of(annual_patient_columns)) - - # get annual patient data for each year - annual_patient_data <- annual_patient_data %>% - dplyr::filter(tracker_year >= 2024) %>% - dplyr::group_by(patient_id, tracker_year) %>% - dplyr::slice_max(tracker_month, n = 1, with_ties = FALSE) %>% # Get the last month for each year - dplyr::ungroup() %>% - dplyr::arrange(tracker_year, tracker_month, patient_id) - - logInfo( - log_to_json( - message = "annual_patient_data dim: {values['dim']}.", - values = list(dim = dim(annual_patient_data)), - script = "script3", - file = "script3_create_table_patient_data_annual.R", - functionName = "create_table_patient_data_annual" - ) - ) - - export_data_as_parquet( - data = annual_patient_data, - filename = "patient_data", - output_root = output_root, - suffix = "_annual" - ) -} diff --git a/R/script3_create_table_patient_data_changes_only.R b/R/script3_create_table_patient_data_changes_only.R deleted file mode 100644 index 92a2dcc..0000000 --- a/R/script3_create_table_patient_data_changes_only.R +++ /dev/null @@ -1,90 +0,0 @@ -#' @title Create CSV with longitudinal patient data for a single variable. -#' -#' @description -#' Read in all cleaned patient data CSV and create a single data.frame. -#' Group this data by id and take only the months when there is a change in the medical data. -#' -#' -#' @param patient_data_files list of CSV files with cleaned patient data from step 2. -#' @param input_root root directory of the input CSV files. -#' @param output_root root directory of the output folder. -#' @param variable name of the column that should be exported. -#' @param name name used to create the export file name. -create_table_longitudinal_data <- - function(patient_data_files, - input_root, - output_root, - variable, - name) { - dynamic_patient_columns <- - c( - "blood_pressure_dias_mmhg", - "blood_pressure_sys_mmhg", - "bmi", - "bmi_date", - "clinic_id", - "fbg_updated_date", - "fbg_updated_mg", - "fbg_updated_mmol", - "file_name", - "hba1c_updated", - "hba1c_updated_exceeds", - "hba1c_updated_date", - "height", - "hospitalisation_cause", - "hospitalisation_date", - "insulin_regimen", - "insulin_type", - "insulin_subtype", - "last_clinic_visit_date", - "last_remote_followup_date", - "observations", - "observations_category", - "patient_id", - "sheet_name", - "status", - "support_from_a4d", - "testing_frequency", - "tracker_date", - "tracker_month", - "tracker_year", - "updated_2022_date", - "weight" - ) - - patient_data <- read_cleaned_patient_data(input_root, patient_data_files) %>% - dplyr::select(tidyselect::all_of(dynamic_patient_columns)) - - # get latest static patient data overall - variable_lag <- paste0(variable, "_lag") - longitudinal_data <- patient_data %>% - tidyr::drop_na(!!variable) %>% - dplyr::filter(get(variable) != ERROR_VAL_NUMERIC) %>% - dplyr::group_by(patient_id) %>% - dplyr::arrange(tracker_year, tracker_month) %>% - dplyr::filter( - get(variable) != tidyr::replace_na( - dplyr::lag(get(variable), default = NULL), - ERROR_VAL_NUMERIC - ) - ) %>% - dplyr::ungroup() %>% - dplyr::arrange(patient_id, tracker_year, tracker_month) - - logInfo( - log_to_json( - message = "longitudinal_data dim: {values['dim']}.", - values = list(dim = dim(longitudinal_data)), - script = "script3", - file = "create_table_patient_data_changes_only.log", - functionName = "create_table_longitudinal_data" - ) - ) - - export_data_as_parquet( - data = longitudinal_data, - filename = paste0("longitudinal_data_", name), - output_root = output_root, - suffix = "" - ) - } diff --git a/R/script3_create_table_patient_data_static.R b/R/script3_create_table_patient_data_static.R deleted file mode 100644 index 3e0ad4a..0000000 --- a/R/script3_create_table_patient_data_static.R +++ /dev/null @@ -1,73 +0,0 @@ -#' @title Create CSV with static patient data -#' -#' @description -#' Read in all cleaned patient data CSV and create a single data.frame. -#' Group this data by id and take the latest available data (latest year and month). -#' -#' -#' @param patient_data_files list of CSV files with cleaned patient data from step 2. -#' @param input_root root directory of the input CSV files. -#' @param output_root root directory of the output folder. -create_table_patient_data_static <- function(patient_data_files, input_root, output_root) { - # THERE MIGHT BE STATIC COLUMNS MISSING - PLEASE ADD THEM - static_patient_columns <- - c( - "clinic_id", - "dob", - "fbg_baseline_mg", - "fbg_baseline_mmol", - "file_name", - "hba1c_baseline", - "hba1c_baseline_exceeds", - "lost_date", - "name", - "patient_consent", - "patient_id", - "province", - "recruitment_date", - "sex", - "status_out", - "t1d_diagnosis_age", - "t1d_diagnosis_date", - "t1d_diagnosis_with_dka", - "tracker_date", - "tracker_month", - "tracker_year" - ) - - - static_patient_data <- read_cleaned_patient_data(input_root, patient_data_files) %>% - dplyr::select(tidyselect::all_of(static_patient_columns)) - - # get latest static patient data overall - # Group by both patient_id AND clinic_id to handle transfer patients - # who may appear in multiple clinics during the same tracker period - static_patient_data <- static_patient_data %>% - dplyr::group_by(patient_id, clinic_id) %>% - dplyr::slice_max(tracker_year, n = 1) %>% - dplyr::slice_max(tracker_month, n = 1, with_ties = FALSE) %>% - dplyr::ungroup() %>% - dplyr::arrange(tracker_year, tracker_month, patient_id, clinic_id) - - # this assertion holds no longer true because we added clinic_id to the static columns - # and patients can switch the clinic (from pediatric to adult clinic for example) - # which means for this patient there is the same static data from both clinics in the data - # testit::assert(sum(duplicated(static_patient_data$patient_id)) == 0) - - logInfo( - log_to_json( - message = "static_patient_data dim: {values['dim']}.", - values = list(dim = dim(static_patient_data)), - script = "script3", - file = "create_table_patient_data_static.R", - functionName = "create_table_patient_data_static" - ) - ) - - export_data_as_parquet( - data = static_patient_data, - filename = "patient_data", - output_root = output_root, - suffix = "_static" - ) -} diff --git a/R/script3_create_table_product_data.R b/R/script3_create_table_product_data.R deleted file mode 100644 index 3c06aca..0000000 --- a/R/script3_create_table_product_data.R +++ /dev/null @@ -1,303 +0,0 @@ -#' @title Create a Single CSV for Table Product Data -#' -#' @description -#' This function reads all parquet files in a specified directory, merges them into a single data frame, -#' and writes the merged data to a new file with reordered and processed columns according to the list of fields. -#' If there are any columns that are present in some files but not others, -#' this function will add those missing columns to the data frames where they are missing and fill them with NA. -#' -#' @param input_root A string specifying the path to the directory containing the input .parquet files. -#' @param output_root A string specifying the path to the directory where the output .parquet file should be written. -#' -#' @return This function does not return a value. It writes the merged data to a new CSV file -#' (with reordered columns according to the list of fields) in the output_root directory. -#' -#' @examples -#' \dontrun{ -#' create_table_product_data("path/to/input/directory", "path/to/output/directory") -#' } -create_table_product_data <- function(input_root, output_root) { - # Get a list of all CSV files in the input_root directory - files <- list.files(input_root, pattern = "*.parquet", full.names = TRUE) - - # Read all CSV files and store them in a list - data_list <- lapply(files, function(x) arrow::read_parquet(x)) - - # Get the union of all column names - all_names <- unique(unlist(lapply(data_list, colnames))) - - # Add missing columns to each data frame and fill them with NA - data_list <- lapply(data_list, function(x) { - x[setdiff(all_names, colnames(x))] <- NA - x - }) - - # Merge all data frames - merged_data <- do.call(rbind, data_list) - - merged_data$orig_product_released_to <- merged_data$product_released_to - - merged_data$product_released_to <- sapply(merged_data$product_released_to, fix_id) - - # Reorder, add, and ensures the correct data type for each column according to the list of fields - merged_data <- preparing_product_fields(merged_data) - - logInfo( - log_to_json( - message = "merged_data dim: {values['dim']}.", - values = list(dim = dim(merged_data)), - script = "script3", - file = "create_table_product_data.R", - functionName = "create_table_product_data" - ) - ) - - # Write the merged and processed data to a file in the output_root directory - export_data_as_parquet( - data = merged_data, - filename = "product_data", - output_root = output_root, - suffix = "" - ) -} - -#' @title Preparing Product Fields -#' -#' @description -#' This function processes fields for a single csv product_data. -#' It checks if all fields are present in merged_data and adds missing fields with NA. -#' It ensures the correct data type for each column, replaces incorrect values with specified error values, -#' and reorders the columns according to the list of fields. -#' Any additional fields in the input dataframe are moved to the end. -#' -#' @param merged_data A data frame that needs to be processed. -#' -#' @return A data frame with processed fields. -#' -#' @examples -#' \dontrun{ -#' df <- data.frame(product = c("A", "B"), product_units_notes = c("note1", "note2")) -#' df <- preparing_product_fields(df) -#' } -#' -preparing_product_fields <- function(merged_data) { - # List of fields and their corresponding data types - fields <- list( - "product" = "character", - "product_units_notes" = "character", - "product_entry_date" = "Date", - "product_units_released" = "numeric", - "product_released_to" = "character", - "product_units_received" = "numeric", - "product_received_from" = "character", - "product_balance" = "numeric", - "product_units_returned" = "numeric", - "product_returned_by" = "character", - "product_table_month" = "integer", - "product_table_year" = "integer", - "product_sheet_name" = "character", - "file_name" = "character", - "product_balance_status" = "character", - "product_category" = "character", - "orig_product_released_to" = "character", - "product_unit_capacity" = "integer", - "product_remarks" = "character" - ) - - # Check if all fields are present in merged_data - missing_fields <- setdiff(names(fields), names(merged_data)) - - # If there are missing fields, add them with NA - if (length(missing_fields) > 0) { - merged_data[missing_fields] <- NA - } - - # Ensure the correct data type for each column - for (field in names(fields)) { - tryCatch( - { - if (fields[[field]] == "Date") { - original_values <- merged_data[[field]] - merged_data[[field]] <- suppressWarnings(as.Date(original_values)) - incorrect_rows <- which(is.na(merged_data[[field]]) & !is.na(original_values)) - if (length(incorrect_rows) > 0) { - logWarn( - log_to_json( - message = "In {values['field']} incorrect date values were replaced with {values['error_val']} in {values['length']} rows: {values['incorrect_rows']}.", - values = list(field = field, error_val = ERROR_VAL_DATE, length = length(incorrect_rows), incorrect_rows = incorrect_rows), - script = "script3", - file = "create_table_product_data.R", - functionName = "preparing_product_fields", - warningCode = "invalid_value" - ) - ) - merged_data[incorrect_rows, field] <- ERROR_VAL_DATE - } - } else if (fields[[field]] == "numeric") { - original_values <- merged_data[[field]] - merged_data[[field]] <- suppressWarnings(as.numeric(original_values)) - incorrect_rows <- which(is.na(merged_data[[field]]) & !is.na(original_values)) - if (length(incorrect_rows) > 0) { - logWarn( - log_to_json( - message = "In {values['field']} incorrect numeric values were replaced with {values['error_val']} in {values['length']} rows: {values['incorrect_rows']}.", - values = list(field = field, error_val = ERROR_VAL_NUMERIC, length = length(incorrect_rows), incorrect_rows = incorrect_rows), - script = "script3", - file = "create_table_product_data.R", - functionName = "preparing_product_fields", - warningCode = "invalid_value" - ) - ) - merged_data[incorrect_rows, field] <- ERROR_VAL_NUMERIC - } - } else if (fields[[field]] == "integer") { - original_values <- merged_data[[field]] - merged_data[[field]] <- suppressWarnings(as.integer(original_values)) - incorrect_rows <- which(is.na(merged_data[[field]]) & !is.na(original_values)) - if (length(incorrect_rows) > 0) { - logWarn( - log_to_json( - message = "In {values['field']} incorrect integer values were replaced with {values['error_val']} in {values['length']} rows: {values['incorrect_rows']}.", - values = list(field = field, error_val = ERROR_VAL_NUMERIC, length = length(incorrect_rows), incorrect_rows = incorrect_rows), - script = "script3", - file = "create_table_product_data.R", - functionName = "preparing_product_fields", - warningCode = "invalid_value" - ) - ) - merged_data[incorrect_rows, field] <- ERROR_VAL_NUMERIC - } - } else { - original_values <- merged_data[[field]] - merged_data[[field]] <- as.character(original_values) - incorrect_rows <- which(is.na(merged_data[[field]]) & !is.na(original_values)) - if (length(incorrect_rows) > 0) { - logWarn( - log_to_json( - message = "In {values['field']} incorrect character values were replaced with {values['error_val']} in {values['length']} rows: {values['incorrect_rows']}.", - values = list(field = field, error_val = ERROR_VAL_CHARACTER, length = length(incorrect_rows), incorrect_rows = incorrect_rows), - script = "script3", - file = "create_table_product_data.R", - functionName = "preparing_product_fields", - warningCode = "invalid_value" - ) - ) - merged_data[incorrect_rows, field] <- ERROR_VAL_CHARACTER - } - } - }, - error = function(e) { - logError( - log_to_json( - message = "Error in converting {values['field']}: {values['e']}.", - values = list(field = field, e = e), - script = "script3", - file = "create_table_product_data.R", - functionName = "preparing_product_fields", - errorCode = "tryCatch" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - message = "Warning in converting {values['field']}: {values['w']}.", - values = list(field = field, w = w), - script = "script3", - file = "create_table_product_data.R", - functionName = "preparing_product_fields", - warningCode = "tryCatch" - ) - ) - } - ) - } - - # Reorder the columns according to the list of fields - merged_data <- merged_data[, c(names(fields), setdiff(names(merged_data), names(fields)))] - - return(merged_data) -} - -#' @title Calculate the Most Frequent Value in a Data Frame -#' -#' @description -#' This function calculates the most frequent value of a specified column for each group in a data frame. -#' It then adds a new column to the data frame with these most frequent values. -#' -#' @param df A data frame. -#' @param group_column The name of the column to group by. -#' @param value_column The name of the column to calculate the most frequent value from. -#' @param new_column The name of the new column to add to the data frame. -#' -#' @return A data frame with an additional column containing the most frequent value of the specified column for each group. -#' @export -#' -#' @examples -#' \dontrun{ -#' library(dplyr) -#' data(mtcars) -#' result <- calculate_most_frequent(mtcars, "cyl", "gear", "most_frequent_gear") -#' } -calculate_most_frequent <- function(df, group_column, value_column, new_column) { - # Group by 'group_column' and find the most frequent 'value_column' value for each group - most_frequent_value <- df %>% - dplyr::filter(!is.na(.data[[value_column]])) %>% - dplyr::group_by(.data[[group_column]]) %>% - dplyr::summarise(across(.data[[value_column]], ~ names(which.max(table(.))))) - - most_frequent_value <- dplyr::rename(most_frequent_value, !!new_column := .data[[value_column]]) - - # Join the original dataframe with the most frequent value dataframe - df <- df %>% - dplyr::left_join(most_frequent_value, by = group_column) - - return(df) -} - -#' @title Report Empty Intersections -#' -#' @description -#' This function reports the names of the rows in a cross-tabulation of two columns -#' from a dataframe where the sum is 0. -#' -#' @param df A dataframe. -#' @param row_category The name of the column in df to use for the rows of the cross-tabulation. -#' @param col_category The name of the column in df to use for the columns of the cross-tabulation. -#' -#' @return A warning message with the number of row_category values that have a sum of 0 in col_category, and a list of these row_category values. -#' @export -#' -#' @examples -#' df <- data.frame(file_name = c("file1", "file2", "file3"), table_country = c("country1", "country2", NA)) -#' report_empty_intersections(df, "file_name", "table_country") -report_empty_intersections <- function(df, row_category, col_category) { - crosstab <- table(df[[row_category]], df[[col_category]]) - - # Calculate the sum by each row - row_sums <- rowSums(crosstab) - - # Convert to a data frame - df_row_sums <- data.frame(row_name = rownames(crosstab), sum = row_sums) - - # Filter the rows where 'sum' is 0 - df_row_sums <- df_row_sums[df_row_sums$sum == 0, ] - - if (nrow(df_row_sums) > 0) { - logWarn( - log_to_json( - message = "There are {values['sum']} empty values for the combination {values['category1']}x{values['category2']}: {values['names']}.", - values = list( - sum = nrow(df_row_sums), - category1 = row_category, - category2 = col_category, - names = df_row_sums$row_name - ), - script = "script3", - file = "create_table_product_data.R", - functionName = "report_empty_intersections", - warningCode = "invalid_value" - ) - ) - } -} diff --git a/R/script3_link_product_patient.R b/R/script3_link_product_patient.R deleted file mode 100644 index 653480e..0000000 --- a/R/script3_link_product_patient.R +++ /dev/null @@ -1,81 +0,0 @@ -#' @title Link Product and Patient Data -#' -#' @description -#' This function links a product file with a patient file. It reads the files, merges them based on 'file_name' and patient 'id', -#' finds the ids that are in the product data but not in the patient data, creates a summary of these ids, and logs the process. -#' -#' @param product_file A string specifying the path to the product file. -#' @param patient_file A string specifying the path to the patient file. -#' -#' @return This function does not return a value. It prints log messages about the linking process. -#' -#' @examples -#' \dontrun{ -#' link_product_patient("path/to/product_data.parquet", "path/to/patient_data.parquet") -#' } -link_product_patient <- function(product_file, patient_file) { - patient_data <- arrow::read_parquet(patient_file) - product_data <- arrow::read_parquet(product_file) - - # Merge the data frames by file_name and patients ids - merged_data <- merge(product_data, patient_data, - by.x = c("file_name", "product_released_to"), - by.y = c("file_name", "patient_id"), - all.x = TRUE - ) - - # Find the ids that are in product_data but not in patient_data - missing_ids <- merged_data[is.na(merged_data$sheet_name), ] - - if (nrow(missing_ids) > 0) { - # Create a summary of the missing_ids data frame - summary_table <- table(missing_ids$file_name, missing_ids$product_released_to) - - # Convert the table to a data frame - summary_df <- as.data.frame(summary_table) - - - summary_df <- dplyr::filter(summary_df, Freq > 0) - - tryCatch( - { - if (nrow(summary_df) > 0) { - logWarn( - log_to_json( - message = "The number of mismatched patient IDs between the product and patient data is {values['len']}. File Name: {values['file_name']}, Patient ID in product: {values['id']}, Count in product: {values['count']}", - values = list(len = nrow(summary_df), file_name = summary_df$Var1, id = summary_df$Var2, count = summary_df$Freq), - script = "script3", - file = "link_product_patient.R", - functionName = "link_product_patient", - warningCode = "invalid_value" - ) - ) - } - }, - error = function(e) { - logError( - log_to_json( - message = "Could not link csv files for product and patient data. Error: {values['e']}", - values = list(e = e$message), - script = "script3", - file = "link_product_patient.R", - functionName = "link_product_patient", - errorCode = "tryCatch" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - message = "Could not link csv files for product and patient data. Warning: {values['w']}", - values = list(w = w$message), - script = "script3", - file = "link_product_patient.R", - functionName = "link_product_patient", - errorCode = "tryCatch" - ) - ) - } - ) - } -} diff --git a/R/script3_read_cleaned_patient_data.R b/R/script3_read_cleaned_patient_data.R deleted file mode 100644 index 4f696e1..0000000 --- a/R/script3_read_cleaned_patient_data.R +++ /dev/null @@ -1,17 +0,0 @@ -#' Read in all processed and cleaned patient data files -#' -#' @param input_root root directory containing all files. -#' @param patient_data_files list of CSV files with cleaned patient data. -#' -#' @return data frame holding all cleaned patient data with correct column types. -#' @export -read_cleaned_patient_data <- - function(input_root, patient_data_files) { - patient_data <- patient_data_files %>% - purrr::map(function(patient_file) { - arrow::read_parquet(file.path(input_root, patient_file)) - }) %>% - dplyr::bind_rows() - - patient_data - } diff --git a/R/sysdata.rda b/R/sysdata.rda deleted file mode 100644 index 6f88324..0000000 Binary files a/R/sysdata.rda and /dev/null differ diff --git a/R/wide_format_2_long_format.R b/R/wide_format_2_long_format.R deleted file mode 100644 index 4fdb391..0000000 --- a/R/wide_format_2_long_format.R +++ /dev/null @@ -1,326 +0,0 @@ -#' @title Check if dataframe contains required columns -#' -#' @description -#' This function checks if a dataframe contains the columns 'Total Units Released' and 'Units Released per person'. -#' (for detection the Mandalay Children's Hospital 2020-2021 wide-format) -#' It returns TRUE if both columns are present, and FALSE otherwise. -#' -#' @param df A dataframe to check for the presence of required columns. -#' @return A logical value indicating whether the dataframe contains the required columns. -#' @examples -#' df <- data.frame("Total Units Released" = c(1, 2, 3), "Units Released per person" = c(4, 5, 6), "Other Column" = c(7, 8, 9)) -#' check_wide_format_columns(df) # Returns: TRUE -check_wide_format_columns <- function(df) { - required_columns <- c("Total Units Released", "Units Released per person") - all(required_columns %in% names(df)) -} - -#' @title Find Columns Containing Specific String -#' -#' @description -#' This function returns the column names of a dataframe that contain a specific string. -#' -#' @param df A dataframe in which to search for the string. -#' @param string The string to search for in the column names. -#' -#' @return A character vector of matching column names. -#' -#' @examples -#' data(mtcars) -#' find_string_cols(mtcars, "mpg") -find_string_cols <- function(df, string) { - matching_cols <- colnames(df)[grepl(string, names(df))] - return(matching_cols) -} - -#' @title Check for Specific Patterns in a Dataframe Column -#' -#' @description -#' This function checks if any cell in a specific column of a dataframe contains a pattern with a hyphen followed by a digit, or a comma (which could be leading with a space). It returns TRUE if any cell in the column contains the pattern, and FALSE otherwise. -#' -#' @param df A dataframe to check for the specific patterns. -#' @param column_name The name of the column to check for the patterns. -#' -#' @return A logical value indicating whether any cell in the specified column contains the patterns. -#' -#' @examples -#' data(mtcars) -#' check_patterns_in_column(mtcars, "mpg") -#' -#' @export -check_patterns_in_column <- function(df, column_name) { - # pattern <- "(-\\d)|(,\\s*)" - pattern <- "(-\\s*\\d)|(,\\s*)" - return(any(grepl(pattern, df[[column_name]]))) -} - -#' @title Remove Original Cells based on Relationship -#' -#' @description -#' This function removes rows from the column `col_units_rel` where the corresponding value in `col_rel_to` contains a comma (","). -#' It also removes those comma-separated values from the `col_rel_to` column itself. -#' -#' @param df A data frame containing the columns `col_units_rel` and `col_rel_to`. -#' @param col_units_rel The name of the column containing units (cells) potentially related to multiple other units. -#' @param col_rel_to The name of the column containing relationships between units, where comma (",") indicates a relationship with multiple units. -#' -#' @return -#' The modified data frame where rows with comma-separated relationships in `col_rel_to` have corresponding values in `col_units_rel` set to NA and the comma-separated values removed from `col_rel_to`. -#' @examples -#' # Create a sample data frame -#' df <- data.frame(col_units_rel = c(1, 2, 3, 4), col_rel_to = c("A", "B,C", "D", "A,B")) -#' -#' # Remove original cells based on relationship -#' df_modified <- remove_original_cells(df, col_units_rel = "col_units_rel", col_rel_to = "col_rel_to") -#' -#' # Print the modified data frame -#' print(df_modified) -remove_original_cells <- function(df, col_units_rel, col_rel_to) { - df[[col_units_rel]][df[[col_rel_to]] %in% grep(",", df[[col_rel_to]], value = TRUE)] <- NA - df[[col_rel_to]][df[[col_rel_to]] %in% grep(",", df[[col_rel_to]], value = TRUE)] <- NA - return(df) -} - -#' @title Replace Extra "Total" Values with NA -#' -#' @description -#' This function replaces values in a specified column (`column_name`) with NA if one of the two preceding columns contains the word "Total". -#' This is intended to handle cases where there might be multiple "Total" values in close proximity, keeping only the first. -#' -#' @param df A data frame containing the column to be processed. -#' @param column_name The name of the column where extra "Total" values will be replaced with NA. -#' -#' @return -#' The modified data frame where any value in `column_name` preceded by "Total" in one of the two preceding columns is replaced with NA. -#' -#' @examples -#' # Create a sample data frame -#' df <- data.frame(col1 = c("A", "B", "Total"), col2 = c(1, 2, 3), col3 = c(4, 5, 6), col4 = c(7, "Total", 9)) -#' -#' # Replace extra "Total" values with NA -#' df_modified <- replace_extra_total_values_with_NA(df, column_name = "col4") -#' -#' # Print the modified data frame -#' print(df_modified) -replace_extra_total_values_with_NA <- function(df, column_name) { - # Get the column index - col_index <- which(names(df) == column_name) - - # Check if the column index is greater than 2 (to avoid negative indexing) - if (col_index > 2) { - # Loop over rows of the dataframe - for (i in 1:nrow(df)) { - # If one of the two previous columns contains 'Total' (case-insensitive), replace the cell with NA - if ("total" %in% tolower(df[i, (col_index - 2):(col_index - 1)])) { - df[i, column_name] <- NA - } - } - } - - return(df) -} - -#' @title Convert Wide-Format Cells to Long-Format Rows -#' -#' @description -#' This function converts a data frame from wide-format to long-format. It assumes specific column naming conventions: -#' - Columns containing "Released To" are expected to have comma-separated values representing units released to. -#' - Columns containing "Units Released" are expected to store the corresponding unit quantity for each "Released To" value. -#' - A column containing "Date" (or similar) should exist to hold the date information. -#' -#' @param df A data frame in wide-format with columns named as described. -#' -#' @return -#' A new data frame in long-format where each row represents a single release event with columns: -#' - The original columns from the input data frame. -#' - "Released To": The recipient unit for a specific release. -#' - "Units Released": The quantity of units released to the corresponding recipient. -#' - "Date": The date of the release event (unchanged from the input). -#' - "Received From": unchanged from the input. -#' -#' @examples -#' # Create a sample data frame in wide-format -#' df <- data.frame( -#' ID = c(1, 2, 3), -#' "Units Available" = c(10, 15, 20), -#' "Released To A,B" = c(NA, "5-2", "8-1"), -#' Date = c("2024-01-01", "2024-02-01", "2024-03-01") -#' ) -#' -#' # Convert wide-format cells to long-format rows -#' df_long <- wide_cells_2_rows(df) -#' -#' # Print the long-format data frame -#' print(df_long) -wide_cells_2_rows <- function(df) { - tryCatch( - { - col_released_to <- find_string_cols(df, "Released To") - col_units_released <- find_string_cols(df, "Units Released") - col_date <- find_string_cols(df, "Date") - col_recived_from <- find_string_cols(df, "Received From") - - # Create an empty dataframe to store the new rows - new_df <- df[FALSE, ] - # Loop over rows of the dataframe - for (i in 1:nrow(df)) { - # Add the current row to the new dataframe - new_df <- rbind(new_df, df[i, ]) - # If the cell in 'Released To ...' is not empty, split its value by comma - if (!is.na(df[[col_released_to]][i]) & grepl(",", df[[col_released_to]][i])) { - # Split the cell content by comma - cell_content <- strsplit(as.character(df[[col_released_to]][i]), ",")[[1]] - for (content in cell_content) { - # Split the content by '-' - split_content <- strsplit(content, "-")[[1]] - new_row <- df[i, ] - new_row[] <- NA - new_row[[col_date]] <- df[[col_date]][i] - new_row[[col_recived_from]] <- df[[col_recived_from]][i] - # Assign the part before '-' to 'Released To ...' - new_row[[col_released_to]] <- split_content[1] - # Assign the part after '-' to 'Units Released' - new_row[[col_units_released]] <- split_content[2] - # Add the new row to the new dataframe - new_df <- rbind(new_df, new_row) - } - } - } - - new_df <- remove_original_cells(new_df, col_units_released, col_released_to) - - logDebug( - log_to_json( - "The wide-format cells has been converted to long-format!", - script = "script1", - file = "wide_format_2_long_format.R", - functionName = "wide_cells_2_rows" - ) - ) - return(new_df) - }, - error = function(e) { - # Handle errors - logError( - log_to_json( - "Error occurred while converting a wide-format cells to a long-format, dataframe was returned without changes. Error: {values['e']}", - values = list(e = e$message), - script = "script1", - file = "wide_format_2_long_format.R", - errorCode = "tryCatch", - functionName = "wide_cells_2_rows" - ) - ) - # Return original df - return(df) - }, - warning = function(w) { - # Handle warnings - logWarn( - log_to_json( - "Warning occurred while converting a wide-format cells to a long-format, dataframe was returned without changes. Warning: {values['w']}", - values = list(w = w$message), - script = "script1", - file = "wide_format_2_long_format.R", - warningCode = "tryCatch", - functionName = "wide_cells_2_rows" - ) - ) - # Return original df - return(df) - } - ) -} - -#' @title Create New Rows in a DataFrame -#' -#' @description -#' This function creates new rows in a given dataframe based on specific conditions. -#' To convert wide-format area in the 2017 - 2020 data to long-format area. -#' -#' @param df A dataframe containing the original data. -#' @return A modified dataframe with additional rows. -#' @details The function checks if 'Total Units Released' and 'Units Released per person' -#' are not empty and not equal. If this condition is met, it adds new rows to the dataframe -#' based on values in columns between 'Released To' and 'Units Released per person'. -#' If an error occurs during processing, the original dataframe is returned. -#' -#' @examples -#' df <- data.frame( -#' "Date" = c("2024-03-01", "2024-03-02"), -#' "Total Units Released" = c(100, 200), -#' "Units Released per person" = c(10, 20), -#' "Released To (select from drop down list)" = c("A", "B") -#' ) -#' new_df <- create_new_rows(df) -create_new_rows <- function(df) { - tryCatch( - { - # Create an empty dataframe to store the new rows - new_df <- df[FALSE, ] - # Loop over rows of the dataframe - for (i in 1:nrow(df)) { - # Add the current row to the new dataframe - new_df <- rbind(new_df, df[i, ]) - # Check if 'Total Units Released' and 'Units Released per person' are not empty and not equal - if (!is.na(df$"Total Units Released"[i]) & !is.na(df$"Units Released per person"[i]) & df$"Total Units Released"[i] != df$"Units Released per person"[i]) { - # Loop over all columns between 'Released To' and 'Units Released per person' - start_col <- which(names(df) == "Released To (select from drop down list)") + 1 - end_col <- which(names(df) == "Units Released per person") - 1 - for (j in start_col:end_col) { - # If the cell is not empty, copy its value to a new row in 'Released To (select from drop down list)' - if (!is.na(df[i, j])) { - new_row <- df[i, ] - new_row[] <- NA - new_row$"Date" <- df$"Date"[i] - new_row$"Released To (select from drop down list)" <- df[i, j] - new_row$"Units Released per person" <- df$"Units Released per person"[i] - # Add the new row to the new dataframe - new_df <- rbind(new_df, new_row) - } - } - } - } - logDebug( - log_to_json( - "The wide-format area has been converted to long-format!", - script = "script1", - file = "wide_format_2_long_format.R", - functionName = "create_new_rows" - ) - ) - - return(new_df) - }, - error = function(e) { - # Handle errors - logError( - log_to_json( - "Error occurred while converting a wide-format area to a long-format, dataframe was returned without changes. Error: {values['e']}", - values = list(e = e$message), - script = "script1", - file = "wide_format_2_long_format.R", - errorCode = "tryCatch", - functionName = "create_new_rows" - ) - ) - # Return original df - return(df) - }, - warning = function(w) { - # Handle warnings - logWarn( - log_to_json( - "Warning occurred while converting a wide-format area to a long-format, dataframe was returned without changes. Warning: {values['w']}", - values = list(w = w$message), - script = "script1", - file = "wide_format_2_long_format.R", - warningCode = "tryCatch", - functionName = "create_new_rows" - ) - ) - # Return original df - return(df) - } - ) -} diff --git a/README.md b/README.md new file mode 100644 index 0000000..f9c8057 --- /dev/null +++ b/README.md @@ -0,0 +1,266 @@ +# A4D Data Processing Pipeline (Python) + +Python implementation of the A4D medical tracker data processing pipeline. + +## Status + +Production. Both arms (patient and product) run as one orchestrated pipeline on +Cloud Run against the A4D GCS bucket, landing in BigQuery. + +This replaced an earlier R implementation, which was retired on 2026-08-24. The +record of that migration is archived in [docs/archive/](docs/archive/); nothing +in it is current guidance. + +## Features + +- ✅ **Dual Pipeline Arms** - Patient and product trackers processed in one run +- ✅ **Incremental Processing** - Skip trackers whose MD5 + completion state match the previous run's manifest +- ✅ **Parallel Execution** - Process multiple trackers concurrently +- ✅ **Stateless GCP Deployment** - Uses BigQuery for state management +- ✅ **Comprehensive Error Tracking** - Detailed error logs per patient/tracker +- ✅ **High Performance** - Built on Polars (10-100x faster than pandas) + +## Quick Start + +### Installation + +```bash +# Install uv (if not already installed) +curl -LsSf https://astral.sh/uv/install.sh | sh + +# Install just (optional, for convenient commands) +# macOS: brew install just +# Other: https://github.com/casey/just + +# Install dependencies +just sync +# or: uv sync --all-extras +``` + +### Configuration + +Create a `.env` file: + +```bash +A4D_ENVIRONMENT=development +A4D_DATA_ROOT=/path/to/tracker/files +A4D_PROJECT_ID=a4dphase2 +A4D_DATASET=tracker +A4D_DOWNLOAD_BUCKET=a4dphase2_upload +A4D_UPLOAD_BUCKET=a4dphase2_output +``` + +### Running the Pipeline + +```bash +# Full end-to-end pipeline (Drive + GCS download → patient + product → GCS + BigQuery upload) +just run +# or: uv run a4d run + +# Common `a4d run` flags +just run --workers 8 +just run --skip-upload # Local testing: process but don't upload +just run --skip-download # Reuse files already in data_root +just run --skip-drive-download # Skip clinic_data.xlsx refresh from Google Drive +just run --skip-product # Patient-only run +just run --skip-patient # Product-only run (mutually exclusive with --skip-product) +just run --incremental # Skip unchanged trackers (MD5 + completion match) +just run --force # Wipe prior local outputs before each arm runs + +# Single-arm runs (no GCS download/upload) +just run-local # Patient extract + clean + tables +just run-local-product # Product extract + clean + tables +just run-file path/to/tracker.xlsx # Single patient tracker +just run-file-product path/to/tracker.xlsx # Single product tracker +``` + +## Architecture + +``` +`a4d run` flow: +0. Download reference data (clinic_data.xlsx) from Google Drive +1. Download tracker files from GCS +2-3. Patient arm: extract → clean → tables (static, monthly, annual) +3b. Clinic static table from clinic_data.xlsx +3c. Product arm: extract → clean → product_data table +3d. Tracker metadata table (MD5 + per-tracker output presence) +3e. Product ↔ patient link validation (logging-only) +4. Upload tables/ and logs/ to GCS under YYYY/MM/DD/HHMMSS/ +5. Ingest tables into BigQuery (WRITE_TRUNCATE) +``` + +Both arms share the same tracker queue. With `--incremental`, the queue is filtered once against the previous run's manifest so both arms see the same set. + +Output tables loaded into BigQuery: + +- `patient_data_static`, `patient_data_monthly`, `patient_data_annual` +- `product_data` +- `clinic_data_static` +- `tracker_metadata` +- `logs` + +## Project Structure + +``` +a4d/ +├── src/a4d/ # Main package +│ ├── cli.py # Typer CLI (run/create/upload/download command groups) +│ ├── config.py # Pydantic settings (A4D_* env vars) +│ ├── logging.py # loguru configuration +│ ├── extract/ # Sheet extraction (patient.py, product.py, wide_format.py) +│ ├── clean/ # Cleaning + schemas (patient.py, product.py, schema*.py) +│ ├── pipeline/ # Per-arm orchestration (patient.py, product.py, tracker.py) +│ ├── tables/ # Aggregation into final parquets (patient, product, clinic, metadata, logs) +│ ├── validate/ # Source-vs-output reconciliation (patient + product) +│ ├── gcp/ # BigQuery, GCS, Drive integration +│ ├── reference/ # Reference data loaders (synonyms, validation rules, provinces) +│ ├── state/ # Manifest + incremental filtering +│ └── utils/ # Shared utilities +├── reference_data/ # Shared YAML configs + clinic_data.xlsx +├── tests/ # Test suite +├── scripts/ # Utility scripts +├── docs/ # Migration + feature docs +├── justfile # Development commands +└── pyproject.toml # Dependencies +``` + +## Development + +### Common Commands + +```bash +# Show all available commands +just + +# Run all CI checks (format, lint, type, test) +just ci + +# Run tests with coverage +just test + +# Run tests without coverage (faster) +just test-fast + +# Format code +just format + +# Lint code +just lint + +# Auto-fix linting issues +just fix + +# Type checking with ty +just check + +# Clean build artifacts +just clean +``` + +### Running Tests + +```bash +# All tests with coverage +just test +# or: uv run pytest --cov + +# Fast tests (no coverage) +just test-fast +# or: uv run pytest -x + +# Specific test file +uv run pytest tests/test_extract/test_patient.py +``` + +### Code Quality + +```bash +# Run all checks (what CI runs) +just ci + +# Individual checks +just lint # Linting +just format # Format code +just format-check # Check formatting without changes +just check # Type checking with ty +just fix # Auto-fix linting issues +``` + +### Pre-commit Hooks + +```bash +# Install hooks +just hooks +# or: uv run pre-commit install + +# Run manually on all files +just hooks-run +# or: uv run pre-commit run --all-files +``` + +### Docker + +```bash +# Build Docker image (tagged :latest and :) +just docker-build + +# Smoke-test the image (CLI reachable inside the container) +just docker-smoke + +# Push both tags to Artifact Registry +just docker-push + +# Or manually (build from the repo root; the image writes under A4D_DATA_ROOT): +docker build --platform=linux/amd64 -t a4d-pipeline:latest . +docker run --rm --env-file .env -v "$(pwd)/data:/workspace/data" a4d-pipeline:latest +``` + +### Other Commands + +```bash +# Update dependencies +just update + +# Show project info +just info +``` + +## Technology Stack + +### Astral Toolchain + +- **uv** - Fast dependency management +- **ruff** - Linting and formatting +- **ty** - Type checking + +### Data Processing + +- **Polars** - Fast dataframe operations (10-100x faster than pandas) +- **DuckDB** - Complex SQL aggregations +- **Pydantic** - Type-safe configuration +- **Pandera** - DataFrame validation + +### Infrastructure + +- **loguru** - Structured JSON logging +- **Google Cloud SDK** - BigQuery & GCS integration +- **pytest** - Testing framework +- **just** - Command runner for development + +## History + +This pipeline replaced an R implementation, retired 2026-08-24. What the +rewrite gained: + +- Patient and product trackers in a single orchestrated run +- Incremental processing (only changed files) +- Row-level error tracking, published as its own BigQuery table +- Single Docker container, deployed to Cloud Run + +The migration was verified cell-by-cell against the old pipeline's frozen +output over 254 trackers before it was retired. Those working papers are in +[docs/archive/](docs/archive/), frozen and not maintained. + +## License + +MIT diff --git a/SETUP.md b/SETUP.md new file mode 100644 index 0000000..d988830 --- /dev/null +++ b/SETUP.md @@ -0,0 +1,336 @@ +# A4D Pipeline — Setup Guide + +## Local Development + +### Prerequisites + +```bash +# uv (Python package manager) +curl -LsSf https://astral.sh/uv/install.sh | sh + +# just (command runner) +brew install just + +# gcloud CLI +brew install google-cloud-sdk +``` + +### Install + +Run everything from the repository root — the Python pipeline lives at the root +(`src/`, `tests/`, `pyproject.toml`). + +```bash +uv sync +cp .env.example .env +``` + +> `.env` is only used for local development. On GCP, environment variables are +> set directly on the Cloud Run Job (see step 5 in the GCP section below) and +> the `.env` file is not present or needed in the container. + +Edit `.env` — only these fields matter locally: + +```bash +A4D_DATA_ROOT=/path/to/tracker/files # folder containing .xlsx trackers +A4D_PROJECT_ID=a4dphase2 +A4D_DATASET=tracker +A4D_DOWNLOAD_BUCKET=a4dphase2_upload +A4D_UPLOAD_BUCKET=a4dphase2_output +``` + +**Paths with spaces** (e.g. a USB drive): write the value unquoted in `.env` — +pydantic-settings reads to end of line and handles spaces correctly: + +```bash +A4D_DATA_ROOT=/Volumes/USB SanDisk 3.2Gen1 Media/a4d/a4dphase2_upload +``` + +### Authenticate + +```bash +gcloud auth login +gcloud auth application-default login +gcloud config set project a4dphase2 +``` + +### Run + +```bash +# Test with a single file (fastest) +just run-file /path/to/tracker.xlsx + +# Process all files already in A4D_DATA_ROOT — no GCS +just run-local + +# Download latest files from GCS, process locally — no upload +just run-download + +# Full pipeline: download from GCS, process, upload results + load BigQuery +just run +``` + +For paths with spaces, wrap the argument in quotes: + +```bash +just run-file "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/2024_Penang.xlsx" +``` + +--- + +## Google Cloud Deployment + +The pipeline runs as a **Cloud Run Job** — a one-shot container that downloads +tracker files from GCS, processes them, and loads the results into BigQuery. +A service account is used instead of personal credentials. + +> **Data residency**: All GCP resources (Artifact Registry, Cloud Run Job, +> Cloud Scheduler, BigQuery dataset, GCS buckets) must be located in +> **`asia-southeast2` (Jakarta)**. Patient data must not be processed or stored +> in the EU. + +> **Steps 1–4 are one-time infrastructure setup.** Once the service account, +> IAM roles, and Artifact Registry repository exist, you only need to rebuild +> and redeploy (steps 4–5) when the code changes. + +### 1. Create the service account + +This only needs to be done once. Check if it already exists first: + +```bash +gcloud iam service-accounts describe \ + a4d-pipeline@a4dphase2.iam.gserviceaccount.com \ + --project=a4dphase2 +``` + +If it doesn't exist yet, create it: + +```bash +gcloud iam service-accounts create a4d-pipeline \ + --display-name="A4D Pipeline Runner" \ + --project=a4dphase2 +``` + +### 2. Grant IAM roles + +The service account needs access to two GCS buckets and the BigQuery dataset. + +> Both GCS buckets (`a4dphase2_upload`, `a4dphase2_output`) must be located in +> `asia-southeast2`. Bucket location is set at creation time and cannot be +> changed. + +**GCS — read tracker files:** + +```bash +gcloud storage buckets add-iam-policy-binding gs://a4dphase2_upload \ + --member="serviceAccount:a4d-pipeline@a4dphase2.iam.gserviceaccount.com" \ + --role="roles/storage.objectViewer" +``` + +**GCS — write pipeline output:** + +```bash +gcloud storage buckets add-iam-policy-binding gs://a4dphase2_output \ + --member="serviceAccount:a4d-pipeline@a4dphase2.iam.gserviceaccount.com" \ + --role="roles/storage.objectCreator" +``` + +> `objectCreator` grants only `storage.objects.create` — sufficient for upload. +> `objectAdmin` (broader) is not needed as the pipeline never reads, lists, or +> manages IAM on the output bucket. + +> The BigQuery dataset `tracker` must be created in `asia-southeast2`. Dataset +> location is set at creation time and cannot be changed. If the dataset already +> exists in another region, it must be deleted and recreated (data loss — export +> first). + +**BigQuery — run jobs (project-level):** + +```bash +gcloud projects add-iam-policy-binding a4dphase2 \ + --member="serviceAccount:a4d-pipeline@a4dphase2.iam.gserviceaccount.com" \ + --role="roles/bigquery.jobUser" +``` + +**BigQuery — read/write tables (project-level):** + +```bash +gcloud projects add-iam-policy-binding a4dphase2 \ + --member="serviceAccount:a4d-pipeline@a4dphase2.iam.gserviceaccount.com" \ + --role="roles/bigquery.dataEditor" +``` + +> `bq add-iam-policy-binding` (dataset-scoped) requires allowlisting and does not +> work on standard projects. Use the project-level grant above instead. +> `dataEditor` allows creating and overwriting tables (`tables.create` + +> `tables.updateData`) which WRITE_TRUNCATE load jobs require. + +### 3. Set up Artifact Registry + +```bash +# Create the repository (once) +gcloud artifacts repositories create a4d \ + --repository-format=docker \ + --location=asia-southeast2 \ + --project=a4dphase2 + +# Allow the service account to pull images +gcloud artifacts repositories add-iam-policy-binding a4d \ + --location=asia-southeast2 \ + --member="serviceAccount:a4d-pipeline@a4dphase2.iam.gserviceaccount.com" \ + --role="roles/artifactregistry.reader" \ + --project=a4dphase2 +``` + +### 4. Build and push the Docker image + +Authenticate Docker to Artifact Registry once: + +```bash +gcloud auth configure-docker asia-southeast2-docker.pkg.dev +``` + +Then build and push (from the repository root): + +```bash +just docker-push +``` + +This builds with the repo root as context — the Dockerfile copies +`pyproject.toml`, `uv.lock`, `src/` and `reference_data/` from there — and +pushes both the `:latest` and `:` tags to Artifact Registry. + +To verify the image was pushed and see what's already in the registry: + +```bash +gcloud artifacts docker images list \ + asia-southeast2-docker.pkg.dev/a4dphase2/a4d \ + --include-tags \ + --project=a4dphase2 +``` + +### 5. Create the Cloud Run Job + +```bash +gcloud run jobs create a4d-pipeline \ + --image=asia-southeast2-docker.pkg.dev/a4dphase2/a4d/pipeline:latest \ + --region=asia-southeast2 \ + --service-account=a4d-pipeline@a4dphase2.iam.gserviceaccount.com \ + --set-env-vars="\ +A4D_PROJECT_ID=a4dphase2,\ +A4D_DATASET=tracker,\ +A4D_DOWNLOAD_BUCKET=a4dphase2_upload,\ +A4D_UPLOAD_BUCKET=a4dphase2_output,\ +A4D_OUTPUT_DIR=output,\ +A4D_MAX_WORKERS=8" \ + --memory=8Gi \ + --cpu=8 \ + --task-timeout=3600 \ + --project=a4dphase2 +``` + +The container writes to `A4D_DATA_ROOT`, which the image already sets to +`/workspace/data` (see the `Dockerfile`) — ephemeral in-container storage. The +job downloads tracker files there, processes them, uploads the output, then +exits. Nothing persists between executions. Override it on the job only if you +want a different path. + +> **What is actually deployed today**: the live `a4d-pipeline` job sets only +> `A4D_MAX_WORKERS=8` as an env var; every other setting resolves from the +> image's own `ENV` or from `src/a4d/config.py` defaults, which already match +> the values above. Check with `just job-settings` or: +> +> ```bash +> gcloud run jobs describe a4d-pipeline --region=asia-southeast2 --project=a4dphase2 +> ``` + +To update the job after a config change: + +```bash +gcloud run jobs update a4d-pipeline --region=asia-southeast2 [--set-env-vars=...] +``` + +To list all existing jobs: + +```bash +gcloud run jobs list --region=asia-southeast2 --project=a4dphase2 +``` + +### 5a. Test the image locally before deploying + +Always verify a newly built image works before creating or updating the Cloud Run Job. + +**Level 1 — smoke test** (image starts, CLI is reachable): + +```bash +just docker-smoke +# or: +docker run --rm asia-southeast2-docker.pkg.dev/a4dphase2/a4d/pipeline:latest \ + uv run a4d --help +``` + +**Level 2 — local pipeline run** (no GCS, process a local file): + +Mount a directory containing tracker files and run `run patient`. Output lands in +`/data/output` inside the container, which is the same mount so you can inspect it +afterward. + +```bash +docker run --rm \ + -v /path/to/trackers:/data \ + -e A4D_DATA_ROOT=/data \ + asia-southeast2-docker.pkg.dev/a4dphase2/a4d/pipeline:latest \ + uv run a4d run patient --file /data/your_tracker.xlsx +``` + +**Level 3 — full pipeline with GCP** (real GCS + BigQuery, no download): + +Mount your local Application Default Credentials so the container can authenticate. +Use `--skip-download` to process files already on disk instead of fetching from GCS. + +```bash +docker run --rm \ + -v /path/to/trackers:/data \ + -v "$HOME/.config/gcloud:/root/.config/gcloud:ro" \ + -e A4D_DATA_ROOT=/data \ + -e GOOGLE_CLOUD_PROJECT=a4dphase2 \ + asia-southeast2-docker.pkg.dev/a4dphase2/a4d/pipeline:latest \ + uv run a4d run --skip-download +``` + +This exercises the full upload path (GCS + BigQuery) without touching the live tracker +source bucket. + +### 6. Execute + +```bash +just run-job # trigger the Cloud Run Job +just logs-job # stream logs from the latest execution +``` + +After a code change, redeploy and run in one step: + +```bash +just deploy && just run-job +``` + +### 7. Schedule (optional) + +To run the pipeline on a schedule, create a Cloud Scheduler job that triggers it: + +```bash +gcloud scheduler jobs create http a4d-pipeline-weekly \ + --schedule="0 6 * * 1" \ + --uri="https://asia-southeast2-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/a4dphase2/jobs/a4d-pipeline:run" \ + --http-method=POST \ + --oauth-service-account-email=a4d-pipeline@a4dphase2.iam.gserviceaccount.com \ + --location=asia-southeast2 +``` + +The service account also needs permission to trigger Cloud Run Jobs for this: + +```bash +gcloud projects add-iam-policy-binding a4dphase2 \ + --member="serviceAccount:a4d-pipeline@a4dphase2.iam.gserviceaccount.com" \ + --role="roles/run.invoker" +``` diff --git a/a4d.Rproj b/a4d.Rproj deleted file mode 100644 index fcabf4c..0000000 --- a/a4d.Rproj +++ /dev/null @@ -1,26 +0,0 @@ -Version: 1.0 -ProjectId: 8e02b934-92ac-47d4-9d0f-16ea83cd14ae - -RestoreWorkspace: No -SaveWorkspace: No -AlwaysSaveHistory: Default - -EnableCodeIndexing: Yes -UseSpacesForTab: Yes -NumSpacesForTab: 4 -Encoding: UTF-8 - -RnwWeave: Sweave -LaTeX: pdfLaTeX - -AutoAppendNewline: Yes -StripTrailingWhitespace: Yes -LineEndingConversion: Posix - -BuildType: Package -PackageUseDevtools: Yes -PackageInstallArgs: --no-multiarch --with-keep.source -PackageCheckArgs: --no-manual -PackageRoxygenize: rd,collate,namespace - -QuitChildProcessesOnExit: Yes diff --git a/docs/CLAUDE.md b/docs/CLAUDE.md new file mode 100644 index 0000000..4cc708c --- /dev/null +++ b/docs/CLAUDE.md @@ -0,0 +1,91 @@ +# CLAUDE.md + +Python pipeline for A4D medical tracker data — processes Excel trackers into BigQuery tables. +Patient pipeline is complete and deployed to production (Cloud Run). + +## Module Overview + +| Module | Purpose | +|--------|---------| +| `extract/patient.py` | Read Excel trackers → raw parquet (openpyxl, multi-sheet) | +| `extract/product.py` | Read Excel trackers → raw product parquet (month sheets, stock section) | +| `extract/wide_format.py` | Mandalay wide-format handling (column expansion 2020-21; cell splitting 2017-19) | +| `clean/patient.py` | Type conversion, validation, transformations → cleaned parquet | +| `clean/product.py` | Product cleaning pipeline (R steps 2.0-2.21) → cleaned product parquet | +| `clean/schema.py` | 83-column patient meta schema matching R output | +| `clean/schema_product.py` | 20-column product meta schema + helpers | +| `clean/converters.py` | Safe type conversion with ErrorCollector | +| `clean/validators.py` | Case-insensitive allowed-values validation | +| `clean/transformers.py` | Explicit transformations (regimen, BP splitting, FBG) | +| `clean/date_parser.py` | Flexible date parsing (Excel serials, DD/MM/YYYY, month-year) | +| `tables/patient.py` | Aggregate cleaned parquets → static, monthly, annual tables | +| `tables/product.py` | Aggregate cleaned product parquets → product_data table | +| `tables/clinic.py` | Create clinic static table from reference_data/clinic_data.xlsx | +| `tables/logs.py` | Aggregate error logs → logs table | +| `tables/metadata.py` | Tracker metadata table (MD5 + per-tracker output presence flags) | +| `pipeline/patient.py` | Orchestrate extract+clean per tracker, parallel workers | +| `pipeline/product.py` | Product pipeline orchestration (mirrors patient, single product_data table) | +| `pipeline/tracker.py` | Per-tracker pipeline execution (patient + product) | +| `pipeline/models.py` | Result dataclasses | +| `gcp/storage.py` | GCS download/upload | +| `gcp/bigquery.py` | BigQuery table load | +| `gcp/drive.py` | Google Drive download (clinic_data.xlsx); file ID hardcoded in module | +| `reference/synonyms.py` | Column name synonym mapping (YAML) | +| `reference/products.py` | Stock_Summary product reference loader (known products, categories) | +| `reference/provinces.py` | Allowed province validation | +| `reference/loaders.py` | YAML loading utilities | +| `state/` | Reserved for incremental-processing logic (design in [migration/MIGRATION_GUIDE.md](migration/MIGRATION_GUIDE.md); not yet implemented) | +| `utils/` | Shared utilities | +| `config.py` | Pydantic settings from `.env` / `A4D_*` env vars | +| `logging.py` | loguru setup, `file_logger()` context manager | +| `errors.py` | Shared error types | +| `cli.py` | Typer CLI entry point | + +## CLI Commands + +Commands are grouped by process (`run` / `create` / `upload` / `download`), not by the object they act on: + +```bash +uv run a4d run # Full end-to-end pipeline (patient + product arms, drive/GCS/BigQuery) +uv run a4d run patient # Extract + clean + tables (local run, patient) +uv run a4d run product # Extract + clean + table (local run, product) + +uv run a4d create tables # Re-create patient/product/clinic/logs tables from existing cleaned parquets +uv run a4d create logs # Re-create only the logs table from existing pipeline log files + +uv run a4d upload tables # Upload tables to BigQuery (--only patient|product|clinic|logs|errors|metadata to restrict) +uv run a4d upload output # Upload output directory to GCS + +uv run a4d download trackers # Download tracker files from GCS +uv run a4d download clinic-data # Download clinic_data.xlsx from Google Drive into reference_data/ +``` + +Key options: `--file` (single tracker), `--workers N`, `--skip-tables`, `--skip-download`, `--skip-upload`, `--skip-drive-download`, `--skip-product`, `--incremental` (skip trackers matching previous run's manifest). + +## Output Directory Structure + +```text +output/ +├── patient_data_raw/ # Raw extracted patient parquets (one per tracker) +├── patient_data_cleaned/ # Cleaned patient parquets (one per tracker) +├── product_data_raw/ # Raw extracted product parquets (one per tracker) +├── product_data_cleaned/ # Cleaned product parquets (one per tracker) +├── tables/ # Final tables: patient_data_{static,monthly,annual}.parquet, product_data.parquet, clinic_data_static.parquet, table_logs.parquet, tracker_metadata.parquet +└── logs/ # Per-tracker log files (JSON) +``` + +## Key Facts + +- `clinic_id` = parent folder name of the tracker file +- Year detected from sheet names (`Jan24` → 2024) or filename +- Error sentinel values: numeric `999999`, string `"Undefined"`, date `"9999-09-09"` +- `ErrorCollector` accumulates row-level data quality errors; never raises +- `reference_data/` holds the pipeline's shared configuration (synonyms, validation rules, provinces); changing it changes cleaning behaviour for every tracker + +## Pipeline Status + +- **Patient pipeline**: complete, deployed to production +- **Product pipeline**: complete, deployed to production +- **Tracker metadata table**: generated on every `create tables` / `run` run (MD5 + output-presence flags) and uploaded to BigQuery `tracker_metadata`. +- **Incremental processing**: shipped 2026-05-01 behind the `--incremental` CLI flag (opt-in) on `run patient`, `run product`, and the bare `run`. Skips trackers whose MD5 + completion state match the previous run's manifest (BigQuery → local parquet → empty fallback). Default behaviour unchanged. See the `a4d.state` module. +- **The R pipeline it replaced**: retired 2026-08-24. Recover its source with `git show r-archive-removed^:r-archive/R/`; the migration's working papers are archived in [archive/](archive/README.md) and are not current guidance. diff --git a/docs/archive/MIGRATION_GUIDE.md b/docs/archive/MIGRATION_GUIDE.md new file mode 100644 index 0000000..0404b5f --- /dev/null +++ b/docs/archive/MIGRATION_GUIDE.md @@ -0,0 +1,287 @@ +# R to Python Migration Guide + +> **ARCHIVED — frozen, not current guidance.** See +> [`docs/archive/README.md`](README.md). This document planned the migration; +> the migration is done. Where it describes intent, read it as history. The R +> pipeline it maps onto was deleted on 2026-08-24 +> (`git show r-archive-removed^:r-archive/R/`). Frozen at tag +> `migration-archive-frozen`. + +Reference for the A4D pipeline migration from R to Python. + +**Final status**: all phases complete. Both arms (patient and product) are +implemented, deployed to production on Cloud Run, and verified cell-by-cell +against the frozen R baseline over 254 real trackers. Incremental processing +shipped 2026-05-01 behind `--incremental`. R retired 2026-08-24. +**Branch**: `migration` (285 commits ahead of `dev`, 364 files changed; +981 tests, 86% coverage at freeze) + +--- + +## Table of Contents + +1. [Strategy & Decisions](#strategy--decisions) +2. [Technology Stack](#technology-stack) +3. [Architecture](#architecture) +4. [Key Code Patterns](#key-code-patterns) +5. [Open Items](#open-items) + +--- + +## Strategy & Decisions + +### Goals +1. **Output Compatibility** — Generate equivalent parquet files (differences documented) +2. **Performance** — 2-5x faster than R +3. **Incremental Processing** — Only reprocess changed trackers (hash-based) +4. **Error Transparency** — Detailed per-row error tracking + +### Key Architectural Decisions + +**Per-Tracker Processing** — Process each tracker end-to-end, then aggregate +- Better for incremental updates; natural parallelization; failed tracker doesn't block others + +**No Orchestrator** — Simple Python + multiprocessing (not Prefect/doit/Airflow) +- DAG is simple: trackers → tables → BigQuery; less complexity, easier to maintain + +**BigQuery Metadata Table for State** — Not SQLite (containers are stateless) +- Query at pipeline start to get previous file hashes; only reprocess changed/new files; same table used for dashboards + +**Hybrid Error Logging** — Vectorized + row-level detail +- Try vectorized conversion (handles 95%+ of data); detect failures; log only failed rows with patient_id, file_name, error details; export error logs as parquet + +--- + +## Technology Stack + +- **uv** — Dependency management & Python version +- **ruff** — Linting & formatting +- **polars** — DataFrames (10-100x faster than pandas) +- **duckdb** — Complex SQL operations +- **pydantic** — Settings & validation +- **loguru** — Logging (JSON output) +- **pytest** — Testing +- **google-cloud-bigquery** — Replaces `bq` CLI +- **google-cloud-storage** — Replaces `gsutil` CLI +- **typer + rich** — CLI interface + +--- + +## Architecture + +### Data Flow + +``` +Excel Trackers (GCS) + | + v +download trackers # GCS → local data_root/ + | + v +run patient # For each tracker (parallel): + ├─ extract/patient.py # Excel → patient_data_raw/*.parquet + └─ clean/patient.py # raw → patient_data_cleaned/*.parquet + | + v +create tables # All cleaned parquets → + ├─ tables/patient.py # tables/static.parquet + | # tables/monthly.parquet + | # tables/annual.parquet + └─ tables/logs.py # tables/logs.parquet + | + v +upload output # local output/ → GCS +upload tables # tables/*.parquet → BigQuery +``` + +### Module Structure + +``` +src/a4d/ +├── extract/ +│ ├── patient.py # Patient: Excel → raw parquet +│ ├── product.py # Product: Excel month sheets → raw parquet +│ └── wide_format.py # Mandalay wide-format handlers (column/cell) +├── clean/ +│ ├── patient.py # Patient cleaning pipeline +│ ├── product.py # Product cleaning pipeline (R steps 2.0-2.21) +│ ├── schema.py # 83-column patient schema +│ ├── schema_product.py # 20-column product schema +│ ├── converters.py # Safe type conversion + ErrorCollector +│ ├── validators.py # Case-insensitive allowed-values +│ ├── transformers.py # Explicit transformations +│ └── date_parser.py # Flexible date parsing +├── tables/ +│ ├── patient.py # static/monthly/annual aggregation +│ ├── product.py # product_data aggregation +│ ├── clinic.py # clinic static table +│ └── logs.py # Error log aggregation +├── pipeline/ +│ ├── patient.py # Patient orchestration + parallel workers +│ ├── product.py # Product orchestration (mirrors patient) +│ ├── tracker.py # Per-tracker execution (patient + product) +│ └── models.py # Result dataclasses +├── gcp/ +│ ├── storage.py # GCS operations +│ ├── drive.py # Google Drive (clinic_data.xlsx) +│ └── bigquery.py # BigQuery load +├── reference/ +│ ├── synonyms.py # Column name mapping (YAML) +│ ├── products.py # Stock_Summary product reference loader +│ ├── provinces.py # Allowed province validation +│ └── loaders.py # YAML loading utilities +├── state/ # State management (exists, not yet wired up) +├── config.py # Pydantic settings from A4D_* env vars +├── logging.py # loguru setup +├── errors.py # Shared error types +└── cli.py # Typer CLI (run/create/upload/download command groups) +``` + +### State Management (Incremental Processing) + +``` +1. Container starts (stateless, fresh) +2. Query BigQuery metadata table + SELECT file_name, clinic_code, md5, complete FROM tracker_metadata +3. Compare with current file MD5s +4. Process only: new + changed + previously incomplete +5. Re-publish metadata table (full replace) at end of run +6. Container shuts down (state persists in BigQuery) +``` + +Wired up via the `a4d.state` module ([src/a4d/state/](../../src/a4d/state/)) and exposed +through the `--incremental` CLI flag on `run patient`, `run product`, and the +bare `run`. Source precedence is BigQuery → local +`output_root/tables/tracker_metadata.parquet` → empty manifest, so local devs +without `gcloud auth` get the local-parquet fallback automatically. + +The flag is **opt-in**: default behaviour is unchanged (process every tracker +found in `data_root`). Flipping the default to incremental is a separate +decision after a soak window. + +--- + +## Key Code Patterns + +### Configuration +```python +from a4d.config import settings +settings.data_root # Path to tracker files +settings.project_id # GCP project +settings.output_root # Local output directory +``` + +### Error Tracking +```python +# ErrorCollector accumulates failures without raising +error_collector = ErrorCollector() + +df = safe_convert_column( + df=df, + column="age", + target_type=pl.Int32, + error_value=settings.error_val_numeric, + error_collector=error_collector, +) +# Errors exported as parquet → aggregated into logs table +``` + +### Vectorized Conversion Pattern +```python +# Try vectorized conversion +df = df.with_columns(pl.col("age").cast(pl.Int32, strict=False)) + +# Detect failures (null after conversion but wasn't null before) +failed_rows = df.filter(conversion_failed) + +# Log each failure; replace with error value +``` + +### Avoiding R's rowwise() Pattern +```python +# R (slow): df %>% rowwise() %>% mutate(age_fixed = fix_age(age, dob, ...)) + +# Python (fast): vectorized +df = df.with_columns([ + fix_age_vectorized(pl.col("age"), pl.col("dob"), pl.col("tracker_year")).alias("age") +]) + +# Only iterate for genuine edge cases (log + replace) +``` + +### DataFrames (R → Python) +```python +# R: df %>% filter(age > 18) %>% select(name, age) +df.filter(pl.col("age") > 18).select(["name", "age"]) + +# R: df %>% mutate(age = age + 1) +df.with_columns((pl.col("age") + 1).alias("age")) +``` + +### GCP Operations +```python +# R: system("gsutil cp ...") +from google.cloud import storage +bucket = storage.Client().bucket("a4dphase2_upload") +bucket.blob("file.parquet").upload_from_filename("local_file.parquet") + +# R: system("bq load ...") +from google.cloud import bigquery +job = bigquery.Client().load_table_from_dataframe(df, table_id) +job.result() +``` + +### Logging +```python +from loguru import logger +logger.info("Processing tracker", file="clinic_001.xlsx", rows=100) + +# File-specific logging (like R's with_file_logger) +with file_logger("clinic_001_patient", output_root) as log: + log.info("Processing patient data") +``` + +--- + +## Completed Phases + +| Phase | Description | +|-------|-------------| +| 0 | Foundation: repo structure, uv, ruff, CI | +| 1 | Core infrastructure: reference, logging, config, ErrorCollector | +| 2 | Extraction: `extract/patient.py` (28 tests, 88% coverage) | +| 3 | Cleaning: `clean/patient.py` (83-column schema, full validation) | +| 4 | Tables: `tables/patient.py` (static, monthly, annual, logs) | +| 5 | Pipeline integration: `pipeline/patient.py` + parallel processing | +| 6 | GCP: `gcp/storage.py`, `gcp/bigquery.py`, CLI commands | +| 7 | Validation: 174 trackers compared, 8 bugs fixed, production verdict | +| 9 | Product pipeline: merged WIP product modules into `src/a4d/`; `run-pipeline` runs both arms. | + +--- + +## Open Items + +### Phase 8: First GCP Production Run + +- Run `run-pipeline` against production GCS bucket (patient data) +- Validate BigQuery table outputs match expected counts/schema +- Compare dashboard reports with R pipeline baseline +- Fix any issues discovered during first real run + +### Production Scheduling + +Cloud Run + Cloud Scheduler wiring (cron, image build, deploy manifest). The +state module shipped behind the `--incremental` CLI flag — see the +[State Management](#state-management-incremental-processing) section above. +The default behaviour remains "process every tracker"; flipping the default to +incremental is a separate decision after a soak window. + +--- + +## Reference Data + +All YAML files in `reference_data/` are shared with the R pipeline — do not modify without testing both: +- `reference_data/synonyms/synonyms_patient.yaml` +- `reference_data/synonyms/synonyms_product.yaml` +- `reference_data/data_cleaning.yaml` +- `reference_data/provinces/allowed_provinces.yaml` diff --git a/docs/archive/MR_DESCRIPTION.md b/docs/archive/MR_DESCRIPTION.md new file mode 100644 index 0000000..1cca20d --- /dev/null +++ b/docs/archive/MR_DESCRIPTION.md @@ -0,0 +1,814 @@ +# Migration: R to Python data pipeline (`migration` -> `dev`, PR #2) + +Replaces the R implementation of the A4D medical tracker pipeline with a Python +one covering both arms (patient + product), plus deployment, state management, +and an R/Python comparison harness used to verify the migration cell by cell. + +285 commits, 364 files changed. **981 tests, 86% coverage.** R retired +2026-08-24 (`r-archive/` deleted; reachable at +`git show r-archive-removed^:r-archive/R/`). + +> This document is live until PR #2 merges, then frozen with the rest of +> [`docs/archive/`](README.md) at tag `migration-archive-frozen`. + +**This MR is ready to merge.** Every ticket that gated it is closed: the +cell-by-cell R/Python triage over 254 trackers, the production verification +run, the dependency audit, the performance profile, R's retirement, and the +documentation overhaul. CI is green on the head commit. What remains open is +listed at the end and none of it blocks the merge. + +> Keep this document current: update it at the end of every working session, +> alongside [docs/wayfinder/map.md](../wayfinder/map.md). + +--- + +## Treasure map + +```mermaid +flowchart TD + subgraph DONE["Shipped and verified"] + A["Patient pipeline
extract - clean - tables"] + B["Product pipeline
extract - clean - table"] + C["Merged into one branch
PR #6, 2026-08-09"] + D["Cloud Run Job
GCS + BigQuery + Drive"] + E["Incremental processing
tracker_metadata + MD5"] + F["Production verification run
verified vs BQ snapshot"] + G["Perf profile
6.6x patient speedup"] + H["Dependency audit
19 CVEs cleared"] + I["Comparison harness
4 stages, run-over-run deltas"] + N["Merged headers propagated
screening columns recovered"] + end + + subgraph TRIAGE["R/Python triage - 57 of 65 tickets closed"] + J["Product cleaned: COMPLETE
20 unclassified, kept as signals"] + K["Product raw: COMPLETE
0 unclassified"] + L["Patient cleaned: COMPLETE
16 unclassified, all owned by open questions"] + M["Patient raw: COMPLETE
0 unclassified"] + M2["Patient raw column divergence: DONE
18,235 rows all accounted for"] + end + + subgraph OPEN["Still open - none block the merge"] + Q["34 - local checks match CI"] + R["35 - Polars 2.0 deprecations"] + T["16 - per-file log drill-down"] + X["40 - source-defect findings Excel"] + Y["41 - 2026 new Patient List fields"] + Z["65 - logs table values naming R scripts"] + end + + subgraph READY["Unblocked"] + V["6 - promote migration to dev"] + end + + subgraph BLOCKED["Blocked"] + W["9 - golden-master snapshot tests"] + end + + A --> C + B --> C + C --> D --> F + C --> I --> TRIAGE + L --> V --> W + + classDef done fill:#1a7f37,stroke:#116329,color:#fff + classDef open fill:#1f6feb,stroke:#0b3d91,color:#fff + classDef blocked fill:#6e7781,stroke:#424a53,color:#fff + classDef ready fill:#8250df,stroke:#5a32a3,color:#fff + class A,B,C,D,E,F,G,H,I,J,K,L,M,M2,N done + class Q,R,T,X,Y,Z open + class V ready + class W blocked +``` + +--- + +# Part 1 — The `a4d` package + +## What it does + +Reads A4D clinic Excel trackers (one workbook per clinic per year, one sheet +per month) and produces analysis-ready BigQuery tables. Two independent arms +run over the same workbooks: + +- **patient** — the per-month patient sheets plus the `Patient List` sheet +- **product** — the `INV` / stock section (insulin and supply movements) + +``` +GCS bucket Excel trackers per-tracker parquet tables BigQuery +a4dphase2_upload --> /_... --> patient_data_raw/ --> patient_static --> tracker.* +Google Drive .xlsx patient_data_cleaned/ patient_monthly +clinic_data.xlsx product_data_raw/ patient_annual + product_data_cleaned/ product_data + logs/ clinic_data_static + table_logs + table_errors + tracker_metadata +``` + +`clinic_id` is the tracker's parent folder name. The tracker year comes from +sheet names (`Jan24` -> 2024) or the filename. + +## Module map + +| Module | Purpose | +|---|---| +| `extract/patient.py` | Excel -> raw patient parquet (openpyxl, multi-sheet, two-row header merge) | +| `extract/product.py` | Excel -> raw product parquet (month sheets, stock section) | +| `extract/wide_format.py` | Mandalay wide-format handling (column expansion 2020-21, cell splitting 2017-19) | +| `clean/patient.py` | Type conversion, validation, transformations -> cleaned parquet | +| `clean/product.py` | Product cleaning (steps 2.0-2.21), running balance, chronological sort | +| `clean/schema.py` / `schema_product.py` | 83-column patient and 20-column product meta schemas | +| `clean/converters.py` | Safe type conversion with `ErrorCollector` | +| `clean/validators.py` | Allowed-value validation, canonical labels + alias map | +| `clean/transformers.py` | Regimen extraction, BP splitting, FBG conversion | +| `clean/date_parser.py` | Flexible date parsing (Excel serials, DD/MM/YYYY, month-year, typo rescue) | +| `tables/*.py` | Aggregate cleaned parquets into the final tables (patient, product, clinic, logs, metadata) | +| `pipeline/*.py` | Per-tracker and per-arm orchestration, parallel workers, result dataclasses | +| `gcp/*.py` | GCS download/upload, BigQuery load, Drive download, production-run verification | +| `reference/*.py` | Column synonyms, product categories, province validation (YAML in `reference_data/`) | +| `validate/*.py` | Source-vs-output reconciliation | +| `migration/compare.py` | R/Python comparison engine (Part 2) — retired, kept as the evidence record | +| `state/*.py` | Incremental processing: manifest, MD5 filter, source resolution | +| `config.py` | Pydantic settings from `.env` / `A4D_*` env vars | +| `cli.py` | Typer CLI | + +Row-level data-quality problems never raise: `ErrorCollector` accumulates them +and they land in `table_errors` / `table_logs` with an `error_code`. Sentinels +for unusable values are numeric `999999`, string `"Undefined"`, date +`"9999-09-09"`, each distinct from null, which means nothing was recorded. + +## Configuration + +Everything is a Pydantic setting, overridable via `.env` or `A4D_*` env vars: + +| Setting | Default | +|---|---| +| `A4D_DATA_ROOT` | the local tracker directory | +| `A4D_OUTPUT_DIR` | `output` (relative to `data_root`) | +| `A4D_PROJECT_ID` / `A4D_DATASET` | `a4dphase2` / `tracker` | +| `A4D_DOWNLOAD_BUCKET` / `A4D_UPLOAD_BUCKET` | `a4dphase2_upload` / `a4dphase2_output` | +| `A4D_MAX_WORKERS` | `4` | +| `A4D_ERROR_VAL_NUMERIC` / `_CHARACTER` / `_DATE` | `999999` / `Undefined` / `9999-09-09` | +| `A4D_MIN_TRACKER_YEAR` / `_MAX_TRACKER_YEAR` | `2017` / `2030` | + +## CLI + +Commands are grouped by *process*, not by the object they act on: + +```bash +uv run a4d run # full end-to-end: Drive + GCS download, both arms, tables, GCS + BigQuery upload +uv run a4d run patient # patient arm only, local +uv run a4d run product # product arm only, local + +uv run a4d create tables # rebuild all tables from existing cleaned parquets +uv run a4d create logs # rebuild only the logs table from existing log files + +uv run a4d upload tables # -> BigQuery (--only patient|product|clinic|logs|errors|metadata) +uv run a4d upload output # -> GCS + +uv run a4d download trackers # <- GCS +uv run a4d download clinic-data # <- Google Drive +``` + +Key flags on `run`: `--file` (single tracker), `--workers/-w`, `--skip-download`, +`--skip-upload`, `--skip-drive-download`, `--skip-patient`, `--skip-product`, +`--skip-tables`, `--incremental`, `--force`. + +`--incremental` skips trackers whose MD5 and completion state match the previous +run's manifest (BigQuery -> local parquet -> empty fallback); both arms see the +same filtered queue. `--force` wipes prior local outputs first and overrides +`--incremental`. + +## Usage scenarios + +```bash +# 1. First-time setup +uv sync && just hooks + +# 2. Debug one tracker end to end (no GCS, no upload) +just run-file "/path/to/2024_Mahosot Hospital A4D Tracker.xlsx" +just run-file-product "/path/to/2024_Mahosot Hospital A4D Tracker.xlsx" + +# 3. Full local run over everything in data_root, both arms, no cloud +uv run a4d run --skip-download --skip-upload --skip-drive-download + +# 4. Reprocess everything from scratch (what triage sessions use before comparing) +uv run a4d run patient --force +uv run a4d run product --force + +# 5. Pull the current production trackers down, process locally, upload nothing +just run-download + +# 6. Cheap daily-style run: only trackers that actually changed +uv run a4d run --incremental + +# 7. Tables only, from parquets already on disk +just create-tables + +# 8. Production -- see "Running in production" below +``` + +## Development commands + +```bash +just ci # format-check + lint + type-check + test, the same set CI runs +just test # unit tests (skips slow/integration) +just test-fast # no coverage, fail fast +just test-all # everything including slow + integration +just format / fix / lint / check +just sync / update / info / clean +just docker-build / docker-smoke / docker-push / docker-list / docker-clean +just job-settings # current Cloud Run CPU/memory/timeout/parallelism +``` + +## Running in production (GCP) + +The pipeline is **already deployed and running** in Google Cloud as a Cloud Run +Job — a one-shot container that downloads trackers from GCS, processes both +arms, uploads output to GCS and loads it into BigQuery, then exits. It has been +executed for real against production and verified against a pre-run snapshot. + +Everything lives in **`asia-southeast2` (Jakarta)** — Artifact Registry, the +Cloud Run Job, both GCS buckets and the BigQuery dataset. This is a data +residency requirement: patient data must not be processed or stored in the EU. +Bucket and dataset locations are fixed at creation time. + +| | | +|---|---| +| Job | `a4d-pipeline` (Cloud Run Job, `asia-southeast2`) | +| Image | `asia-southeast2-docker.pkg.dev/a4dphase2/a4d/pipeline:latest`, also tagged per git SHA | +| Resources | 8 vCPU, 8 GiB, 3600s task timeout, `A4D_MAX_WORKERS=8` | +| Service account | `a4d-pipeline@a4dphase2.iam.gserviceaccount.com` — `storage.objectViewer` on `a4dphase2_upload`, `storage.objectCreator` on `a4dphase2_output`, `bigquery.jobUser` + `bigquery.dataEditor` project-level | +| Storage | `A4D_DATA_ROOT=/tmp/data`, ephemeral in-container — nothing persists between executions | +| Base image | `python:3.14-slim`, `uv sync --frozen --no-dev`, default `CMD` is `a4d run` | +| Scheduling | Cloud Scheduler is **not** enabled yet; runs are triggered manually | + +Start, monitor, verify, roll back: + +```bash +just backup-bq # snapshot BigQuery tables (7-day expiry) -- the rollback point +just deploy # build, push, point the job at the new image +just run-job # trigger an execution +just logs-job # stream logs from the running execution +just job-settings # current CPU / memory / timeout / parallelism +just rollback abc1234 # revert the job to a previous git SHA +uv run python scripts/verify_production_run.py # live BigQuery tables vs the snapshot + +just deploy && just run-job # redeploy + run after a code change +``` + +`verify_production_run.py` (+ `src/a4d/gcp/verify.py`, unit-tested) compares row +counts, distinct clinic counts and schema against the `backup-bq` snapshot. The +first combined patient+product production execution passed it cleanly — all +four tables grew, 51 -> 53 clinics, no anomalies. + +Full instructions — one-time infrastructure setup (service account, IAM grants, +Artifact Registry, job creation), the three levels of local image testing before +deploying, and the optional Cloud Scheduler wiring — are in +[SETUP.md](../../SETUP.md). + +--- + +# Part 2 — The R/Python comparison harness + +The single most important tool in this migration. It is how "is the Python +pipeline right?" was turned into a number that goes down each session. + +Migration-only, deliberately **not** wired into `a4d.cli`: it has a defined +end of life at R's retirement (ticket 12). Engine in +`src/a4d/migration/compare.py` (pure, unit-tested), thin CLI in +`scripts/compare_outputs.py`. + +## How to run it + +```bash +just compare-outputs \ + "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_r" \ + "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_python" \ + output/comparison + +# equivalently +uv run python scripts/compare_outputs.py \ + --r-dir ".../output_r" \ + --py-dir ".../output_python" \ + --output-dir output/comparison \ + --only-mismatches # print only files with at least one measure flagged +``` + +It takes two *existing* output directories and diffs them — it never runs +either pipeline. The R side is a frozen baseline on the test-data drive; R is +not re-run (one deliberate exception, when the tracker set grew, framed as the +final capture before R is retired). + +## What it compares + +Four stages, each producing its own workbook, so a divergence can be localised +to extraction vs. cleaning: + +| Stage | Directory | Row-alignment key | +|---|---|---| +| Patient (raw) | `patient_data_raw/` | `patient_id` + `sheet_name`, ordinal tie-break | +| Patient (cleaned) | `patient_data_cleaned/` | `patient_id` + `sheet_name`, ordinal tie-break | +| Product (raw) | `product_data_raw/` | ordinal position within `(clinic_id, sheet)` | +| Product (cleaned) | `product_data_cleaned/` | ordinal position within `(clinic_id, sheet)` | + +Product has no natural identity key — `product_entry_date` is null on many rows +and collapsed the join, so `add_row_ordinal()` computes a positional key at +comparison time instead (never stored: the frozen R baseline cannot be re-run +to pick up a new column). + +Patient does have one, and keeps it: rows pair only when they are the same +patient on the same monthly sheet. The same ordinal breaks ties in the handful +of sheets that list a patient twice — by **content, not position**, since +cleaning reorders the copies. `RowAlignment.IDENTITY` vs +`RowAlignment.POSITIONAL` names which arm does which. + +Seven measures per file, coarse to fine: + +| Measure | What it answers | +|---|---| +| Shape match | same row count? Structural only. | +| ID divergence | patients/products present on only one side. Independent of the row key. | +| Column divergence | columns only on one side, plus dtype differences. | +| Categorical divergence | label values appearing on one side but never the other. | +| Totals divergence | numeric columns whose column-sum differs beyond tolerance. | +| Row-key divergence | rows that found no partner at all. **Read this before Cell divergence** — 0 cell mismatches can mean "everything agreed" or "nothing was paired". | +| Cell divergence | matched rows diffed value by value. | + +## Normalisation, before anything is called a mismatch + +Representation differences are not divergences, and they used to drown +everything else. Three normalisers run per stage, on columns declared in the +`STAGES` table: + +- `normalize_date_column` — R stores unparsed Excel serials as strings, Python + stores parsed dates. Both sides go through the pipeline's own + `parse_date_flexible`. *Product raw `product_entry_date`: 65,743 -> 91. + Patient raw overall: 564,096 -> 46,788.* +- `normalize_numeric_column` — R and Python round float-to-string differently. + Parse both back to `float` so the tolerance applies. On the patient raw + stage the target columns come from each frame itself, not the cleaned + schema, which cannot name the Patient List join's `.static` copies or a + measurement column typed as a string. *434 mismatches -> 0.* +- `normalize_whitespace_column` — readxl's `trim_ws=TRUE` strips what openpyxl + keeps, and represents an embedded line break as `\r\n` vs `\n`. On the + patient raw stage its target columns come from each frame itself, for the + same reason as the numeric list: the cleaned schema cannot name a raw-only + column (`dm_complications`) and types as `Float64` columns the raw stage + still holds as text (`insulin_injections`, `hba1c_updated`). *40 mismatches + -> 0.* +- `normalize_boolean_literal_column` — readxl writes an Excel boolean as + `FALSE`, openpyxl as `False`. Only the exact literals are folded. Raw stage + only; cleaning already canonicalizes both. *20 mismatches -> 0.* + +## Cause classifiers + +Every cell mismatch is run through a per-column registry and labelled with a +cause, or left `unclassified`. ~20 classifiers exist, each named after the +*mechanism* it identifies, e.g.: + +`r_extraction_gap`, `r_non_latin_header_miss`, `r_category_lookup_miss`, +`r_insulin_dedup_drop`, +`r_join_suffix_collision`, `r_ifelse_na_propagation`, `r_date_error_sentinel`, +`r_numeric_error_sentinel`, `row_order_divergence`, +`derived_running_total_row_order`, `buddhist_era_typo`, +`python_buddhist_era_converted`, `buddhist_era_conversion_row_order`, +`python_canonical_label`, `python_future_date_sentinel`, +`stray_date_zeroed`, `wide_format_fragment`, `r_drops_richtext_space`, +`python_trims_merged_subvalue`, `r_ymd_first_misparse`, +`python_rejects_beyond_tracker_year`, `r_unicode_sanitizer_rejects_accent`. + +**`unclassified` is the number that matters.** A classifier records that a +difference is *understood*, not that Python won. Where a difference is +genuinely undecidable, or the source file itself is corrupt, that is recorded +as the answer rather than papered over with a label. + +## What a run writes + +``` +output/comparison/2026-08-16T231826Z/ # one self-contained folder per run +├── compare_report_patient_data_raw.xlsx +├── compare_report_patient_data_cleaned.xlsx +├── compare_report_product_data_raw.xlsx +├── compare_report_product_data_cleaned.xlsx +└── snapshot_.json # per-column / per-cause counts +``` + +Each workbook carries summary sheets (per-column and per-cause mismatch counts, +files only in R, files only in Python) and one detail sheet per measure: +`column_divergence`, `id_overlap`, `categorical_overlap`, `row_key_overlap`, +`totals`, `cell_mismatches`. Excel rather than HTML on purpose — triage means +loading the result as a dataframe, filtering, sorting and adding columns. + +The console prints the same summary plus a **run-over-run delta** (red/green) +against the previous run folder's snapshot for that stage, so a fix's effect is +visible by count without needing a classifier to prove it worked. + +## How it was actually used + +The loop each session: + +1. `just compare-outputs ...` -> pick the largest `unclassified` population in + the `cell_mismatches` sheet. +2. Filter that column in Excel, look for the shape of the difference + (R-null vs Python-has-value? sentinel? ordering?). +3. **Open the real source Excel workbook** and read the cell. This is the step + that mattered — a shape-matching heuristic is not a diagnosis. Nearly every + real bug in the table below was found here, not in the report. +4. Also read R's own source when the mechanism lives there. This was + `r-archive/` for the whole of the triage; the archive is now deleted, and the + same files are reachable at `git show r-archive-removed^:r-archive/R/`. +5. Then either: fix Python (most sessions), or add a named classifier + explaining the mechanism, or record it as an open question. +6. Re-run the pipeline (`a4d run patient --force`) and the comparison; the + delta shows the effect. +7. Whatever did not converge is split into a new ticket rather than left + sprawling. + +Rules that made it work, learned the hard way: + +- **Triage means deciding, not labelling.** Two bars: explain the actual + mechanism, and say what you checked. +- **Not 1:1 R parity.** R can be wrong and often is. The arbiter is the source + workbook, not R's output. +- **A corrupt source is a valid terminal answer** — "this tracker needs human + inspection" is a finding, not a failure. +- **Split rather than sprawl.** 39 tickets exist because sessions ended by + handing the residual forward with its numbers attached. + +Ticket 32 exists to re-audit every classifier written before that bar was set. + +--- + +# Part 3 — Results + +## Real defects found and fixed + +Triage was overwhelmingly a bug-hunt, not a labelling exercise. The ones that +changed production output: + +| Fix | Effect | +|---|---| +| `_fix_t1d_diagnosis_age` recomputed from dates, discarding recorded ages | 25,968 -> 4,807 mismatches | +| `merge_headers` left the 2022 template's `Updated 2022` header unmapped | recovered 7,165 blood-pressure / education dates | +| `_apply_type_conversions` split on the first space; dateutil then completed from *today* | non-deterministic output, real dates destroyed | +| `parse_date_flexible` deleted the 4th letter of month names (`March` -> `Marh`) | every full month name was unparseable | +| Date path knew 4 missing-value markers where the numeric path knew 11 | sentinel-stamped date cells 6,186 -> 1,994 | +| `extract_regimen` lowercased every unmatched value (`NPH` -> `nph`) | live data corruption | +| `validate_allowed_values` picked the last of two identically-sanitising spellings | now a loud config error; canonical labels declared in config | +| `remove_header_rows` missed rows blank except one formula-emptied cell | row insertion/shift across product raw | +| `read_patient_rows` accepted a row on its ID alone, turning a stray list of patient IDs below the data block into records | 24 invented patient-months, now 0 | +| Inconsistent whitespace trimming across both arms | recovered 72 rows of patient `sex` | +| `find_data_start_row` was O(n^2) on read-only worksheets | 6.6x speedup, 145.8s -> 22.0s | +| `clean_product_data` crashed on pre-product-tracking trackers | 4 trackers now yield empty schema-conformant output | +| `run-pipeline` aborted the whole run on one patient tracker failure | soft-fail-and-continue, both arms | +| Columns with data under an empty header cell were dropped silently | 17 sites recovered from the sheets that label them; 194 insulin-regimen rows back on one tracker | +| `rename_columns` kept only the first of several source columns sharing a canonical name | 2,489 recorded screening values recovered across 27 trackers | +| Glucose recorded in mmol/L under an mg/dL header, and no analytical bounds on any FBG column | 29 columns corrected, 5,673 readings flagged, 832 impossible readings rejected | +| A tracker changing shape mid-year was invisible | `tracker_layout_changed` flags it: 37 trackers, 284 positions, incl. one clinic's baseline FBG silently dropped for 5 months | +| `read_patient_rows` returned openpyxl's datetime for a number typed into a date-formatted cell, which the numeric conversion then sentinelled | 24 real systolic readings were reaching BigQuery as 999999; now recovered from the Excel serial | +| A header merged across a block never reached the sub-headers past a blank column, and blank-header recovery then filed a screening selection under `observations` | complication-screening results and dates recovered (11 -> 0 and 31 -> 0 mismatches); merged spans read from the sheet XML, so no read-write workbook load | + +Also added: a `balance_reconciliation` error code that fires when the computed +closing stock contradicts the tracker's own recorded total (113 groups across +21 files), and a `blank_header_with_data` error code for columns holding real +values under an empty header cell — a clinician's slip that silently costs the +column (26 trackers, the clearest being 194 insulin-regimen rows in +`2021_Kantha Bopha`). Where the tracker's other month sheets unanimously label +that column, `recover_blank_headers` takes the name from them and the data +survives; where they don't, the site is reported for correction in the source +workbook, which is what the planned source-defect report (ticket 40) exists to +drive. + +Alongside it, `tracker_layout_changed` flags any position whose meaning is not +stable across a tracker's month sheets — a tracker is one workbook for one +clinic-year and should not change shape partway through. Of the 359 disagreeing +positions, 75 are pure renames the synonym file already absorbs; the other 284 +(37 trackers) change the canonical column. This is also why deriving a single +header set per tracker and applying it positionally was rejected: `2018_CDA`'s +April sheet is missing the `Insulin Regimen` column entirely, shifting seven +columns left for that month. + +## Where verification stands + +Current baseline: `output/comparison/2026-08-21T215325Z`, 254 trackers. +Earlier counts on the wayfinder map were measured against smaller tracker sets +and should be read as historical. + +| Stage | Mismatches | Unclassified | +|---|---|---| +| Product (raw) | 89 | **0** | +| Product (cleaned) | 22,706 | **20** (kept on purpose as signals) | +| Patient (cleaned) | 114,284 | **16** | +| Patient (raw) | 26,171 | **0** (from 14,844) | + +**All four stages are now triaged.** Patient's cleaned stage has **99.99%** of +its flagged cells explained; the 16 left are not unexplained but deliberately +withheld — 13 belong to the standing null-vs-`Undefined` question, 2 are +R-null against Python's `999999` where neither side published a reading, and 1 +is R's inability to read a slash-separated month. Each is a decision about what +the pipeline *should* publish, not a difference nobody understands. + +--- + +# Still open + +**None of this blocks the merge.** It is the work the wayfinder map carries +forward past this PR. + +**Frontier (takeable now)** + +**Closed since this section was last written** + +- **64 — the documentation overhaul, now closed.** Every module outside + `src/a4d/migration/` explains itself in its own terms: `src/` 174 R references + -> **0**, `tests/` 46 -> **0**, markdown 65 -> **0**. The comparison package + keeps its 179 + 42 and is marked *historical* in its own module docstring — + R divergence is its subject, and it will never run again. `docs/migration/` + moved to `docs/archive/` with an index, each document corrected to the true + current state before being frozen; `docs/VALIDATION_SUMMARY.md` deleted as a + superseded 174-tracker verdict. New CI guard + `tests/test_docs_have_no_r_framing.py` stops regressions. Suite **1,080 + passed**, 86% coverage. + +- **12 — retire R from the workspace, now closed. `r-archive/` is deleted**: 156 + tracked files, 1.9M, gone from the working tree and recoverable through the + annotated tag `r-archive-removed` (`r-archive-removed^` is the last commit + containing it). `CLAUDE.md`'s "R Archive / do not modify" section is replaced + by that recovery recipe; `docs/CLAUDE.md`, `SETUP.md`, `pyproject.toml`'s ruff + exclusion, `.dockerignore` and `.gitignore` all updated. Inlining the cited R + excerpts first was rejected once the citation count was measured properly — + **35 across 12 modules**, not 8 in one — which is what became ticket 64. + +- **44 — cleaned-stage FBG cells where R has nothing, now closed. It ends the + patient cleaned-stage triage: 2,955 unclassified cells -> 16, every one of + the 16 already owned by an open question.** The ticket described one + population and a file-level obstacle; scanning the whole column first found + the decisive fact is row-level and much stronger. **Neither pipeline reads + the mmol glucose column from the workbook** — both derive it from the mg cell + beside it (`convert_glucose_units` in Python, `fix_fbg`'s `fbg/18` in R) — so + an mmol divergence is an mg divergence restated, and all 2,935 cells, without + exception, sat beside an mg cell that already carried a named cause. + + The reframing surfaced **110 cells running the other way that no ticket had + described**: Python null, R holding a value, each exactly `mg_r / 18`. R + divides a number `fix_fbg` manufactured out of text (140 from "Lost follow + up") or one far past the analytical ceiling (2013 mg/dL) and publishes the + quotient as a measurement — `2023_CDA` gets **111.8 mmol/L**, above the level + A4D's medical advisor called impossible. A further 28, also unlooked-at, run + a third way: 2018 CDA writes `148 mg/dl (Mar-18)`, R's `as.numeric` fails + on the whole string and Python reads it. Python is the correct side in all + three. `mmol_derived_from_mg_sibling` names the cascade rather than + re-deciding it, bounded by the ÷18 identity so it cannot claim an mmol cell + carrying a reading its mg sibling does not account for. + + It also closed a standing tooling limitation. `column_unit_swapped` — read + back from the run's own `glucose_unit_swapped` error records rather than + restated in the comparison — lets the range test judge a reading against the + unit the column *holds* rather than the one its name claims, which resolves + the six 2020 Kantha Bopha cells round 10 could only write out in prose + (`fbg_baseline_mg` unclassified 6 -> 0). Verified across two full runs + against the real 254-tracker data: `per_column` counts byte-identical, the + other three stage snapshots unchanged — nothing suppressed, only named. + +- **57 — patient cleaned triage, round 10, now closed.** It ended the + ten-round chain: 96 in-scope cells -> **19**, all of them carrying written + verdicts (13 the standing `Undefined` question, 6 a named tooling + limitation), so there is no round 11. The last real Python defect was one + character: R's `extract_date_from_measurement` makes the closing parenthesis + optional and Python's did not, so `180(May-2017` lost its date. 24 cells + recovered. What follows is the round-9/10 picture as it stood, which is where + the 96 came from — 96 in-scope cells left after + round 9, excluding the FBG population ticket 44 owns and the + `hospitalisation_date` population ticket 39 owns. Unlike every round before + it these share no mechanism: ~27 are 2017/2018 trackers where R lifts a date + out of the measurement cell (`8.53 (28/8/2017)`) and Python has no equivalent + path; 18 are one undecidable cell (`25-Ma4-2025` — Mar or May, and R answers + April); ~19 are glued digit groups (`26/102022`, `10/1023`) where the missing + separator's position is a guess; 14 are a numeric tail, two of whose FBG + values sit *inside* the analytical bounds and should not have been + sentinelled; and 13 are `Undefined` cells that belong to a standing open + question rather than to triage. + + Round 9 closed the whole nine-column date family as **one R mechanism**, and + it was established by installing lubridate and *executing* R rather than + reading it. `parse_dates` deletes the **fourth letter** of any word of four + or more letters — `April-17` becomes `Aprl-17` — and then walks a fixed order + list ending in `my` and `y` that cannot fail loudly. So a spelled-out month + collapses to 1 January of its year (88 cells), an unreadable month makes R + read the **day** as the month — `9-Dce-20` becomes 2020-09-01 (143 cells) — + and a day past 12 leaves R with no reading at all (107 cells). Every + reproduction matched R's frozen output exactly. Python is the correct side of + all three. + + The same investigation exposed **three Python gaps, all real data loss, all + fixed**. Month spellings Python did not know: the Bahasa Malaysia `Mac`, + `Mei` and `Okt` that the Malaysian clinics write in a column every other + clinic writes in English, all twelve Thai abbreviations for Nakornping's + trackers, plus the transposition `Dce` and the dropped letter in `ug`. + Separator runs damaged by a stray keystroke — `26-05- 2007`, `19-Jan_2023`, + `02-Apr=-2026`, `23/05//2025` — which R recovers because lubridate splits on + any non-alphanumeric run where dateutil needs a well-formed separator. And + zero-width characters pasted in from another application. Checked old parser + against new over all **5,187** distinct raw date strings on the 254-tracker + set: 32 changed, every one from the sentinel to a real date, and no + already-parsing value altered its reading. + + **The repair's narrowness was forced by measurement, not caution.** The first + version — lubridate's own rule, any non-alphanumeric run is a separator — + turned `11-15 /01/2019`, a *range* of two visit days, into a single date of + 2001-11-15. The repair now runs only when at most three numbers remain, which + is also why `26/102022` and `10/1023` are deliberately still rejected. + + `insulin_subtype`'s 11 cells were traced and deliberately **left + unclassified**. R's derivation is a chain of `ifelse(x == "Y", ...)`: an + all-`-` row yields `""` and becomes null, an all-null row yields `NA` and + becomes `Undefined`. R's apparent distinction is NA propagation, not design, + so it cannot be cited as evidence for what the output should say — which + makes these cells part of a standing open question, not a residual. + +- **47 — four trackers where cleaning merged several patients into one ID, now + closed.** The pipeline was publishing `KH_NPH02` — an identifier in no source + workbook — with four different patients' September records filed under it. + Cause: `fix_patient_id` inherited R's rule of truncating any malformed ID + over 8 characters to its first 8, and `2023_NPH`'s `Sep23` sheet types a + stray `H` into four IDs its own Patient List and three other month sheets + spell correctly. **Truncation is now dropped** — a divergence from R, taken + deliberately: a malformed ID is recovered against the well-formed IDs the + same tracker carries (edit distance 1, unique candidate only) and sentinelled + to `Undefined` otherwise, so the pipeline never invents an identity. Measured + by re-cleaning all 254 raw parquets: one file changes, four patients each + regain their September row, and zero non-conforming IDs remain corpus-wide. + The other three files of ticket 45's count turned out to be two correct + merges (hyphen-vs-underscore spellings of one patient) and one unrepairable + source defect (`2026_NOGH`'s `MM_NO97/98/99`, written that way in its own + Patient List, three patients still sharing `Undefined`). + +**Frontier, continued** + +- **64 — rewrite every docstring and doc that explains the code by what R did.** + Spawned by 12 and the only frontier ticket on the route to the merge. + Documentation that justifies a behaviour with "to match R" was fine as working + documentation during the migration and is wrong now that R is gone: it + explains the code by something that no longer exists. **35 citations across 12 + Python modules**, including production cleaning code — not the 8 in + `compare.py` that ticket 12's inventory claimed. Not a find-and-replace: round + 4 found `_validate_dates` claiming its future-date guard "matches R pipeline + behavior" when R has no such guard at all, so a stale citation can also be + wrong about R. Also has to decide the fate of `src/a4d/migration/compare.py` + itself, whose subject is a pipeline no longer in this repo. +- **34 — make the local pre-push checks match CI.** CI was red for four days + unnoticed because the locally-run check set was a strict subset. +- **35 — 17 Polars 2.0 deprecation warnings.** Each asks about a behaviour + change; they need decisions, not silencing. +- **40 — one Excel of every source-tracker defect**, so the workbooks + themselves can be corrected rather than guessed at in the pipeline. +- **41 — the 2026 template's five new Patient List fields**: in or out. +- **16 — per-file log drill-down.** Replaces `LogViewerA4D`'s job. Not yet + decided whether it gates rollout or is a nice-to-have. +- **40 — one Excel of every source-tracker defect** (file, sheet, patient, row, + finding), so the workbooks themselves get corrected rather than the pipeline + guessing. Inherits a backlog of already-verified findings; may turn out to be + the same deliverable as 16. +- **41 — the 2026 template's five new Patient List fields** (`Phone Number`, + `Insurance Card Status`, `Current Insulin Regimen`, `BGM A4D`, `Insulin A4D`). + Neither pipeline carries them forward. Left for the data owner to decide + after looking at the new trackers. + +**Blocked** + +- **6 — promote `migration` into `dev`** (this PR). **Fully unblocked**: every + ticket that gated it is closed, including 64. Nothing on this map now stands + between here and the merge. + +**Frontier, not blocking the merge** + +- **65 — two values published into the BigQuery `logs` table still name R + scripts**: `function_name="read_product_data_step1"` (the only one of twenty + that is not the emitting Python function) and `script="script1"`/`"script3"` + in a column whose default is the stage name `"clean"`. These are *data*, not + comments, so ticket 64 recorded them rather than changing published output. + Deliberately not wired as a merge blocker — inconsistency, not falsehood. +- **9 — golden-master/snapshot regression tests.** Deliberately deferred until + after promotion. + +**Two production defects fixed by the classifier audit (ticket 32)** + +- `_validate_entry_dates` exempted every parsed year `>= 2400` so Thai + Buddhist-era dates could flow through, which also let corrupt Excel serials + (1,339,576 and 411,384) reach the product table as `5567-08-19` and + `3026-04-30`. The exemption is now the tracker's own BE band, so all 22 + genuine BE dates survive and the 3 corrupt cells are sentinelled and logged + under a new `implausible_era_date` error code. +- `a4d run` published an errors table holding the **patient arm only** — the + patient arm writes it from inside `run_patient_pipeline` and nothing wrote + the product arm's. BigQuery's `errors` table has therefore never carried a + product finding (e.g. zero of the 116 `balance_reconciliation` records). + `run` now writes it once after both arms: 63,295 -> 97,326 records. + +**Buddhist-era dates are converted, not published as written (ticket 61)** + +Thai clinics keep their trackers in a Thai-locale Excel, so dates arrive with a +Buddhist-era year (BE = CE + 543) — the calendar the clinic uses, not an error +it made. The cleaned stage now shifts them to Gregorian on both arms and logs +each one under a new `buddhist_era_converted` code; the raw stage still keeps +what the workbook says. + +- **375 patient cells recovered.** `_validate_dates` saw a year centuries ahead + and clobbered every one with the 9999-09-09 sentinel, so these readings were + destroyed rather than published oddly — and invisible to the comparison, + because R sentinels them too. +- **22 product rows converted.** `product_data` no longer carries a stock + movement dated 543 years in the future. +- **6 cells that decode to no plausible year stay sentinelled** (`3035-03-01`, + `5025-05-19`), as do two `hospitalisation_date` cells whose note names a date + *range*. All are reported as source defects instead. + +**A cleared row number no longer costs a whole month sheet (ticket 59)** + +`find_data_start_row` scanned column A for the first *numeric* cell. Where a +clinician clears a patient's row number, Excel keeps a whitespace-only string in +its place — so the scan started one row late, read the header rows from the +data, found no `patient_id` after harmonization, and skipped the sheet entirely. + +- **15 patients' October records recovered** on `2022_Children's Hospital 2`, + whose `Oct22` sheet was being dropped whole. +- The rule is deliberately narrow: only a whitespace-only cell *directly + abutting* the numeric block counts. Matching R's looser "first non-empty cell" + rule was measured across all 254 trackers and rejected — it would start 14 + sheets at row 1 on a stray letter in `A1`. + +**A row whose patient ID is a broken formula is reported, not silently dropped +(ticket 59)** + +Rows whose ID cell holds `#REF!` were discarded with no error record. They are +still discarded — `Undefined` is a bucket rather than an identity, and keeping +them would pool many patients' measurements under one group key — but each +discard now emits the new `excel_error_patient_id` code so the workbook can be +repaired. + +- **120 rows across 9 trackers**, the largest being `2026_Preah Kossamak`'s + `May26` sheet (98 rows, an entire month, carrying ages, FBG readings, HbA1c, + weight, height and insulin regimens). + +**Patients whose ID is spelled two ways keep their demographics (ticket 58)** + +Both whole-tracker joins in extraction — the `Patient List` demographics and +the `Annual` sheet — attached on the raw `patient_id`, while the fix that makes +a respelled ID usable runs later, in cleaning. Where a month sheet spells a +patient differently from the Patient List in the same workbook, the join could +not match: the row kept its measurements and lost every static column. Both +joins now key on a derived normalized ID; the raw `patient_id` column is +untouched. + +- **680 rows / 6,863 Patient List cells recovered** across 6 trackers, plus 8 + rows / 40 cells from the Annual join. `dob`, `sex`, `province`, + `t1d_diagnosis_date`, `hba1c_baseline`, `recruitment_date`, + `patient_consent`, `lost_date` and `status_out` among them. +- **672 of those rows are the 2023 and 2024 Mahosot trackers**, whose month + sheets write `LA-MH056` and up with a hyphen against a Patient List spelled + `LA_MH056` — 38 patients in 2024, 27 in 2023. +- R has the same gap, so the comparison never showed it: both sides were null. + Verified against R's own frozen cleaned output. +- **No fan-out risk**: measured across all 192 Patient-List-bearing trackers, + zero have two entries folding to one normalized key. +- The 15 rows that still miss are source defects — `2023_NPH`'s stray `H`, and + `2024_Mandalay General`'s `MM_YG013_MG`, which that Patient List does not + contain under any spelling. + +**Naming what that fix changed, on the stage that cannot see it (ticket 63)** + +The cause above recognises a cell by the month row's own ID still carrying the +hyphen — and cleaning normalizes exactly that, on both sides. So the fix took +the cleaned stage from 16 unexplained cells to **4,949**. The comparison now +derives the affected identities per file from the run's own *raw* output and +carries them to the cleaned stage as a flag; production output is untouched. +Cleaned `unclassified` is back to **16**, with the four-stage totals unchanged. + +- **A column R lost file-wide is a different defect**, and the naive wiring + claimed 43 files instead of 6. 2024 Preah Kossamak reads zero + `recruitment_date` in 863 rows (R's `xml:space` header defect) where 2024 + Mahosot reads 672 of 1,068. A new `r_column_empty_in_file` guard keeps 2,336 + cells off the wrong label. +- **Four existing causes were silently holding join-miss cells** and handed + back 671 `recruitment_date`, 677 `fbg_baseline_mg`, 224 each of `edu_occ` + and `edu_occ_updated`, and 9 per blood-pressure column — every figure now + matching the raw stage's own count for that column, computed a different way. +- **`age` is derived, so it got its own cause.** `_fix_age_from_dob` overrides + the sheet's Age cell with the D.O.B.-derived age; R has no D.O.B. for these + patients (null in all 275 rows) and keeps the clinic's typed figure. 113 + cells; Python is the correct side. + +**Known and accepted** + +- The frozen R baseline covers 254 trackers via a documented, reversible rename + map; new clinics added after the last R run have no R counterpart and show as + Python-only. +- `compare_columns` deliberately flags every dtype difference, including + harmless representation artifacts (R `Float64` vs Python `Int32` on integer + columns); these are documented rather than normalised away. +- A local, untracked `a4d-python/` directory at the repo root is a stale copy + predating the current `src/` layout — not in git, pending a decision to delete. + +Full history, per-ticket evidence and every decision: +[docs/wayfinder/map.md](../wayfinder/map.md). diff --git a/docs/archive/PRODUCT_DATA_PIPELINE_FEATURE.md b/docs/archive/PRODUCT_DATA_PIPELINE_FEATURE.md new file mode 100644 index 0000000..58d613a --- /dev/null +++ b/docs/archive/PRODUCT_DATA_PIPELINE_FEATURE.md @@ -0,0 +1,146 @@ +# Feature Proposal: Product Data Processing Pipeline + +> **OBSOLETE — historical reference only.** The product data pipeline described here was implemented and merged into `src/a4d/` on 2026-04-23. This document predates that work and is preserved for context on the original proposal; the "Problem Statement", "Implementation Plan", "Success Criteria", and "Questions for User" sections no longer reflect reality. For current state, see [docs/CLAUDE.md](../CLAUDE.md). Frozen at tag `migration-archive-frozen` — see [`docs/archive/README.md`](README.md). + +## Summary + +Implement a complete product data processing pipeline in the Python codebase to match the functionality of the R pipeline. This is a critical gap as the R pipeline processes both patient AND product data, while the Python pipeline currently only handles patient data. + +## Problem Statement + +The Python pipeline is missing product data processing capabilities that exist in the R pipeline: + +1. **R Pipeline has extensive product data processing:** + - `script1_process_product_data.R` - Extract product data from Excel + - `script2_process_product_data.R` - Clean product data + - `script3_create_table_product_data.R` - Create product data table + - `script3_link_product_patient.R` - Link product and patient data + - `helper_product_data.R` - Product data utilities + - `read_product_data.R` - Product data reading functions + +2. **Python Pipeline only has patient data processing:** + - `extract/patient.py` - Extract patient data + - `clean/patient.py` - Clean patient data + - `tables/patient.py` - Create patient tables + - **No product data modules exist** + +3. **Infrastructure already exists for product data:** + - `reference/synonyms.py` has `load_product_mapper()` function + - `gcp/bigquery.py` includes `product_data` in table schemas + - `reference_data/synonyms/synonyms_product.yaml` exists + +## Proposed Solution + +Implement a complete product data processing pipeline with the following components: + +### 1. Product Data Extraction (`src/a4d/extract/product.py`) + +Extract product data from Excel tracker files: + +- Read product sheets from Excel files +- Use product column mapper to rename columns +- Extract product information (product name, units, dates, etc.) +- Handle multiple product entries per tracker +- Export to parquet format + +### 2. Product Data Cleaning (`src/a4d/clean/product.py`) + +Clean and validate product data: + +- Validate product names against reference data +- Clean product units and quantities +- Parse product dates +- Validate product balances +- Track data quality errors +- Handle missing values + +### 3. Product Data Table Creation (`src/a4d/tables/product.py`) + +Create final product data tables: + +- Merge all cleaned product parquet files +- Create product_data table with proper schema +- Link product data with patient data +- Calculate product balance status +- Export to parquet for BigQuery upload + +### 4. Product Data Pipeline (`src/a4d/pipeline/product.py`) + +Orchestrate product data processing: + +- Discover tracker files with product data +- Process product data in parallel (like patient pipeline) +- Create product tables +- Integrate with existing pipeline + +### 5. CLI Commands (`src/a4d/cli.py`) + +Add product data CLI commands: + +- `process-product` - Run product data pipeline +- `create-product-tables` - Create tables from existing cleaned data +- Update `process-patient` to optionally process product data + +## Architecture + +``` +Product Data Pipeline Flow: +1. Discover tracker files with product sheets +2. For each tracker (parallel): + - Extract product data from Excel → raw parquet + - Clean raw data → cleaned parquet +3. Create final product table from all cleaned parquets +4. Link product data with patient data +5. Upload to BigQuery +``` + +## Implementation Plan + +### Phase 1: Core Product Data Processing + +1. Create `src/a4d/extract/product.py` - Product data extraction +2. Create `src/a4d/clean/product.py` - Product data cleaning +3. Create `src/a4d/tables/product.py` - Product table creation +4. Create `src/a4d/pipeline/product.py` - Product pipeline orchestration + +### Phase 2: CLI Integration + +5. Add `process-product` command to CLI +2. Add `create-product-tables` command to CLI +3. Update `process-patient` to support product data + +### Phase 3: Testing & Documentation + +8. Add unit tests for product data modules +2. Add integration tests for product pipeline +3. Update documentation + +## Benefits + +1. **Feature Parity**: Python pipeline matches R pipeline functionality +2. **Complete Data Processing**: Process both patient AND product data +3. **BigQuery Integration**: Product data table can be uploaded to BigQuery +4. **Data Linking**: Link product usage with patient outcomes +5. **Inventory Tracking**: Track medical supply distribution + +## Dependencies + +- Existing product mapper: `load_product_mapper()` in `reference/synonyms.py` +- Product synonyms YAML: `reference_data/synonyms/synonyms_product.yaml` +- BigQuery schema: `product_data` table already defined in `gcp/bigquery.py` + +## Success Criteria + +- [ ] Product data can be extracted from Excel tracker files +- [ ] Product data is cleaned and validated +- [ ] Product data table is created with correct schema +- [ ] Product data can be linked with patient data +- [ ] Product data can be uploaded to BigQuery +- [ ] All tests pass +- [ ] Documentation is updated + +## Questions for User + +1. Should product data processing be integrated into the existing `process-patient` command, or should it be a separate `process-product` command? +2. Are there any specific product data validation rules that should be added? +3. Should product data processing be enabled by default, or require an explicit flag? diff --git a/docs/archive/PYTHON_IMPROVEMENTS.md b/docs/archive/PYTHON_IMPROVEMENTS.md new file mode 100644 index 0000000..60648fd --- /dev/null +++ b/docs/archive/PYTHON_IMPROVEMENTS.md @@ -0,0 +1,230 @@ +# Python Pipeline Improvements Over R + +> **ARCHIVED — frozen, and known to be incomplete.** See +> [`docs/archive/README.md`](README.md). This was a hand-maintained list +> written early in the migration, and the completeness audit +> ([wayfinder ticket 7](../wayfinder/tickets/07-pipeline-completeness-audit.md)) +> found it **undercounting the known divergences**. It was superseded by two +> things that derive rather than list: the cause registry in +> `src/a4d/migration/compare.py`, where every divergence is tied to a mechanism +> traced in real source workbooks, and the wayfinder map's Decisions-so-far. +> Read those for the real inventory; read this for early context only. Frozen +> at tag `migration-archive-frozen`. + +This document tracks cases where the Python pipeline implementation is **more correct** than the R pipeline, resulting in intentional differences between R and Python outputs. + +## 1. insulin_type Derivation Bug Fix + +**Status**: ✅ Fixed in Python + +**Issue in R**: R's insulin_type derivation logic only checks the human insulin columns to decide between "human insulin" and "analog insulin". When all human insulin columns are None/NA, the condition evaluates to NA, and `ifelse()` returns NA - **even if the analog insulin columns have "Y" values**. + +**R Code (Buggy)**: +```r +insulin_type = ifelse( + human_insulin_pre_mixed == "Y" | + human_insulin_short_acting == "Y" | + human_insulin_intermediate_acting == "Y", + "human insulin", + "analog insulin" +) +``` + +**Problem**: For patients with ONLY analog insulin (human columns = None, analog columns = 'Y'): +- `None == "Y"` evaluates to NA in R +- `NA | NA | NA` → NA +- `ifelse(NA, "human insulin", "analog insulin")` → NA + +**Python Fix**: Check if ANY insulin column has data first, then derive the type: +```python +pl.when( + # Only derive if at least one insulin column is not null + pl.col("human_insulin_pre_mixed").is_not_null() + | pl.col("human_insulin_short_acting").is_not_null() + | pl.col("human_insulin_intermediate_acting").is_not_null() + | pl.col("analog_insulin_rapid_acting").is_not_null() + | pl.col("analog_insulin_long_acting").is_not_null() +) +.then( + pl.when( + (pl.col("human_insulin_pre_mixed") == "Y") + | (pl.col("human_insulin_short_acting") == "Y") + | (pl.col("human_insulin_intermediate_acting") == "Y") + ) + .then(pl.lit("human insulin")) + .otherwise(pl.lit("analog insulin")) +) +.otherwise(None) +``` + +**Impact**: For 2024 Sibu Hospital tracker, 5 patients correctly get `insulin_type = 'Analog Insulin'` in Python vs `None` in R. + +**File**: `src/a4d/clean/patient.py:_derive_insulin_fields()` + +## 2. insulin_subtype Typo Fix + +**Status**: ✅ Fixed in Python + +**Issue in R**: R has a typo - uses "rapic-acting" instead of "rapid-acting" when deriving insulin_subtype. + +**R Code (Typo)**: +```r +paste(ifelse(analog_insulin_rapid_acting == "Y", "rapic-acting", ""), sep = ",") +``` + +**Python Fix**: Uses correct spelling "rapid-acting" + +**Impact**: Derived insulin_subtype values use correct medical terminology. However, since comma-separated values get replaced with "Undefined" by validation, the final output for insulin_subtype is still "Undefined" in both R and Python. + +**File**: `src/a4d/clean/patient.py:_derive_insulin_fields()` + +## 3. insulin_total_units Extraction Bug Fix + +**Status**: ✅ Fixed in Python + +**Issue in R**: R's header merge logic has a condition that fails for 2024+ trackers, causing it to skip the two-row header merge and lose columns. + +**R Code (Buggy)** - `script1_helper_read_patient_data.R:92`: +```r +if (header_cols[2] == header_cols_2[2]) { + # Only merge if column 2 matches in both rows + diff_colnames <- which((header_cols != header_cols_2)) + header_cols[diff_colnames] <- paste(header_cols_2[diff_colnames], header_cols[diff_colnames]) +} +``` + +**Problem for 2024 Sibu Hospital tracker**: +- Row 75 (header_cols_2), Col 2: `"Patient \nID*"` +- Row 76 (header_cols), Col 2: `None` (part of merged cell above) +- Condition `header_cols[2] == header_cols_2[2]` evaluates to `FALSE` +- **Headers NOT merged**, only row 76 used + +**Result**: +- Col 27 in R: Only gets "per day" (row 76 alone) +- "per day" doesn't match synonym "TOTAL Insulin Units per day" +- **Column lost during synonym mapping** + +**Python Fix**: Python always merges both header rows without conditions: +```python +for h1, h2 in zip(header_1, header_2, strict=True): + if h1 and h2: + headers.append(f"{h2} {h1}".strip()) +``` + +**Result**: +- Col 27 in Python: "TOTAL Insulin Units per day" (row 75 + row 76) +- Matches synonym perfectly ✅ + +**Impact**: For 2024 Sibu Hospital tracker, Python correctly extracts insulin_total_units for 50/53 patients. R loses this column entirely due to header merge failure. + +**File**: `src/a4d/extract/patient.py:merge_headers()` + +## 4. BMI Float Precision + +**Status**: ℹ️ Negligible difference + +**Observation**: Minor floating point precision differences at the ~10^-15 level. + +**Example**: +- R: `19.735976492259113` +- Python: `19.73597649225911` + +**Cause**: Different floating point arithmetic between R and Python/Polars. + +**Impact**: Negligible - differences are below any meaningful precision threshold for BMI measurements. + +## 5. Product Pipeline: Date Parsing Robustness + +**Status**: ✅ Improved in Python + +Three distinct date-parsing patterns surfaced during product-pipeline diff investigation against R goldens. In each case Python yields a more correct result than R; no R-parity fix is warranted. Investigation: the underlying notebook (`Ali_internship/residual_dig.ipynb`) was never committed to the repo and no longer exists; the surviving record of this analysis is [`Product pipeline parity presentation.pdf`](Product%20pipeline%20parity%20presentation.pdf), kept until the automated comparison script (see [ticket 2](../wayfinder/tickets/02-documentation-strategy.md)) reproduces and supersedes its numbers. + +### 5.1 "Sept" → "Sep" month abbreviation + +**Issue in R**: `lubridate` does not recognize the 4-letter abbreviation "Sept" as September. Source strings like `"05-Sept-2025"` or `"25-Sept-2025"` are rejected → `null`. + +**Python Fix**: `parse_date_flexible` strips the trailing letter from any 4-letter month abbreviation before matching: + +```python +re.sub(r"([a-zA-Z]{3})[a-zA-Z]", r"\1", date_str) +``` + +This converts `"Sept"` → `"Sep"`, after which the standard `%d-%b-%Y` parse succeeds. + +**Impact**: 46 rows across 28 product `(file, sheet, product)` groups where R has `null` and Python has a valid date — the `py_set_r_null` class in the joined-view diff. Affected trackers include `2024_Putrajaya Hospital A4D Tracker` (Sep24) and `2025_NPH A4D Tracker` (Sep25). + +**File**: [src/a4d/clean/date_parser.py:67-69](../../src/a4d/clean/date_parser.py#L67-L69) + +### 5.2 D/M/YYYY single-digit-month dates + +**Issue in R**: `lubridate::dmy()` with default formats does not parse `"20/5/2025"` (single-digit month, slash separator) — rejects → `null`. + +**Python Fix**: `parse_date_flexible` includes a slash-format match path that accepts both `D/M/YYYY` and `DD/MM/YYYY`. + +**Impact**: 5 rows in `2025_Putrajaya Hospital A4D Tracker` / May25 / WIZ Test Strips with cell values like `"20/5/2025"`, `"22/5/2025"`, `"28/5/2025"`. R rejects all; Python parses them correctly. + +**File**: [src/a4d/clean/date_parser.py](../../src/a4d/clean/date_parser.py) + +### 5.3 Future-year sentinel guard for malformed date strings + +**Issue in R**: `lubridate` is permissive — strings with structural typos like `"10-Oct-2-24"` (extra `-2-` injected) get force-parsed into plausible-but-incorrect dates (e.g. `2024-02-10`). The wrong date sorts at a different position in the cumulative-balance sequence, so intermediate `product_balance` values diverge from what the source spreadsheet shows. R has no future-year guard. + +**Python Fix**: `parse_date_flexible` returns successfully or returns the sentinel `9999-09-09` for unparseable input. `_validate_entry_dates` then re-sentinels any successfully-parsed date whose year is between the tracker year and the Buddhist-era threshold (2400) — this catches fat-fingered Gregorian years (e.g. `"2099-..."` typed in a 2024 tracker) while exempting genuine Buddhist-era dates (BE 25xx → CE 20xx). Sentinelled rows sort to the end of the cumulative-balance sequence so they don't corrupt intermediate values. + +```python +BUDDHIST_ERA_THRESHOLD = 2400 +invalid_mask = ( + pl.col("product_entry_date").is_not_null() + & (pl.col("product_entry_date") > max_valid) + & (pl.col("product_entry_date").dt.year() < BUDDHIST_ERA_THRESHOLD) +) +``` + +**Impact**: 31 rows across 6 product groups (the post-Buddhist-fix `real_divergence` class in `product_balance`) — R force-parses or silently rejects malformed strings while Python correctly sentinels them. Confirmed by raw-Excel inspection of: + +- `"10-Oct-2-24"` (×2) — Putrajaya 2024 / Oct24 / WIZ Alcohol Swabs. R parses as `2024-02-10`; Python sentinels. +- `"15-Seep-225"`, `"08-Sep02025"` — NPH 2025 / Sep25. Both R and Python sentinel/null these; balance still diverges via the date-sort interaction with the legitimate Sept rows from §5.1. +- `"01-Ju-2025"` — Sarawak 2025 / Jul25. Both pipelines reject ("Ju" is too short to disambiguate Jun/Jul); balance diverges because R returns `null` (sorts in nulls-first/last position differing from sentinel) while Python returns `9999-09-09` (sorts to end deterministically). + +**File**: [src/a4d/clean/product.py:_validate_entry_dates](../../src/a4d/clean/product.py) + +## 6. Product Pipeline: Running Balance FP Precision + +**Status**: ℹ️ Negligible difference + +**Observation**: Python's vectorized `cum_sum().over([sheet, product])` and R's iterative `for (i in 1:nrow)` loop produce running balances that drift by ~5.7 × 10⁻¹⁴ at the deepest accumulation step. + +**Cause**: IEEE-754 floating-point accumulation order differs between Polars' vectorized cumsum and R's row-by-row addition. + +**Impact**: 411 rows across 80 product groups in the joined-view diff are flagged as different but classify as `fp_precision_only`. Both pipelines produce identical final-row balances per group; only sub-display-precision intermediates differ. + +**File**: [src/a4d/clean/product.py:_compute_running_balance](../../src/a4d/clean/product.py) + +## Summary + +| Issue | R Behavior | Python Behavior | Classification | +|-------|-----------|-----------------|----------------| +| insulin_type derivation | Bug - returns None for analog-only patients (doesn't check analog columns) | Correct derivation (checks all insulin columns) | **Python Fix** | +| insulin_subtype typo | "rapic-acting" (typo) | "rapid-acting" (correct spelling) | **Python Fix** | +| insulin_total_units extraction | Not extracted (header merge fails for 2024+ trackers) | Correctly extracted (unconditional header merge) | **Python Fix** | +| BMI precision | 16 decimal places | 14-15 decimal places | **Negligible** | +| product entry date "Sept" | Rejects 4-letter month abbreviation → null | Truncates to "Sep", parses correctly | **Python Fix** | +| product entry date `D/M/YYYY` | Rejects single-digit-month slash format → null | Parses correctly | **Python Fix** | +| product entry date — malformed strings | Force-parses to plausible-but-incorrect dates (e.g. `"10-Oct-2-24"` → 2024-02-10), distorting cumulative balance | Sentinels to `9999-09-09`, sorts to end, preserves correct intermediate balances | **Python Fix** | +| product running balance | Iterative cumsum, slightly different IEEE-754 accumulation order | Vectorized cumsum, identical final-row balances | **Negligible** | + +## Migration Validation Status + +✅ **Schema**: 100% match (83 patient columns + 20 product columns) +✅ **Extraction**: Improved (unconditional header merge fixes insulin_total_units) +✅ **Cleaning**: Improved (insulin_type, insulin_subtype, product date parsing) +ℹ️ **Precision**: Acceptable float differences (~10⁻¹⁵ BMI, ~10⁻¹⁴ product running balance) + +**All value differences are Python improvements over R bugs or negligible precision drift.** + +The Python pipeline is production-ready with significant improvements over the R pipeline: + +1. **More robust header parsing** - No conditional merge that fails on 2024+ trackers +2. **Better null handling** - Correctly checks all insulin columns before derivation +3. **Correct terminology** - Uses proper medical terms ("rapid-acting" not "rapic-acting") +4. **More robust date parsing** - Accepts "Sept" and `D/M/YYYY`; sentinels malformed strings instead of force-parsing them into plausible-but-wrong dates that distort cumulative balances diff --git a/docs/archive/Product pipeline parity presentation.pdf b/docs/archive/Product pipeline parity presentation.pdf new file mode 100644 index 0000000..c4c52d5 Binary files /dev/null and b/docs/archive/Product pipeline parity presentation.pdf differ diff --git a/docs/archive/README.md b/docs/archive/README.md new file mode 100644 index 0000000..b94a8bd --- /dev/null +++ b/docs/archive/README.md @@ -0,0 +1,51 @@ +# Archive — the R-to-Python migration + +**These documents are frozen. Nothing here is current guidance.** + +They are the record of a finished piece of work: the migration of the A4D +medical tracker pipeline from R to Python. Every one of them is written in +terms of the R pipeline, because comparing against R is what the work +consisted of. The R pipeline itself was deleted from this repository on +2026-08-24. + +Frozen at the tag **`migration-archive-frozen`**, commit **`0dcc02d`** — the +last state in which these documents were checked against the code. The tag is +the durable reference; the SHA is recorded so a reader who has only this file +can still find it. These documents are not maintained past that commit, and a +fact stated here should be assumed stale unless the code agrees. + +## Where to look instead + +| For | Read | +|---|---| +| What the pipeline does and how to run it | [`README.md`](../../README.md), [`SETUP.md`](../../SETUP.md) | +| Module-by-module map, CLI, output layout | [`docs/CLAUDE.md`](../CLAUDE.md) | +| Why a particular cleaning rule exists | the docstring on the rule itself | +| The migration's decision history | [`docs/wayfinder/`](../wayfinder/map.md) | +| The old R source | `git show r-archive-removed^:r-archive/R/` | + +## What is here + +- **`MR_DESCRIPTION.md`** — the description of PR #2 (`migration` -> `dev`). + The fullest single account of what changed and what was verified. Live until + that PR merges; frozen thereafter. +- **`MIGRATION_GUIDE.md`** — the phase-by-phase plan the migration followed, + plus the R-to-Python step mapping. +- **`PYTHON_IMPROVEMENTS.md`** — cases where Python was judged more correct + than R. Superseded as a *catalogue* by the cause registry in + `src/a4d/migration/compare.py` and by the wayfinder map's decisions, both of + which are evidence-backed where this document is not. +- **`PRODUCT_DATA_PIPELINE_FEATURE.md`** — the original proposal for the + product arm, written before it was built. Already marked obsolete in its own + header. +- **`Product pipeline parity presentation.pdf`** — the parity presentation the + comparison script was built to supersede. + +## Why they were kept rather than deleted + +The migration's conclusions rest on cell-by-cell evidence gathered over 56 +wayfinder tickets. Those conclusions are now expressed where they belong — in +the code's own docstrings, in the wayfinder map, and in the comparison tool's +cause registry. These documents are the working papers behind them. They cost +nothing to keep and they are the only place some of the intermediate reasoning +survives. diff --git a/docs/wayfinder/assets/fbg_header_inventory.md b/docs/wayfinder/assets/fbg_header_inventory.md new file mode 100644 index 0000000..1b75981 --- /dev/null +++ b/docs/wayfinder/assets/fbg_header_inventory.md @@ -0,0 +1,34 @@ +# FBG header spellings across the 254-tracker set + +Generated while closing [ticket 30](../tickets/30-triage-patient-raw-column-divergence.md); +the evidence base for [ticket 42](../tickets/42-fbg-unit-headers-and-implausible-values.md). + +`scale_suggests` is inferred from the median (>35 reads as mg/dL); `values_over_35` counts +readings that are implausible as mmol/L. No patient-level data here -- header text and +aggregates only. + +| header_in_tracker | unit_claimed_by_header | resolves_to | files | values | median | min | max | scale_suggests | values_over_35 | pct_over_35 | disagrees | +|---|---|---|---|---|---|---|---|---|---|---|---| +| Baseline FBG (mg%) | mg | fbg_baseline_mg | 4 | 1314 | 291 | 91 | 500 | mg/dL | 1314 | 100 | | +| Baseline FBG (mg/dL) | mg | fbg_baseline_mg | 44 | 9517 | 190.8 | 0 | 760 | mg/dL | 8103 | 85.1 | | +| Baseline FBG or CBG (mg/dL) | mg | fbg_baseline_mg | 9 | 2679 | 151 | 3.1 | 565.2 | mg/dL | 1573 | 58.7 | | +| Baseline FBG* mg/dL | mg | fbg_baseline_mg | 23 | 8132 | 174.6 | 0 | 1287 | mg/dL | 6324 | 77.8 | | +| Updated FBG (mg/dL) | mg | fbg_updated_mg | 6 | 40 | 135 | 94 | 324 | mg/dL | 40 | 100 | | +| Updated FBG FBG (mg/dL) | mg | fbg_updated_mg | 2 | 572 | 160 | 0 | 384 | mg/dL | 518 | 90.6 | | +| Updated FBG mg/dL | mg | fbg_updated_mg | 199 | 46814 | 122.4 | 0 | 2013 | mg/dL | 42331 | 90.4 | | +| Updated FBG or CBG mg/dL | mg | fbg_updated_mg | 9 | 2594 | 106.2 | 0 | 1800 | mg/dL | 1469 | 56.6 | | +| Updated Mean FBG (mg/%) | mg | fbg_updated_mg | 3 | 14 | 133.2 | 67 | 217.8 | mg/dL | 14 | 100 | | +| Updated Mean FBG (mg/dL) | mg | fbg_updated_mg | 6 | 1 | 231 | 231 | 231 | mg/dL | 1 | 100 | | +| Baseline FBG (mmol/L or mg/dL) | mmol | (UNMAPPED - column dropped) | 1 | 155 | 7.7 | 3.6 | 29.7 | mmol/L | 0 | 0 | | +| Baseline FBG (mmol/dL) | mmol | (UNMAPPED - column dropped) | 1 | 45 | 233 | 67 | 500 | mg/dL | 45 | 100 | HEADER SAYS MMOL, VALUES LOOK MG | +| Updated FBG (mmol/L or mg/dL) | mmol | (UNMAPPED - column dropped) | 1 | 0 | | | | | 0 | | | +| Baseline FBG (mmol/L) | mmol | fbg_baseline_mmol | 17 | 2787 | 21.8 | 3.1 | 71.5 | mmol/L | 386 | 13.9 | | +| Baseline FBG mmol/L | mmol | fbg_baseline_mmol | 3 | 449 | 11.1 | 3.1 | 53.8 | mmol/L | 40 | 8.9 | | +| Baseline FBG* mmol/L | mmol | fbg_baseline_mmol | 4 | 1293 | 16.6 | 0 | 81.9 | mmol/L | 129 | 10 | | +| Updated FBG (mmol/L) | mmol | fbg_updated_mmol | 1 | 0 | | | | | 0 | | | +| Updated FBG FBG mmol/L | mmol | fbg_updated_mmol | 1 | 0 | | | | | 0 | | | +| Updated FBG mmol/L | mmol | fbg_updated_mmol | 48 | 11108 | 8 | 1.4 | 516.8 | mmol/L | 66 | 0.6 | | +| Updated FBG mmol/dL | mmol | fbg_updated_mmol | 2 | 178 | 9.4 | 3 | 600 | mmol/L | 49 | 27.5 | | +| Updated FBG or CBG mmol/dL | mmol | fbg_updated_mmol | 1 | 52 | 8.4 | 3.5 | 23.6 | mmol/L | 0 | 0 | | +| Updated Mean FBG (mmol/L) | mmol | fbg_updated_mmol | 2 | 119 | 8.5 | 2.2 | 30 | mmol/L | 0 | 0 | | +| Baseline FBG (m/dL) | none | fbg_baseline_mg | 1 | 20 | 122.5 | 5 | 306 | mg/dL | 18 | 90 | | diff --git a/docs/wayfinder/assets/glucose_readings_by_unit.md b/docs/wayfinder/assets/glucose_readings_by_unit.md new file mode 100644 index 0000000..8d01999 --- /dev/null +++ b/docs/wayfinder/assets/glucose_readings_by_unit.md @@ -0,0 +1,42 @@ +# Glucose readings grouped by when measured and stated unit + +Measured across all 254 trackers while closing [ticket +30](../tickets/30-triage-patient-raw-column-divergence.md); the evidence sent to +A4D's medical advisor for [ticket +42](../tickets/42-fbg-unit-headers-and-implausible-values.md). + +Every glucose column in every month sheet, grouped by whether it records the +reading at diagnosis (baseline) or during treatment (updated), and by the unit +its header states. Per-spelling detail is in +[fbg_header_inventory.md](fbg_header_inventory.md). + +| When measured | Unit stated | Files | Readings | Median | p90 | p99 | Lowest | Highest | Median as mmol/L | Exactly 0 | mg/dL under 30 | mg/dL 30-54 | Over 100 mmol/L | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| At diagnosis | Ambiguous (header offers both) | 1 | 155 | 7.7 | 17.5 | 29.7 | 3.6 | 29.7 | 7.7 | 0 | | | 0 | +| At diagnosis | Not stated | 1 | 20 | 122.5 | 306.0 | 306.0 | 5.0 | 306.0 | 122.5 | 0 | | | 12 | +| At diagnosis | mg/dL | 71 | 21,642 | 190.0 | 393.0 | 581.9 | 0.0 | 1287.0 | 10.5 | 134 | 3,714 | 1,040 | 0 | +| At diagnosis | mmol/L | 25 | 4,574 | 19.3 | 38.0 | 80.0 | 0.0 | 500.0 | 19.3 | 244 | | | 30 | +| During treatment | mg/dL | 214 | 50,035 | 122.0 | 265.0 | 445.0 | 0.0 | 2013.0 | 6.8 | 60 | 5,550 | 647 | 3 | +| During treatment | mmol/L | 52 | 11,457 | 8.0 | 15.5 | 36.9 | 1.4 | 600.0 | 8.0 | 0 | | | 92 | + +## What this shows + +**Unit confusion runs in both directions, and is far larger in one of them.** +9,264 readings sit in columns labelled mg/dL but below 30 mg/dL -- a value that +cannot be an ambulatory reading and lands exactly where a mmol/L number would. +122 readings sit in columns labelled mmol/L but above 100, where a mg/dL number +would. Whether these really are mixed units is the advisor's call, not the +pipeline's. + +**Zeros are placeholders, not hypoglycaemia.** All 244 baseline mmol/L readings +under 3 mmol/L are exactly 0. + +**The baseline distribution is broad but continuous**, with no values above 100 +at all -- so it does not have the bimodal shape a bulk mg/dL contamination +would produce. A median of 19.3 mmol/L may simply be presentation +hyperglycaemia; that is question 3 to the advisor. + +**The two cut-offs used here are physiological, not chosen for convenience**: +above 100 mmol/L and below 30 mg/dL are both impossible readings. An earlier +version of this analysis bucketed on 35 mmol/L, which was an invented threshold +and was dropped. diff --git a/docs/wayfinder/graph.mmd b/docs/wayfinder/graph.mmd new file mode 100644 index 0000000..95cf24d --- /dev/null +++ b/docs/wayfinder/graph.mmd @@ -0,0 +1,115 @@ +flowchart TD + subgraph FRONTIER["Frontier · 7"] + direction TB + T6["6 · task
Promote migration into dev
via PR #2"] + T16["16 · grilling
Build a drill-down log
analyzer for admins to
inspect a specific tracker
file's errors/logs"] + T34["34 · grilling
Make the local pre-push
check set actually match
CI, and make running it
automatic"] + T35["35 · task
Resolve the Polars 2.0
deprecation warnings —
decide the behaviour each
one is asking about"] + T40["40 · task
Produce one Excel of every
source-tracker defect, so
the trackers themselves
can be corrected"] + T41["41 · grilling
Decide whether the 2026
template's five new
Patient List fields enter
the pipeline"] + T65["65 · grilling
Two values published into
the logs table still name
R scripts"] + end + subgraph BLOCKED["Blocked · 1"] + direction TB + T9["9 · task
Add golden-master/snapshot
regression tests for
patient and product"] + end + subgraph DECIDED["Decided · 56"] + direction TB + T2["2 · grilling
Retire the PDF/notebook
analysis docs for an
automated, script-based
report"] + T3["3 · task
Merge product-pipeline (PR
#6) into migration"] + T4["4 · task
Diagnose and fix why CI is
red at migration HEAD"] + T5["5 · task
Define and execute the
real GCP production
verification run"] + T7["7 · research
Is the product pipeline
(and patient's own claimed
completeness) actually
complete and sound,
audited against R's
product logic and
patient's structure?"] + T8["8 · grilling
Does the pytest suite
reach unit/integration/e2e
/regression parity between
patient and product,
excluding any
R-comparison/USB-drive-
dependent tests?"] + T10["10 · task
Profile the combined
pipeline's performance
against the R baseline
before promoting to dev"] + T11["11 · grilling
Decide what CLI/TUI UX and
error-log observability
improvements
admins/developers need
before rollout"] + T12["12 · task
Retire R from the
workspace once the
pipeline is fully verified
Python-only"] + T13["13 · task
Audit and update all
dependencies and library
versions before rollout"] + T14["14 · task
Fix product pipeline's
unable to find column
product failures on 4 real
trackers"] + T15["15 · task
Build and run the R/Python
output comparison script,
then triage every flagged
difference"] + T17["17 · task
Fix the product
comparison's row-alignment
key, then triage every
flagged R/Python
difference"] + T18["18 · task
Triage every flagged
R/Python difference for
both arms, and resolve the
189-vs-155-tracker
discrepancy"] + T19["19 · task
Persist comparison run
history and show run-over-
run deltas"] + T20["20 · task
Normalize the raw-stage
product_entry_date
comparison so it stops
flagging near-universal
false mismatches"] + T21["21 · task
Triage the remaining
product cleaned-stage
column mismatches
(balance, received_from,
released_to, remarks,
units_received, product)"] + T22["22 · task
Triage the remaining
product raw-stage column
mismatches"] + T23["23 · task
Triage every flagged
R/Python difference for
the patient arm (raw and
cleaned)"] + T24["24 · task
Triage the residual produc
t_units_received/product_u
nits_released/product_rece
ived_from raw-stage
mismatches"] + T25["25 · task
Triage the
product_units_released
cleaned-stage column
mismatches"] + T26["26 · task
Triage the product
pipeline's column-
existence and dtype
divergence (Column
divergence)"] + T27["27 · task
Triage the residual
patient raw-stage column
mismatches after date
normalization"] + T28["28 · task
Triage the patient
cleaned-stage column
mismatches"] + T29["29 · task
Triage the residual
patient cleaned-stage
column mismatches"] + T30["30 · task
Triage the patient
pipeline's raw-stage
column-existence
divergence"] + T31["31 · task
Triage the residual
patient raw-stage column
mismatches (round 2)"] + T32["32 · task
Re-audit every existing
cause classifier — is
Python actually right, or
was the diff merely
labelled?"] + T33["33 · task
Fix red CI — ruff format
--check fails on Python
snippets inside markdown
docs"] + T36["36 · task
Triage the product
cleaned-stage mismatches
no ticket owns
(product_balance,
sheet_name, entry_date,
units_received, file_name)"] + T37["37 · task
Triage the residual
patient cleaned-stage
mismatches (round 2)"] + T38["38 · task
Triage the patient
cleaned-stage date-column
family (round 3)"] + T39["39 · grilling
Decide whether a date
buried inside a clinical
note should be recovered
or discarded"] + T42["42 · grilling
Decide how FBG unit
headers are resolved, and
what to do about
physiologically
implausible mmol values"] + T43["43 · task
Triage the residual
patient raw-stage column
mismatches (round 3)"] + T44["44 · task
Classify the cleaned-stage
FBG cells where R has
nothing and Python has a
corrected reading"] + T45["45 · task
Give the patient
comparison an ordinal row
key, so duplicated patient
IDs stop faking mismatches"] + T46["46 · task
Triage the residual
patient raw-stage column
mismatches (round 4)"] + T47["47 · task
Four trackers where
cleaning merges several
patients into one patient
ID"] + T48["48 · task
Python drops complication-
screening results and
dates where a merged
header spans the block"] + T49["49 · task
Triage the residual
patient raw-stage column
mismatches (round 5)"] + T50["50 · task
Triage the residual
patient raw-stage column
mismatches (round 6)"] + T51["51 · task
Triage the residual
patient cleaned-stage
mismatches (round 4)"] + T52["52 · task
Triage the residual
patient cleaned-stage
mismatches (round 5)"] + T53["53 · task
Triage the residual
patient cleaned-stage
mismatches (round 6)"] + T54["54 · task
Triage the residual
patient cleaned-stage
mismatches (round 7)"] + T55["55 · task
Triage the residual
patient cleaned-stage
mismatches (round 8)"] + T56["56 · task
Triage the residual
patient cleaned-stage
mismatches (round 9)"] + T57["57 · task
Triage the residual
patient cleaned-stage
mismatches (round 10)"] + T58["58 · task
Monthly rows with a
misspelled ID silently
lose their Patient List
demographics"] + T59["59 · task
Rows that pair with
nothing on the other side,
which no ticket has ever
triaged"] + T60["60 · task
Audit the eight pre-bar
causes the first
classifier pass did not
reach"] + T61["61 · grilling
Decide whether a Thai
clinic's Buddhist-era
entry date is published as
2567 or converted to 2024"] + T62["62 · task
Finish the pre-bar
classifier audit — the two
causes and the one bulk
population it did not
reach"] + T63["63 · task
The cleaned stage has
4,949 cells with no cause,
because the ID spelling
that explains them is gone
by then"] + T64["64 · task
Rewrite every docstring
and doc that explains the
code by what R did"] + end + subgraph DROPPED["Out of scope · 1"] + direction TB + T1["1 · grilling
Does product-pipeline's
test suite meet the same
cell-by-cell rigor as
patient's?"] + end + + T2 --> T15 + T3 --> T2 + T3 --> T4 + T3 --> T5 + T3 --> T6 + T3 --> T10 + T3 --> T11 + T3 --> T13 + T4 --> T5 + T4 --> T6 + T5 --> T6 + T6 --> T9 + T7 --> T8 + T8 --> T3 + T8 --> T6 + T10 --> T6 + T11 --> T6 + T12 --> T6 + T13 --> T6 + T13 --> T10 + T14 --> T6 + T20 --> T6 + T21 --> T6 + T22 --> T6 + T23 --> T6 + T45 --> T46 + T64 --> T6 + + classDef frontier fill:#1f6feb,stroke:#0b3d91,stroke-width:3px,color:#ffffff + class T6,T16,T34,T35,T40,T41,T65 frontier + classDef blocked fill:#6e7781,stroke:#424a53,stroke-width:1px,color:#ffffff + class T9 blocked + classDef decided fill:#1a7f37,stroke:#116329,stroke-width:1px,color:#ffffff + class T2,T3,T4,T5,T7,T8,T10,T11,T12,T13,T14,T15,T17,T18,T19,T20,T21,T22,T23,T24,T25,T26,T27,T28,T29,T30,T31,T32,T33,T36,T37,T38,T39,T42,T43,T44,T45,T46,T47,T48,T49,T50,T51,T52,T53,T54,T55,T56,T57,T58,T59,T60,T61,T62,T63,T64 decided + classDef dropped fill:#eaeef2,stroke:#afb8c1,stroke-width:1px,color:#57606a + class T1 dropped diff --git a/docs/wayfinder/graph.svg b/docs/wayfinder/graph.svg new file mode 100644 index 0000000..d9c2741 --- /dev/null +++ b/docs/wayfinder/graph.svg @@ -0,0 +1 @@ +

Out of scope · 1

1 · grilling
Does product-pipeline's
test suite meet the same
cell-by-cell rigor as
patient's?

Decided · 56

2 · grilling
Retire the PDF/notebook
analysis docs for an
automated, script-based
report

15 · task
Build and run the R/Python
output comparison script,
then triage every flagged
difference

3 · task
Merge product-pipeline (PR
#6) into migration

4 · task
Diagnose and fix why CI is
red at migration HEAD

5 · task
Define and execute the
real GCP production
verification run

10 · task
Profile the combined
pipeline's performance
against the R baseline
before promoting to dev

11 · grilling
Decide what CLI/TUI UX and
error-log observability
improvements
admins/developers need
before rollout

13 · task
Audit and update all
dependencies and library
versions before rollout

7 · research
Is the product pipeline
(and patient's own claimed
completeness) actually
complete and sound,
audited against R's
product logic and
patient's structure?

8 · grilling
Does the pytest suite
reach unit/integration/e2e
/regression parity between
patient and product,
excluding any
R-comparison/USB-drive-
dependent tests?

45 · task
Give the patient
comparison an ordinal row
key, so duplicated patient
IDs stop faking mismatches

46 · task
Triage the residual
patient raw-stage column
mismatches (round 4)

12 · task
Retire R from the
workspace once the
pipeline is fully verified
Python-only

14 · task
Fix product pipeline's
unable to find column
product failures on 4 real
trackers

17 · task
Fix the product
comparison's row-alignment
key, then triage every
flagged R/Python
difference

18 · task
Triage every flagged
R/Python difference for
both arms, and resolve the
189-vs-155-tracker
discrepancy

19 · task
Persist comparison run
history and show run-over-
run deltas

20 · task
Normalize the raw-stage
product_entry_date
comparison so it stops
flagging near-universal
false mismatches

21 · task
Triage the remaining
product cleaned-stage
column mismatches
(balance, received_from,
released_to, remarks,
units_received, product)

22 · task
Triage the remaining
product raw-stage column
mismatches

23 · task
Triage every flagged
R/Python difference for
the patient arm (raw and
cleaned)

24 · task
Triage the residual produc
t_units_received/product_u
nits_released/product_rece
ived_from raw-stage
mismatches

25 · task
Triage the
product_units_released
cleaned-stage column
mismatches

26 · task
Triage the product
pipeline's column-
existence and dtype
divergence (Column
divergence)

27 · task
Triage the residual
patient raw-stage column
mismatches after date
normalization

28 · task
Triage the patient
cleaned-stage column
mismatches

29 · task
Triage the residual
patient cleaned-stage
column mismatches

30 · task
Triage the patient
pipeline's raw-stage
column-existence
divergence

31 · task
Triage the residual
patient raw-stage column
mismatches (round 2)

32 · task
Re-audit every existing
cause classifier — is
Python actually right, or
was the diff merely
labelled?

33 · task
Fix red CI — ruff format
--check fails on Python
snippets inside markdown
docs

36 · task
Triage the product
cleaned-stage mismatches
no ticket owns
(product_balance,
sheet_name, entry_date,
units_received, file_name)

37 · task
Triage the residual
patient cleaned-stage
mismatches (round 2)

38 · task
Triage the patient
cleaned-stage date-column
family (round 3)

39 · grilling
Decide whether a date
buried inside a clinical
note should be recovered
or discarded

42 · grilling
Decide how FBG unit
headers are resolved, and
what to do about
physiologically
implausible mmol values

43 · task
Triage the residual
patient raw-stage column
mismatches (round 3)

44 · task
Classify the cleaned-stage
FBG cells where R has
nothing and Python has a
corrected reading

47 · task
Four trackers where
cleaning merges several
patients into one patient
ID

48 · task
Python drops complication-
screening results and
dates where a merged
header spans the block

49 · task
Triage the residual
patient raw-stage column
mismatches (round 5)

50 · task
Triage the residual
patient raw-stage column
mismatches (round 6)

51 · task
Triage the residual
patient cleaned-stage
mismatches (round 4)

52 · task
Triage the residual
patient cleaned-stage
mismatches (round 5)

53 · task
Triage the residual
patient cleaned-stage
mismatches (round 6)

54 · task
Triage the residual
patient cleaned-stage
mismatches (round 7)

55 · task
Triage the residual
patient cleaned-stage
mismatches (round 8)

56 · task
Triage the residual
patient cleaned-stage
mismatches (round 9)

57 · task
Triage the residual
patient cleaned-stage
mismatches (round 10)

58 · task
Monthly rows with a
misspelled ID silently
lose their Patient List
demographics

59 · task
Rows that pair with
nothing on the other side,
which no ticket has ever
triaged

60 · task
Audit the eight pre-bar
causes the first
classifier pass did not
reach

61 · grilling
Decide whether a Thai
clinic's Buddhist-era
entry date is published as
2567 or converted to 2024

62 · task
Finish the pre-bar
classifier audit — the two
causes and the one bulk
population it did not
reach

63 · task
The cleaned stage has
4,949 cells with no cause,
because the ID spelling
that explains them is gone
by then

64 · task
Rewrite every docstring
and doc that explains the
code by what R did

Blocked · 1

9 · task
Add golden-master/snapshot
regression tests for
patient and product

Frontier · 7

6 · task
Promote migration into dev
via PR #2

16 · grilling
Build a drill-down log
analyzer for admins to
inspect a specific tracker
file's errors/logs

34 · grilling
Make the local pre-push
check set actually match
CI, and make running it
automatic

35 · task
Resolve the Polars 2.0
deprecation warnings —
decide the behaviour each
one is asking about

40 · task
Produce one Excel of every
source-tracker defect, so
the trackers themselves
can be corrected

41 · grilling
Decide whether the 2026
template's five new
Patient List fields enter
the pipeline

65 · grilling
Two values published into
the logs table still name
R scripts

 \ No newline at end of file diff --git a/docs/wayfinder/map.md b/docs/wayfinder/map.md new file mode 100644 index 0000000..5e97130 --- /dev/null +++ b/docs/wayfinder/map.md @@ -0,0 +1,4337 @@ +--- +id: 0 +title: Close out the R-to-Python migration +labels: [wayfinder:map] +status: open +--- + +## Destination + +`product-pipeline` (PR #6) merged into `migration` with conflicts resolved; the +combined pipeline (patient + product + state management) run for real against +the GCP production bucket, with output landed in BigQuery; every Python/R +difference documented and explicitly decided; CI green; performance +re-profiled against the R baseline now that the merge has landed; the +CLI/TUI's admin/developer UX and error-log observability judged good enough to +operate the pipeline day to day; R retired from the workspace (`r-archive/`, +`tools/LogViewerA4D`, stray R scripts) now that the pipeline is fully verified +Python-only; all dependencies and library versions audited and updated; and +`migration` merged into `dev` (PR #2). +This prevents promoting a merge that looks clean but was never exercised as a +whole, prevents documenting or promoting based on claims ("the intern says it +works") rather than verified fact, and prevents rolling out something that +runs correctly but nobody can operate or debug confidently — the migration is +large enough, and detail-sensitive enough, that things get missed unless +checked cell-by-cell. + +**Destination redrawn 2026-08-09** (mid-[ticket 5](tickets/05-production-verification-run.md) +session): performance re-profiling, CLI/UX + observability, retiring R from +the workspace, and a dependency/library audit were added after the user +confirmed each belongs to "are we really ready to roll out", not separate +follow-on efforts. Originally the destination stopped at CI green + a +validated production run + promotion to `dev`. + +## The tickets + + +```mermaid +flowchart TD + subgraph FRONTIER["Frontier · 7"] + direction TB + T6["6 · task
Promote migration into dev
via PR #2"] + T16["16 · grilling
Build a drill-down log
analyzer for admins to
inspect a specific tracker
file's errors/logs"] + T34["34 · grilling
Make the local pre-push
check set actually match
CI, and make running it
automatic"] + T35["35 · task
Resolve the Polars 2.0
deprecation warnings —
decide the behaviour each
one is asking about"] + T40["40 · task
Produce one Excel of every
source-tracker defect, so
the trackers themselves
can be corrected"] + T41["41 · grilling
Decide whether the 2026
template's five new
Patient List fields enter
the pipeline"] + T65["65 · grilling
Two values published into
the logs table still name
R scripts"] + end + subgraph BLOCKED["Blocked · 1"] + direction TB + T9["9 · task
Add golden-master/snapshot
regression tests for
patient and product"] + end + subgraph DECIDED["Decided · 56"] + direction TB + T2["2 · grilling
Retire the PDF/notebook
analysis docs for an
automated, script-based
report"] + T3["3 · task
Merge product-pipeline (PR
#6) into migration"] + T4["4 · task
Diagnose and fix why CI is
red at migration HEAD"] + T5["5 · task
Define and execute the
real GCP production
verification run"] + T7["7 · research
Is the product pipeline
(and patient's own claimed
completeness) actually
complete and sound,
audited against R's
product logic and
patient's structure?"] + T8["8 · grilling
Does the pytest suite
reach unit/integration/e2e
/regression parity between
patient and product,
excluding any
R-comparison/USB-drive-
dependent tests?"] + T10["10 · task
Profile the combined
pipeline's performance
against the R baseline
before promoting to dev"] + T11["11 · grilling
Decide what CLI/TUI UX and
error-log observability
improvements
admins/developers need
before rollout"] + T12["12 · task
Retire R from the
workspace once the
pipeline is fully verified
Python-only"] + T13["13 · task
Audit and update all
dependencies and library
versions before rollout"] + T14["14 · task
Fix product pipeline's
unable to find column
product failures on 4 real
trackers"] + T15["15 · task
Build and run the R/Python
output comparison script,
then triage every flagged
difference"] + T17["17 · task
Fix the product
comparison's row-alignment
key, then triage every
flagged R/Python
difference"] + T18["18 · task
Triage every flagged
R/Python difference for
both arms, and resolve the
189-vs-155-tracker
discrepancy"] + T19["19 · task
Persist comparison run
history and show run-over-
run deltas"] + T20["20 · task
Normalize the raw-stage
product_entry_date
comparison so it stops
flagging near-universal
false mismatches"] + T21["21 · task
Triage the remaining
product cleaned-stage
column mismatches
(balance, received_from,
released_to, remarks,
units_received, product)"] + T22["22 · task
Triage the remaining
product raw-stage column
mismatches"] + T23["23 · task
Triage every flagged
R/Python difference for
the patient arm (raw and
cleaned)"] + T24["24 · task
Triage the residual produc
t_units_received/product_u
nits_released/product_rece
ived_from raw-stage
mismatches"] + T25["25 · task
Triage the
product_units_released
cleaned-stage column
mismatches"] + T26["26 · task
Triage the product
pipeline's column-
existence and dtype
divergence (Column
divergence)"] + T27["27 · task
Triage the residual
patient raw-stage column
mismatches after date
normalization"] + T28["28 · task
Triage the patient
cleaned-stage column
mismatches"] + T29["29 · task
Triage the residual
patient cleaned-stage
column mismatches"] + T30["30 · task
Triage the patient
pipeline's raw-stage
column-existence
divergence"] + T31["31 · task
Triage the residual
patient raw-stage column
mismatches (round 2)"] + T32["32 · task
Re-audit every existing
cause classifier — is
Python actually right, or
was the diff merely
labelled?"] + T33["33 · task
Fix red CI — ruff format
--check fails on Python
snippets inside markdown
docs"] + T36["36 · task
Triage the product
cleaned-stage mismatches
no ticket owns
(product_balance,
sheet_name, entry_date,
units_received, file_name)"] + T37["37 · task
Triage the residual
patient cleaned-stage
mismatches (round 2)"] + T38["38 · task
Triage the patient
cleaned-stage date-column
family (round 3)"] + T39["39 · grilling
Decide whether a date
buried inside a clinical
note should be recovered
or discarded"] + T42["42 · grilling
Decide how FBG unit
headers are resolved, and
what to do about
physiologically
implausible mmol values"] + T43["43 · task
Triage the residual
patient raw-stage column
mismatches (round 3)"] + T44["44 · task
Classify the cleaned-stage
FBG cells where R has
nothing and Python has a
corrected reading"] + T45["45 · task
Give the patient
comparison an ordinal row
key, so duplicated patient
IDs stop faking mismatches"] + T46["46 · task
Triage the residual
patient raw-stage column
mismatches (round 4)"] + T47["47 · task
Four trackers where
cleaning merges several
patients into one patient
ID"] + T48["48 · task
Python drops complication-
screening results and
dates where a merged
header spans the block"] + T49["49 · task
Triage the residual
patient raw-stage column
mismatches (round 5)"] + T50["50 · task
Triage the residual
patient raw-stage column
mismatches (round 6)"] + T51["51 · task
Triage the residual
patient cleaned-stage
mismatches (round 4)"] + T52["52 · task
Triage the residual
patient cleaned-stage
mismatches (round 5)"] + T53["53 · task
Triage the residual
patient cleaned-stage
mismatches (round 6)"] + T54["54 · task
Triage the residual
patient cleaned-stage
mismatches (round 7)"] + T55["55 · task
Triage the residual
patient cleaned-stage
mismatches (round 8)"] + T56["56 · task
Triage the residual
patient cleaned-stage
mismatches (round 9)"] + T57["57 · task
Triage the residual
patient cleaned-stage
mismatches (round 10)"] + T58["58 · task
Monthly rows with a
misspelled ID silently
lose their Patient List
demographics"] + T59["59 · task
Rows that pair with
nothing on the other side,
which no ticket has ever
triaged"] + T60["60 · task
Audit the eight pre-bar
causes the first
classifier pass did not
reach"] + T61["61 · grilling
Decide whether a Thai
clinic's Buddhist-era
entry date is published as
2567 or converted to 2024"] + T62["62 · task
Finish the pre-bar
classifier audit — the two
causes and the one bulk
population it did not
reach"] + T63["63 · task
The cleaned stage has
4,949 cells with no cause,
because the ID spelling
that explains them is gone
by then"] + T64["64 · task
Rewrite every docstring
and doc that explains the
code by what R did"] + end + subgraph DROPPED["Out of scope · 1"] + direction TB + T1["1 · grilling
Does product-pipeline's
test suite meet the same
cell-by-cell rigor as
patient's?"] + end + + T2 --> T15 + T3 --> T2 + T3 --> T4 + T3 --> T5 + T3 --> T6 + T3 --> T10 + T3 --> T11 + T3 --> T13 + T4 --> T5 + T4 --> T6 + T5 --> T6 + T6 --> T9 + T7 --> T8 + T8 --> T3 + T8 --> T6 + T10 --> T6 + T11 --> T6 + T12 --> T6 + T13 --> T6 + T13 --> T10 + T14 --> T6 + T20 --> T6 + T21 --> T6 + T22 --> T6 + T23 --> T6 + T45 --> T46 + T64 --> T6 + + classDef frontier fill:#1f6feb,stroke:#0b3d91,stroke-width:3px,color:#ffffff + class T6,T16,T34,T35,T40,T41,T65 frontier + classDef blocked fill:#6e7781,stroke:#424a53,stroke-width:1px,color:#ffffff + class T9 blocked + classDef decided fill:#1a7f37,stroke:#116329,stroke-width:1px,color:#ffffff + class T2,T3,T4,T5,T7,T8,T10,T11,T12,T13,T14,T15,T17,T18,T19,T20,T21,T22,T23,T24,T25,T26,T27,T28,T29,T30,T31,T32,T33,T36,T37,T38,T39,T42,T43,T44,T45,T46,T47,T48,T49,T50,T51,T52,T53,T54,T55,T56,T57,T58,T59,T60,T61,T62,T63,T64 decided + classDef dropped fill:#eaeef2,stroke:#afb8c1,stroke-width:1px,color:#57606a + class T1 dropped +``` + + +## Notes + +- **This map carries execution** (overrides wayfinder's plan-only default): once + a ticket's decision is made — or immediately, for Task-type tickets — the + same session also implements it, using `tdd` / `systematic-debugging` / + `executing-plans` (the superpowers skills) where applicable, rather than + stopping at the decision. + - Guardrail: commits/pushes to any branch are fine, fully revertable. + - Guardrail: merging a pull request is a human-only action, never the agent's + — prepare the merge, stop short of clicking it. + - Guardrail: triggering the real production GCP run / spending GCP budget is + the user's job. Read-only GCP access (logs, BigQuery, GCS listings) and + sandbox/POC runs are fine. + - Guardrail: the cloud routine's checkout has no GCP credentials and no + access to the real tracker files (patient/clinic data — sensitive, not + committed to the repo, not shared with the cloud sandbox). It must work + only with what's already in the repo (code, fixtures, synthetic test + data). If a ticket turns out to need real trackers or live GCP auth to + proceed, it stops and hands that specific piece back to the user to run + locally — it never fabricates substitute data or asks for real patient + data to be pasted into the session. +- A daily cloud routine (`a4d-migration-wayfinder-daily`, 9am Europe/Berlin) + works this map one session at a time — claims the top frontier ticket, does + what it can autonomously, and posts findings/questions for HITL tickets + rather than answering them itself. Kept on cloud rather than local + `launchd` deliberately: it only needs to fetch code and run it, not touch + sensitive data or production GCP — see guardrails above. +- Domain: A4D medical tracker data pipeline, R-to-Python migration. See + [CLAUDE.md](../../CLAUDE.md) and [docs/CLAUDE.md](../CLAUDE.md) for the + codebase map, and [MIGRATION_GUIDE.md](../archive/MIGRATION_GUIDE.md) for + the migration's own history (note: the copy on `migration` is stale relative + to `product-pipeline`'s copy, which claims Phases 0-9 complete — that claim + is unverified, which is exactly what this map exists to check). +- Standing preference: no notebooks for analysis, ever — write scripts. + Analysis/reports should be automated and stay in sync with the code, not + hand-written documents (PDFs, notebooks) that go stale. +- Standing preference: verification means cell-by-cell (shape, columns, exact + values), not spot-checks — this is why the patient pipeline validation took + as long as it did (174 trackers), and the product pipeline is held to the + same bar. +- Standing preference (decided in the [test-rigor](tickets/01-product-pipeline-test-rigor.md) + session): the migration is not 1:1 R-parity. Python may correctly diverge + from R — R can be wrong. Judge divergence against the original source Excel + trackers (`a4dphase2_upload` on the test-data drive), not against R's output + alone. R-vs-Python comparison is *analysis* (a judgment call), not a pytest + concern — pytest covers unit/integration/e2e/regression only. +- User sequencing preference: make `product-pipeline` ready first, then merge, + then make `migration` ready, then promote. Tickets are blocked accordingly + even where the underlying dependency is looser than the sequencing implies. + "Ready" for the merge means tests green (ticket 8) plus ordinary pre-merge + hygiene (code style, an implementation review confirming R's steps are + actually migrated, doc alignment with patient) — **not** R/Python output + parity. The comparison script (ticket 2) is explicitly post-merge: it only + makes sense once patient and product share one branch, and the user judges + green tests + a clean review sufficient grounds to merge without it + (corrected mid-session on 2026-08-08, after ticket 2 had originally been + wired as a merge blocker). +- **Standing preference — triage means deciding, not labelling** (set by the + user 2026-08-12g, after [ticket 27](tickets/27-triage-patient-raw-residual.md) + had to be reopened for exactly this). Every flagged R/Python difference + has to clear two bars, not one: + 1. **Explain the diff** — the actual mechanism, traced to source Excel or + to R's/Python's own code, not a shape-matching heuristic. + 2. **Decide whether Python is doing the right thing** — and say so + explicitly, with what was checked. + + **Refined by the user 2026-08-13**: the bar is a clear *understanding*, + not a verdict on which pipeline is right. Deciding R-vs-Python correctness + is not this phase's job. Where the evidence shows the **source file + itself is corrupt**, "the source is wrong, this tracker needs human + inspection" is a legitimate final conclusion — and a valuable finding in + its own right, not a failure to converge. + + **Scoped by the user 2026-08-15**: the current tracker template is the + golden rule. A column that appears in one year and is gone the next was + very likely a test that did not survive, so triage effort goes to the core + fields that run through every year. Where a divergence traces to a human + error in the source workbook, the right fix is the workbook, not inference + in the pipeline -- so such findings are detected and reported (see [ticket + 40](tickets/40-source-defect-findings-report.md)) rather than guessed at. + + Observing "Python has A where R has B" and adding a named classifier is + **not** a decision in favour of A. A classifier records that a difference + is understood; it says nothing about whether Python is correct, and + writing one is not permission to stop. Where Python turns out to be + wrong, or to be losing information the source file carried, the pipeline + gets fixed — ticket 27's precedent: what looked like a labelling job was + really extraction silently discarding data, and the classifier would have + cemented the bug as "explained". A cause that is genuinely undecidable + from the evidence available is recorded as an open question, not closed + with a label. +- **R is retired.** `r-archive/` was deleted 2026-08-24 by [ticket + 12](tickets/12-retire-r-workspace.md). The rule that governed it for eleven + rounds -- *R retires only when nothing still needs to read it*, the user's + 2026-08-20 standing instruction that made ticket 12's `blocked_by` a derived + list -- is now discharged and is recorded here as history, not as a live + constraint. To read R again: `git show r-archive-removed^:r-archive/R/` + (the `r-archive-removed` tag marks the removal commit, so `^` is the last + state containing it). The frozen output baseline on the data drive + (`output_r/`) is untouched by any of this and remains the comparison's + reference -- it never lived in the repo. +- Redraw command: `~/.claude/skills/wayfinder/scripts/render-map.sh docs/wayfinder` + +## Where this map stands + +Three tickets resolved. [Does product-pipeline's test suite meet the same +cell-by-cell rigor as patient's?](tickets/01-product-pipeline-test-rigor.md) +was superseded — it presupposed R-parity was the goal and that patient's +exception-dict pytest pattern was the bar to replicate for product; the user +rejected both, and it split into tickets 7 and 8. [Is the product pipeline +(and patient's own claimed completeness) actually complete and sound, audited +against R's product logic and patient's structure?](tickets/07-pipeline-completeness-audit.md) +is decided (research, read-only): R-logic coverage is essentially +complete on both pipelines, but product has real structural test gaps +(no integration/e2e tests, no coverage of `wide_format.py`, no +`test_tables/test_product.py`) and patient's "174 trackers validated" claim +has no committed record of an actual passing run. Full detail: +[research/07-pipeline-completeness-audit.md](research/07-pipeline-completeness-audit.md). +[Does the pytest suite reach unit/integration/e2e/regression parity between +patient and product, excluding any R-comparison/USB-drive-dependent +tests?](tickets/08-pytest-suite-parity.md) is now decided: parity means an +85%+ coverage floor enforced in CI plus product gaining the three +integration/e2e files it's missing (mirroring patient's existing +fixture/skip-if-missing convention) and `test_tables/test_product.py`; +`test_r_validation.py` leaves pytest entirely, with no product equivalent. +Mid-session the user clarified "regression test" means golden-master/snapshot +testing (fixed input, output snapshotted per stage, diffed on future +changes) rather than R-comparison or edge-case testing — that work doesn't +need real/sensitive data and was split off into [Add golden-master/snapshot +regression tests for patient and product](tickets/09-snapshot-regression-tests.md), +which the user wants deferred until both pipelines' other test suites are in +place and green. + +**Four tickets resolved.** [Merge product-pipeline (PR #6) into +migration](tickets/03-merge-product-pipeline.md) is done: ticket 8's test +files, the product-only coverage gate, `test_r_validation.py` removal, a +repo-wide ruff/ty cleanup, an implementation review that found and fixed two +real logging-parity gaps (product never populated `TrackerResult.data_errors` +or created a logs/errors table, unlike patient), and doc alignment (verified, +no changes needed) are all implemented and pushed to `product-pipeline`. PR +#6's conflicts (`gcp/bigquery.py`, `cli.py`) are resolved — it is now +`mergeable: MERGEABLE`. Landing it is left to the user (human-only merge +guardrail). Full detail: [ticket 3](tickets/03-merge-product-pipeline.md). + +**Ticket 2 was re-sequenced** (same session ticket 3 closed in — the user +corrected this): it no longer blocks the merge, since the comparison script +only makes sense once patient and product share a branch, and green tests + +a clean review is judged sufficient trust to merge without it first. Ticket 2 +is blocked on ticket 3 instead of the reverse (`blocked_by: [3]`) — and since +ticket 3 is now closed, ticket 2 is unblocked. + +**Five tickets resolved.** [Diagnose and fix why CI is red at migration +HEAD](tickets/04-fix-migration-ci.md) is done: root cause was Typer's +`FORCE_TERMINAL` freezing to `True` at import time because GitHub Actions +always sets `GITHUB_ACTIONS=true`, which forces colorized `--help` output +that splits options like `--file` into separate ANSI spans and breaks +plain substring assertions — independent of the `NO_COLOR`/`COLUMNS` +overrides already in place. `tests/conftest.py` on `product-pipeline` now +sets Typer's own `_TYPER_FORCE_DISABLE_TERMINAL` escape hatch before +`typer.rich_utils` is first imported. Pushed as `b970cf6`; PR #6's CI is +green (428 tests), and PR #6 is `mergeable: MERGEABLE`. Full detail: +[ticket 4](tickets/04-fix-migration-ci.md). + +**PR #6 was merged by the user** (2026-08-09, merge commit `7713fea`, +human-only action per this map's guardrails). `migration` now contains the +combined patient + product pipeline; CI on the merge commit is green +(`gh run list --branch migration` — run `31286057254`, `conclusion: +success`). PR #2 (`migration` -> `dev`) is `mergeable: MERGEABLE`. This +closes out the "make `product-pipeline` ready, then merge" half of the +user's sequencing preference; what remains before ticket 6 (promotion) is +"make `migration` ready" — tickets 2 and 5. + +Key facts already gathered while charting (verified via `git`/`gh`, not +assumed): PR #2 (`migration` -> `dev`) is open and mergeable, but CI has +failed on `migration` HEAD for its last 3 runs (ticket 4's target). +`source_vs_output_product.py` is deliberately group-granularity only ("v1"), +not cell-by-cell, per its own docstring. `PYTHON_IMPROVEMENTS.md`'s parity +claims cite a notebook (`Ali_internship/residual_dig.ipynb`, not in the +tracked tree) and a patient-only comparison script — i.e. one-off analysis, +not a repeatable test. Its two PDF reports haven't been read yet — ticket 2's +remit. + +**Six tickets resolved.** [Define and execute the real GCP production +verification run](tickets/05-production-verification-run.md) is done: the +combined patient + product pipeline ran for the first time as one execution +via the existing `a4d-pipeline` Cloud Run Job against real production +GCS/BigQuery, preceded by a `just backup-bq` snapshot, and verified clean +against that snapshot (row counts, distinct clinics, schema — not R, which +the user decided is out of this ticket's scope). Full detail: [ticket +5](tickets/05-production-verification-run.md). + +**Four tickets were added mid-session, none resolved**: [Profile the +combined pipeline's performance against the R baseline before promoting to +dev](tickets/10-performance-profiling.md) — the user's standing understanding +that Python is much faster than R predates this merge's additions and hasn't +been re-checked; [Decide what CLI/TUI UX and error-log observability +improvements admins/developers need before rollout](tickets/11-cli-ux-observability.md), +graduated from fog once the user confirmed it's in this map's scope; [Retire +R from the workspace once the pipeline is fully verified +Python-only](tickets/12-retire-r-workspace.md) — `r-archive/`, +`tools/LogViewerA4D` (an R Shiny log viewer another developer wrote, now +confirmed removable outright with no Python replacement needed), and a stray +root `test_full_pipeline_debug.R`, gated on tickets 2 and 10 since both still +need R as a live reference; and [Audit and update all dependencies and +library versions before rollout](tickets/13-dependency-audit.md), wired ahead +of ticket 10 so profiling doesn't run against a soon-to-change dependency +set. All four are new blockers on ticket 6. **The destination itself was +redrawn** to name all four concerns explicitly (see Destination section) — +this map now covers operational rollout readiness, not just "merge, verify, +promote." + +**Seven tickets resolved.** [Audit and update all dependencies and library +versions before rollout](tickets/13-dependency-audit.md) is done: see the +Decisions-so-far entry above for detail. This unblocks [ticket +10](tickets/10-performance-profiling.md) (performance re-profiling against +the R baseline), since its other blocker (ticket 3) was already closed. + +**Eight tickets resolved.** [Profile the combined pipeline's performance +against the R baseline before promoting to dev](tickets/10-performance-profiling.md) +is done — retitled in substance mid-session: the user dropped the R-baseline +comparison entirely (R's already known to be slower; re-confirming that +teaches nothing) and reframed it as a function-level performance/robustness +profile of the Python pipeline itself, run with `pyinstrument` against the +full 177-tracker real dataset (from the USB drive, at the user's suggestion, +since current production trackers aren't available locally). That profile +found `find_data_start_row` (`src/a4d/extract/patient.py`) was O(n^2) on +read-only worksheets — each `.cell()` call re-parses a sheet's XML from row +1 — and fixed it with a single sequential scan: 6.6x speedup on the full +patient arm (145.8s -> 22.0s, 171 real trackers, 4 workers, identical +output), pushed as `97479f8` with a regression test. Full combined +patient+product run: 73.86s wall, ~556MB peak RSS (rough floor, not +exhaustive). Also surfaced 4 product trackers failing outright +("unable to find column \"product\"") — unrelated to the performance fix, +not previously documented anywhere on this map — spawned as [ticket +14](tickets/14-product-column-detection-failures.md) rather than fixed +inline, now also blocking ticket 6. Full detail: [ticket +10](tickets/10-performance-profiling.md). + +**Nine tickets resolved.** [Fix product pipeline's "unable to find column +product" failures on 4 real trackers](tickets/14-product-column-detection-failures.md) +is done: confirmed legitimate (2018-2020 KBH and 2020 JVM trackers predate +product/stock tracking entirely — no "product" keyword and no `INV` sheet +anywhere in any of their month sheets, verified directly against the real +files on the USB drive; product tracking starts with KBH's 2021 `INV` +sheet), not a synonym gap or regression. The actual bug was in +`clean_product_data` (`src/a4d/clean/product.py`): it assumed a `"product"` +column always exists and crashed on the fully columnless frame extraction +correctly hands back for these trackers, because `apply_schema` invents a +spurious 1-row output from a columnless input instead of preserving 0 rows. +Fixed with an early return to an empty, schema-conformant (0, 20) DataFrame +when the raw frame has no columns. Reproduced the original crash and the fix +against the real files; all 4 now process successfully with 0-row output. +One regression test added; full suite (490 tests), ruff, `ty check` all +pass. Full detail: [ticket 14](tickets/14-product-column-detection-failures.md). + +**Ten tickets resolved.** [Retire the PDF/notebook analysis docs for an +automated, script-based report](tickets/02-documentation-strategy.md) is +decided (design only, not built): the R baseline is the already-frozen +`/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_r/` (dated 2025-11-14, same +still-unchanged trackers, covers both patient and product) — no R re-run, +ever. The comparison script (`scripts/compare_outputs.py` + `just +compare-outputs`, not `a4d.cli`, since it's migration-only tooling that dies +with R's retirement) is decoupled from pipeline execution — it diffs two +existing output directories — and runs four layered checks (shape, totals, +columns, cell-by-cell), with an extensible cause-classifier registry seeded +from the four causes already known from the parity-presentation PDF, output +as an HTML report. Docs cleanup done inline: the unrelated dashboarding PDF +deleted, the parity-presentation PDF kept (holds the real numbers needed to +validate the new script; removed only once superseded), and +`PYTHON_IMPROVEMENTS.md`'s dead notebook citation fixed to point at that PDF. +`test_r_validation.py` was confirmed already gone from pytest (removed in the +ticket 3/8 merge). Actually building the script and triaging the flagged +differences (the bulk of the real work — most causes aren't known ahead of +time) was explicitly deferred, spawning [ticket +15](tickets/15-build-and-run-comparison-script.md). Full detail: [ticket +2](tickets/02-documentation-strategy.md). + +Closing ticket 2 also required correcting two other tickets whose premises +assumed R needed to stay live in this repo for ticket 2's work: [ticket +12](tickets/12-retire-r-workspace.md) is now `blocked_by: [15]` instead of +`[2, 10]` (retiring `r-archive/` was never going to touch the frozen +USB-drive baseline, so the real remaining reason to wait is ticket 15's +investigative work possibly needing one more look at R's behavior, not the +archive itself), and [ticket 6](tickets/06-promote-migration-to-dev.md)'s +`blocked_by` swaps `2` for `15`, since the destination's requirement that +"every Python/R difference [be] documented and explicitly decided" isn't met +until ticket 15 actually runs the comparison, not just designs it. + +**Eleven tickets resolved.** [Decide what CLI/TUI UX and error-log +observability improvements admins/developers need before +rollout](tickets/11-cli-ux-observability.md) is decided and implemented, for +its CLI/UX half: `run-pipeline` (the actual production entry point behind +the Cloud Run Job) was found to never render any of the rich per-arm summary +tables `process-patient`/`process-product` already have — it only printed a +one-line count per arm, with no cross-arm view. Demonstrated the gap with a +real synthetic-data run rather than reasoning about it, then built and +shipped `_render_combined_run_summary()`: a combined patient+product view +crossing each file's outcome into four buckets (both ok / patient failed +only / product failed only / lost entirely) plus a merged per-file error +count. That exposed a real behavior bug — `run-pipeline` aborted the entire +run on any single patient tracker failure, before the product arm even ran, +which made "patient-only failed" structurally unobservable — fixed by +switching the patient arm to the same soft-fail-and-continue posture the +product arm already used. Fixing that in turn exposed a real test-isolation +bug: two `run-pipeline` CLI tests mocked only `run_patient_pipeline`, so +`run_product_pipeline` silently processed 185 real local tracker files as a +side effect once soft-fail let execution reach it; fixed by mocking both +arms in every `run-pipeline` test. Full suite (494 tests), ruff, `ty check +src/` all pass; pushed as `33694b4`. The error-log observability half (a +drill-down view into one specific file's full detail, replacing +`LogViewerA4D`'s job) was explicitly split off rather than answered here — +spawned as [ticket 16](tickets/16-log-analyzer-drill-down.md), including its +own open question of whether it's needed before promotion or is a +nice-to-have outside the promotion path. Full detail: [ticket +11](tickets/11-cli-ux-observability.md). + +**Twelve tickets resolved.** [Build and run the R/Python output comparison +script, then triage every flagged difference](tickets/15-build-and-run-comparison-script.md) +is done for its infrastructure half: `src/a4d/migration/compare.py` (four +pure, unit-tested layers — shape, totals, columns, cell-by-cell — plus the +seeded cause classifier and HTML report renderer, 23 tests, TDD'd) and +`scripts/compare_outputs.py` + `just compare-outputs` were built per ticket +2's design. A fresh Python pipeline pass ran against `a4dphase2_upload` +(both arms), landing on the USB drive next to the frozen `output_r/` +(replacing a stale 2025-11-15 copy) — 174/174 patient and 174/174 product +trackers succeeded. Running the comparison found the **patient row-alignment +key is sound** (`patient_id` + `sheet_name`, duplicate keys in only 5/172 +files) but the **product row-alignment key is broken**: `product_entry_date` +being null on many rows collapses the key onto far fewer distinct values +than rows exist (one file: 154 distinct keys for 1,194 rows, up to 35-way +duplication), which both inflates reported mismatches by orders of magnitude +via join fan-out and silently excludes `product_entry_date` — the PDF's +largest single divergence column — from classification entirely, since it's +one of the join keys. Building a corrected alignment strategy and actually +triaging the flagged differences (for both arms) was split off into [ticket +17](tickets/17-fix-product-row-alignment-and-triage.md), per ticket 15's own +pre-authorization to split if triage didn't converge in one session. Full +detail: [ticket 15](tickets/15-build-and-run-comparison-script.md). + +**Same session, continued past ticket 15's closure** (recorded in its +[addendum](tickets/15-build-and-run-comparison-script.md#addendum-same-session-after-closure) +rather than as a new ticket, since it's the same deliverable maturing, not a +new decision): the comparison tool was substantially hardened at the user's +direction. HTML output was dropped entirely in favor of Excel (triage means +loading results as a dataframe, filtering, sorting, adding columns — a +static HTML page doesn't support that); `compare_id_overlap` and +`compare_categorical_overlap` were added (identity/label-set checks +independent of the row-alignment key, which confirmed product *names* match +100% between R and Python even where cell comparison is meaningless); +`compare_row_key_overlap` was added after the user noticed cell-mismatch +counts near 0 for raw product files didn't add up — it measures, per file, +how many rows found *no partner at all* via the full row-alignment key, and +confirmed those near-0 cell counts meant "nothing was paired to compare," +not agreement (one file: 560/560 rows unmatched). Raw pipeline output +(`patient_data_raw`/`product_data_raw`) is now compared alongside cleaned, +one report per stage, so a divergence can be localized to extraction vs. +cleaning. All display names were made consistent ("X divergence" for every +count-based check). [Ticket 17](tickets/17-fix-product-row-alignment-and-triage.md) +inherits this more capable tool — its premise was updated to note +`RowKeyOverlap.matched` can validate a proposed alignment fix directly. + +**Thirteen tickets resolved.** [Fix the product comparison's row-alignment +key, then triage every flagged R/Python difference](tickets/17-fix-product-row-alignment-and-triage.md) +is done for its row-alignment-key half: the old equi-join key (`clinic_id`, +`product`, `product_sheet_name`, `product_entry_date`) is replaced by +`add_row_ordinal()` (`src/a4d/migration/compare.py`) — ordinal position +within `(clinic_id, product_sheet_name)`, computed at comparison time +(never stored, since R's frozen baseline can't be re-run to pick up a new +column), with whitespace-normalized group keys so a divergence like R's +un-trimmed sheet names still surfaces as an ordinary cell mismatch instead +of breaking alignment. Along the way, found R's own `index` helper +(`clean/product.py` step 2.5) resets per-sheet while Python's is a single +global counter across the whole file — a real, previously undocumented +divergence between the two pipelines, though not one this ticket needed to +fix in the pipeline itself. Validated directly against the real R/Python +output pair on the USB drive: row-key match on cleaned product jumped from +near-0% to 97.2% (46,314/47,644), with the remaining 2.8% isolated entirely +to one clinic — R's frozen output has `clinic_id = "NGH"` for North +Okkalapa General Hospital across all three of its tracker years, where +Python (matching the tracker's actual folder name) correctly has `"NOH"`. +`product_sheet_name`'s mismatch count (201) now reproduces the +parity-presentation PDF's number exactly, the strongest available +confirmation the new key is sound. Actually triaging the flagged +differences did not converge in this session — initial per-column counts +are still far larger than the PDF's even with alignment fixed +(`product_category`: 13,638 vs 214; `product_entry_date`: 10,424 vs 559), +and a first look found `product_category` mismatches are systematically +`r_value=None` where Python has a real value, an unexplained pattern rather +than a diagnosis — so that work, plus patient-arm triage and the +still-unreconciled 189-vs-155-tracker discrepancy (confirmed unchanged by +this ticket's fix — the currently-frozen `output_r/` genuinely has 155 +files/47,644 rows), split into [ticket +18](tickets/18-triage-comparison-flagged-differences.md). Full detail: +[ticket 17](tickets/17-fix-product-row-alignment-and-triage.md). + +**Fourteen tickets resolved.** [Persist comparison run history and show +run-over-run deltas](tickets/19-compare-run-history-deltas.md) is done, +spawned and closed in the same session as ticket 17: a same-session +conversation about how to work ticket 18 settled that cell-mismatch causes +should be classified opportunistically (start `unclassified`, name a cause +the moment a real pattern is noticed, drop it if it doesn't earn its keep) +rather than pre-built into a taxonomy, and that a fix's actual effect should +be visible by count rather than needing classification to prove it worked — +which needed run-over-run tracking that didn't exist yet. +`snapshot_from_summary()`/`compute_deltas()` (`src/a4d/migration/compare.py`) +reduce/diff `build_summary_rows()`'s per-column/per-cause counts. Refined +twice more in the same session after user review: `--report-out` (a single +misleadingly-named filename the tool derived four sibling files from) was +renamed to `--output-dir`, and — on explicit request — every stage of one +`just compare-outputs` run now writes into a shared subfolder named by that +run's own UTC timestamp (`output/comparison//`), so a run is +a self-contained unit on disk (its four Excel reports and four JSON +snapshots together) rather than scattering same-named files across the cwd +every time. The CLI prints + writes a delta (previous vs. current, +red/green) against the most recent prior *run folder's* snapshot for that +stage. Same review also prompted a `scripts/` cleanup: ten stale one-off +debug scripts predating this map's work (hardcoded to a drive layout that +no longer exists) were removed, superseded by the pytest suite, ticket 10's +profiling approach, or this comparison tool. Validated end-to-end against +the real drive data across both layouts (no-previous-run, no-change, and a +synthetic-edit check that the delta table and Excel sheets render +correctly) — not just unit-tested. This is tooling ticket 18 will use, not +part of ticket 18's own triage scope. Full detail: [ticket +19](tickets/19-compare-run-history-deltas.md). + +**Fifteen tickets resolved.** [Triage every flagged R/Python difference for +both arms, and resolve the 189-vs-155-tracker discrepancy](tickets/18-triage-comparison-flagged-differences.md) +is done for its diagnostic half: the 189-vs-155-tracker gap is resolved as +unreconcilable — every product-output snapshot that exists anywhere on the +USB drive (`output_r/`, `output_vm/`, `output.zip`, +`a4dphase2_upload/output/`, `a4dphase2_upload.zip`) tops out at 174 files; +none reach 189, so the parity-presentation PDF's baseline predates +everything now on the drive and per-column counts should be judged by +pattern, not by exact reproduction of the PDF's numbers. `product_category` +(13,638 mismatches) is 100% `r_value=None`-with-Python-present, root-caused +to `read_product_data.R`'s `add_product_categories()` doing a case/whitespace- +sensitive left-join with no normalization (`src/a4d/reference/products.py` +normalizes both sides) — a genuine, verified R limitation, not a Python +defect. The dominant share of `product_entry_date` mismatches (10,211 of +10,424) share the same R-null pattern; a concrete spot-check against the +real source Excel (`2018_Mahosot Hospital A4D Tracker`, sheet `Jan18`) found +R nulling out a perfectly clean, unambiguous date present on every row — a +plain R extraction gap, not a typo, so the seeded `typo_rescue` classifier +(which just matched any R-null case) was renamed `r_value_missing` to stop +mislabeling it. A new `PRODUCT_CATEGORY_CLASSIFIERS` registry +(`r_category_lookup_miss`) was added to `src/a4d/migration/compare.py`; +`tests/test_migration/test_compare.py` updated and passing (60 tests). Also +found the raw-stage `product_entry_date` report is ~99% a representation +artifact (R stores unparsed Excel serials as strings, Python stores already- +parsed dates) rather than real divergence — not fixed this session. The +remaining product columns (`product_balance`, `product_received_from`, +`product_released_to`, `product_remarks`, `product_units_received`, +`product`, both stages) and the entire patient arm (never triaged since +ticket 15 ran it) didn't converge in this session either — split into +[ticket 20](tickets/20-fix-raw-entry-date-representation.md) (raw-stage +entry_date representation fix), [ticket +21](tickets/21-triage-remaining-product-columns.md) (remaining product +cleaned-stage columns), [ticket +22](tickets/22-triage-product-raw-columns.md) (remaining product raw-stage +columns), and [ticket 23](tickets/23-triage-patient-arm.md) (patient arm, +both stages). Full detail: [ticket +18](tickets/18-triage-comparison-flagged-differences.md). + +**The frontier was tickets 16, 20, 21, 22, and 23**; ticket 20 is now closed +(see above), leaving [Build a drill-down log analyzer for admins to inspect +a specific tracker file's errors/logs](tickets/16-log-analyzer-drill-down.md), +[Triage the remaining product cleaned-stage column mismatches](tickets/21-triage-remaining-product-columns.md), +[Triage the remaining product raw-stage column mismatches](tickets/22-triage-product-raw-columns.md), +and [Triage every flagged R/Python difference for the patient arm](tickets/23-triage-patient-arm.md) +as the frontier. Ticket 12 is blocked_by `[20, 21, 22, 23]`, so ticket 20's +closure doesn't unblock it yet — tickets 21-23 still remain. Ticket 6 +(promote to `dev`) is `blocked_by: [8, 3, 4, 5, 10, 11, 12, 13, 14, 20, 21, +22, 23]` — tickets 3, 4, 5, 8, 10, 11, 13, 14, and now 20 are closed; ticket +12 and tickets 21-23 are what remain (ticket 16 isn't wired as a blocker +yet — its own question 4 is whether it should be). The user has said they +intend to keep working this map session by session on `migration` until +confident enough to roll out, rather than promoting early. + +Also noted, not yet acted on: a local, untracked `a4d-python/` directory at +the repo root (stale leftover copy predating the current `src/` layout, not +in git) — the user hasn't yet said whether to delete it; separate from +ticket 12's git-tracked R cleanup. + +**Same day, after ticket 18 closed:** production tracker uploads grew from +177 to 248 files (confirmed via `a4d download trackers` against live GCS, +read-only). The user directed a one-time R re-run against the current +tracker set to get comparable numbers — R code unchanged, framed +explicitly as the final capture before ticket 12 removes R, not a reversal +of ticket 2's "no R re-run, ever" decision. `output_r/` now holds fresh R +output (229 product-cleaned files/66,020 rows, 243 patient-cleaned +files/81,859 rows) with the old 155-file baseline preserved at +`output_r_155_frozen_backup_2025-11-14`. This substantially changed +ticket 18's picture: `product_category` mismatches dropped from 13,638 to +866 (most of the original count was baseline staleness, not the +case-sensitive-join bug itself, which is still real for what remains), the +row-alignment key now matches 100% (up from 97.2%), and the 189-vs-155 +question is now much closer to reconciled (229/66,020 vs the PDF's +189/61,077) though not re-verified as exactly resolved. Tickets 20, 21, 22, +and 23 all had their premise numbers refreshed against the new baseline +(none needed re-scoping — the columns/questions they ask are unchanged). +Full detail: [ticket 18's addendum](tickets/18-triage-comparison-flagged-differences.md#addendum-same-day-after-closure-r-re-run-against-the-current-tracker-set). +Also fixed in passing: `justfile`'s `compare-outputs` recipe default was +out of sync with the script's own documented default, a leftover from +ticket 19 — now both say `output/comparison`. + +**Sixteen tickets resolved.** [Normalize the raw-stage product_entry_date +comparison so it stops flagging near-universal false mismatches](tickets/20-fix-raw-entry-date-representation.md) +is done: `normalize_date_column()` (`src/a4d/migration/compare.py`) reuses +the cleaning stage's own flexible date parser (`a4d.clean.date_parser.parse_date_flexible`, +Excel-serial and typo-rescue handling included) to parse both sides' +raw `product_entry_date` strings to a common `date` before diffing, wired +in for the `Product (raw)` stage only via a new `date_normalize_cols` field +on `scripts/compare_outputs.py`'s `STAGES` table. Verified against the real +`output_r`/`output_python` on the USB drive: raw-stage `product_entry_date` +mismatches dropped from 65,743 to 91 (99.86% was the representation +artifact). Triaged the remaining 91: 46 already land in the existing seeded +classifiers, and the other 50 — spread across 14 files, including a +confirmed real bug (a stray `"\n"`-dated row in Python's raw extraction for +`2020_Sarawak General Hospital..._May20`) and several files showing a +similar single-row insertion/shift pattern — were folded into [ticket +22](tickets/22-triage-product-raw-columns.md)'s scope rather than left +untracked in this closed ticket, per the destination's "every difference +explicitly decided" bar. Full detail: [ticket +20](tickets/20-fix-raw-entry-date-representation.md). + +**Seventeen tickets resolved.** [Triage the remaining product raw-stage +column mismatches](tickets/22-triage-product-raw-columns.md) is done: found +and fixed a real Python bug (`remove_header_rows` in +`src/a4d/extract/product.py` missed rows where every cell was blank except +one stray empty string from a formula-emptied Excel cell -- R's `is.na()` +check drops these, Python's `is_null()` check didn't), which turned out to +be the root cause of the "single-row insertion/shift pattern" ticket 20 had +flagged but not chased (confirmed: the Sarawak `"\n"`-dated row is gone +too, 191/191 rows). Added two more comparison-tool normalizations +mirroring ticket 20's `normalize_date_column` precedent: +`normalize_numeric_column()` (R's and Python's own float-to-string +conversions round a raw float's trailing digits differently -- parse both +back to `float` so the existing tolerance applies) and +`normalize_whitespace_column()` (readxl's `trim_ws=TRUE` default strips +whitespace R-side that openpyxl preserves, and readxl represents an +embedded line break as `\r\n` where openpyxl normalizes to `\n`). Verified +end-to-end against the real drive data (248-tracker set): raw-stage product +mismatches across all 9 columns dropped from 2,007 to 105 (95%), fully +resolving `product`, `product_balance`, `product_entry_date` (now 0 +`unclassified`), `product_remarks`, `product_released_to`, and +`product_units_returned`. The 105-row residual across +`product_units_received`, `product_units_released`, and +`product_received_from` didn't converge to a single cause (at least two +more distinct patterns found, one possibly systemic) -- split into [ticket +24](tickets/24-triage-remaining-raw-column-residual.md) per the map's +"split rather than sprawl" rule. Full suite (571 passed), ruff, `ty check +src/` all pass. Full detail: [ticket +22](tickets/22-triage-product-raw-columns.md). + +**Eighteen tickets resolved.** [Triage the remaining product cleaned-stage +column mismatches](tickets/21-triage-remaining-product-columns.md) is done: +five of its six columns (`product_balance`, `product_received_from`, +`product_released_to`, `product_remarks`, `product_units_received`) turned +out to share one root cause, already half-diagnosed by ticket 18 — +R's per-(clinic, sheet, product) row sort falls back to raw input-row order +whenever `product_entry_date` fails to parse, which is *near-universal* +(confirmed 100% of "change"-status rows null for several major clinics, +e.g. 2024/2023/2022 Mahosot) rather than occasional, while Python correctly +parses the same cells and sorts chronologically instead — both sides +implement the identical documented rank algorithm, so this is R's +already-known date-extraction gap resurfacing as a *sort-order* divergence, +not a Python bug. Since the row-alignment key (`add_row_ordinal`) is purely +positional, that legitimate order difference cascades into value-level +mismatches on every column compared through it, even though the underlying +data (e.g. end-of-group balance, confirmed matching in 98.3% of affected +groups) is unaffected. Added a `row_order_divergence` classifier +(`PRODUCT_ROW_ORDER_CLASSIFIERS` in `src/a4d/migration/compare.py`, backed +by a new opt-in `order_group_cols` parameter on `compare_cells`): fully or +almost fully explains four of the five columns (100%, 100%, 100%, 97.4%); +`product_balance`'s cumulative running total under-detects via simple +value-membership (27.9% caught directly) despite sharing the same root +cause, left as future work. The sixth column, `product` (652 mismatches), +had a different, single cause — an embedded `\r\n`-vs-`\n` line break that +ticket 22 already normalizes for the raw stage but that survives cleaning; +extending `normalize_whitespace_column` to the cleaned stage (scoped to +`product` alone) resolved it fully (652 -> 0). `product_units_received`'s +residual 8 mismatches are the same Excel-date-serial-leak pattern [ticket +24](tickets/24-triage-remaining-raw-column-residual.md) already flagged for +the raw stage, now confirmed to reach the cleaned stage too — left for +ticket 24. Discovered `product_units_released` (2,144 cleaned-stage +mismatches) was never assigned to any ticket — spawned [ticket +25](tickets/25-triage-product-units-released-cleaned.md). Full suite (577 +tests), ruff, `ty check src/` all pass; verified end-to-end against the real +248-tracker drive data. Full detail: [ticket +21](tickets/21-triage-remaining-product-columns.md). + +**Gap found, not a session's answer: `compare_columns` (the CLI's `Column +divergence` count — column existence + dtype, one of the comparison tool's +four structural layers since ticket 15) was never triaged by any ticket.** +Every triage ticket since worked exclusively off the `cell_mismatches` +sheet; this structural layer sat unaddressed in the CLI output the whole +time. Spawned [ticket 26](tickets/26-triage-product-column-divergence.md) +with concrete current-data findings (product cleaned stage: 4 +divergences in every one of 229 files — +`orig_product_released_to` only-in-Python, `product_table_month`/ +`product_table_year`/`product_unit_capacity` Float64-vs-Int32 — plus a 5th, +`product_remarks` Boolean-vs-String, in 103/229 files; raw stage has a +different, smaller set, also unlooked-at; patient not yet checked at all). + +**Nineteen tickets resolved.** [Triage every flagged R/Python difference for +the patient arm](tickets/23-triage-patient-arm.md) is done for its +raw-stage-dominant-cause half: patient raw-stage mismatches turned out to be +the same date-representation artifact [ticket +20](tickets/20-fix-raw-entry-date-representation.md) already fixed for +product's raw `product_entry_date` — R's raw extraction stores the unparsed +source text (an Excel serial for date-formatted cells) while Python's raw +extraction already ISO-formats parsed dates. Wired +`normalize_date_column` into `scripts/compare_outputs.py`'s `Patient (raw)` +stage using the cleaned schema's own `get_date_columns()` helper (18 +`pl.Date` columns, derived rather than hand-listed) rather than pipeline +code. Verified against the real 248-tracker drive comparison: raw-stage +mismatches dropped from 564,096 to 46,788 (91.7%) across 70 -> 67 columns, +with 17 of 18 date columns individually collapsing by 99%+ (`dob`: 79,836 -> +2). The cleaned stage (120,639 mismatches, 61 columns) was confirmed +byte-identically unaffected, as expected. Both the raw residual and the +cleaned stage didn't converge in this session — split into [ticket +27](tickets/27-triage-patient-raw-residual.md) (raw residual, with two +leads already noted: `complication_screening`'s Python side looks like a +genuine multi-select extraction the R side only partially captures, and +`meter_received_date` is a raw-only date column the schema-derived list +didn't catch) and [ticket 28](tickets/28-triage-patient-cleaned.md) (cleaned +stage, entirely untouched). Full suite (83 `test_migration` tests), ruff +clean. Full detail: [ticket 23](tickets/23-triage-patient-arm.md). + +**Ticket 12 is now unblocked.** Its `blocked_by: [20, 21, 22, 23]` is now +fully closed (ticket 23 was the last of the four) — [Retire R from the +workspace once the pipeline is fully verified +Python-only](tickets/12-retire-r-workspace.md) joins the frontier for the +first time this map. It is the ticket most directly on the destination's +critical path: closing it removes one of [ticket +6](tickets/06-promote-migration-to-dev.md)'s eight remaining blockers. +Tickets 24, 25, 26, 27, and 28 are not wired as blockers of ticket 12 (same +precedent as before: they're residuals of already-closed ticket scope, or a +newly surfaced gap the destination doesn't yet name as a promotion +blocker). + +**The frontier is now [Retire R from the workspace once the pipeline is +fully verified Python-only](tickets/12-retire-r-workspace.md), [Build a +drill-down log analyzer for admins to inspect a specific tracker file's +errors/logs](tickets/16-log-analyzer-drill-down.md), [Triage the residual +product_units_received/product_units_released/product_received_from +raw-stage mismatches](tickets/24-triage-remaining-raw-column-residual.md), +[Triage the product_units_released cleaned-stage column +mismatches](tickets/25-triage-product-units-released-cleaned.md), [Triage +the product pipeline's column-existence and dtype +divergence](tickets/26-triage-product-column-divergence.md), [Triage the +residual patient raw-stage column mismatches after date +normalization](tickets/27-triage-patient-raw-residual.md), and [Triage the +patient cleaned-stage column mismatches](tickets/28-triage-patient-cleaned.md) — +seven tickets, up from five. Ticket 12 is the one most directly on the +route to the destination; the other six are independent triage residuals +and a separate feature (ticket 16), takeable in any order.** + +**Ticket 12's "unblocked" status turned out to be premature — caught and +corrected mid-session, before anything was committed.** Taking ticket 12 +this session surfaced a dead-premise problem the previous session's sweep +missed: tickets 20-23 closing doesn't mean the pipeline is actually verified +Python-only, because tickets 24-28 (their own spawned residuals) carry +forward the exact same "might need to consult R's source" risk ticket 12 +already names as its reason for waiting — and past triage has repeatedly +needed to read `r-archive/`'s actual R source to root-cause a mismatch (e.g. +ticket 18 reading `read_product_data.R` directly), not just diff output. +`r-archive/` was briefly staged for deletion outside of ticket 12's own +execution (156 files) and fully restored (`git restore --staged` + +`git checkout --`) before any commit landed — no data was lost, this branch's +git history was never touched. Ticket 12's `blocked_by` is corrected to +`[20, 21, 22, 23, 24, 25, 26, 27, 28]`; its claim was released. Also +corrected in passing: its inventory's `test_full_pipeline_debug.R` item was +stale — that file was already removed by an earlier, unrelated "remove +deprecated files" commit, predating this ticket. Full detail: [ticket +12](tickets/12-retire-r-workspace.md). + +**Twenty tickets resolved.** [Triage the patient cleaned-stage column +mismatches](tickets/28-triage-patient-cleaned.md) is done for three of its +five dominant columns: `t1d_diagnosis_age` (25,968 mismatches, the single +largest column on the whole cleaned-stage report) was a real Python bug — +`_fix_t1d_diagnosis_age` unconditionally recomputed the value from +`dob`/`t1d_diagnosis_date`, discarding a real recorded age whenever a date +failed to parse, on the mistaken belief (per its own docstring) that this +"matches R" — R's equivalent function is dead code, never called from +`script2_process_patient_data.R`, so R always just keeps the raw recorded +value. Fixed to prefer the raw recorded value whenever present and not the +Excel error sentinel, falling back to date-based calculation only when +genuinely missing; verified with a real 248-tracker pipeline re-run +(`a4d run patient --force`) and comparison re-run, dropping the column's +mismatches 25,968 -> 4,807 (81.5%). `recruitment_date` (28,512) and +`insulin_subtype` (15,724) turned out to be genuine, already-correct Python +divergences rather than bugs: `recruitment_date`'s R-null/Python-has-value +pattern was confirmed directly against the real source Excel (Quirino +Memorial Medical Center, patient PH_QM001's "Date of Recruitment" in its +Patient List sheet) as an R extraction gap; `insulin_subtype`'s was already +documented in code as a deliberate correction of an R validator bug and +typo. Both got new named classifiers in `src/a4d/migration/compare.py` +(`r_extraction_gap`, `r_validator_rejects_multivalue`) explaining 98%+ of +each column's mismatches — no pipeline behavior change, compare-tool-only. +Total cleaned-stage mismatches dropped 120,639 -> 99,478 (17.5%, entirely +from the `t1d_diagnosis_age` fix). `insulin_total_units` (16,985) and +`fbg_baseline_mg` (9,041) show the same R-null/Python-has-value shape but +weren't verified against source or traced to a specific conversion-step +cause; left unclassified. The other 56 columns were never sampled. Split +into [ticket 29](tickets/29-triage-patient-cleaned-residual.md) rather than +forcing convergence, per this ticket's own pre-authorization. Full test +suite (585 tests), ruff, `ty check src/` all pass. Full detail: [ticket +28](tickets/28-triage-patient-cleaned.md). + +**The frontier is now [Build a drill-down log analyzer for admins to +inspect a specific tracker file's errors/logs](tickets/16-log-analyzer-drill-down.md), +[Triage the residual product_units_received/product_units_released/product_received_from +raw-stage mismatches](tickets/24-triage-remaining-raw-column-residual.md), +[Triage the product_units_released cleaned-stage column +mismatches](tickets/25-triage-product-units-released-cleaned.md), [Triage +the product pipeline's column-existence and dtype +divergence](tickets/26-triage-product-column-divergence.md), [Triage the +residual patient raw-stage column mismatches after date +normalization](tickets/27-triage-patient-raw-residual.md), and [Triage the +residual patient cleaned-stage column mismatches](tickets/29-triage-patient-cleaned-residual.md) — +six tickets. Ticket 12 has moved from frontier back to blocked (see +correction above) — every remaining frontier ticket is an independent +triage residual or a separate feature (ticket 16), takeable in any order, +and all of them now sit between the map and re-unblocking ticket 12.** + +**Twenty-one tickets resolved.** [Triage the product pipeline's +column-existence and dtype divergence](tickets/26-triage-product-column-divergence.md) +is done: added a `column_divergence` sheet to every stage's Excel report +(`build_mismatch_rows()`, `src/a4d/migration/compare.py`), closing the +"CLI-only, no report sheet" gap the ticket opened with, and root-caused +every divergence named in its premise plus patient's (previously +unchecked), verified against the real 248-tracker drive data and, for the +raw-stage gap, real source Excel. `product_table_month`/`product_table_year`/ +`product_unit_capacity` (R `Float64` vs Python `Int32`, all 229 cleaned +files) and `product_remarks` (R `Boolean` vs Python `String`, 103/229) are +confirmed harmless representation artifacts (0 non-integer values checked; +an all-null-column parquet-writer inference quirk, respectively) — +`compare_columns()` itself was deliberately left unfiltered (an existing +test pins "flag every dtype difference" as policy), so these are documented +rather than normalized away. `orig_product_released_to` is confirmed an +expected Python-only helper column. `product_returned_by` (51 files) / +`product_units_returned` (25 files), only-in-Python at the raw stage, is a +**genuine R extraction gap** — spot-checked directly against +`2017_Mahosot Hospital A4D Tracker.xlsx`'s `INV` sheet, whose header row +literally reads "Units Returned"; Python's raw extraction holds 11 real +non-null values for it that R's raw output drops entirely for that file. +Patient's cleaned stage is essentially clean (one single-file dtype +mismatch). Patient's raw stage did not converge — 18,783 `column_divergence` +rows across 245 files, dominated by hundreds of uniquely-numbered +only-in-R junk columns (`na`, `na1`, ... `na10064`) and a large +only-in-Python set of unmapped literal source header text — split into +[ticket 30](tickets/30-triage-patient-raw-column-divergence.md). Full suite +(584 passed, 1 skipped, incl. 1 new test), ruff, `ty check src/` all pass. +Full detail: [ticket 26](tickets/26-triage-product-column-divergence.md). + +**The frontier is now [Build a drill-down log analyzer for admins to +inspect a specific tracker file's errors/logs](tickets/16-log-analyzer-drill-down.md), +[Triage the residual product_units_received/product_units_released/product_received_from +raw-stage mismatches](tickets/24-triage-remaining-raw-column-residual.md), +[Triage the product_units_released cleaned-stage column +mismatches](tickets/25-triage-product-units-released-cleaned.md), [Triage +the residual patient raw-stage column mismatches after date +normalization](tickets/27-triage-patient-raw-residual.md), [Triage the +residual patient cleaned-stage column mismatches](tickets/29-triage-patient-cleaned-residual.md), +and [Triage the patient pipeline's raw-stage column-existence +divergence](tickets/30-triage-patient-raw-column-divergence.md) — six +tickets, ticket 26 replaced by its spawned residual ticket 30. Ticket 12 +remains blocked, now on `[20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]`.** + +**Twenty-third ticket resolved.** [Triage the residual patient raw-stage +column mismatches after date normalization](tickets/27-triage-patient-raw-residual.md) +is done for its two original leads: patient's raw stage got the same +`normalize_numeric_column` treatment ticket 22 gave product's (derived from +the schema rather than hand-listed), plus `meter_received_date` hand-added +to the date-normalize list (it has no cleaned-stage counterpart to derive +from). Two new classifiers were added and verified against the real drive +data — `r_formula_error` (bmi/t1d_diagnosis_age are formula-derived in the +source trackers; a source formula error leaves R's raw extraction holding +the literal Excel error string while Python correctly has no cached value) +and `buddhist_era_typo` (a newly-found pattern: a clinician typed a Thai +Buddhist-Era year straight into a Gregorian date cell — verified directly +against the real source Excel for one patient, and confirmed **not** a +pipeline bug, since the cleaned stage's existing future-date guard already +replaces the same cell with R's own sentinel). Raw-stage patient mismatches +dropped 46,788 -> 18,813 (59.8%). `complication_screening` (12,566, 84% of +what remains, already carrying a probable multi-select-extraction lead) +and roughly 50 smaller columns were left unchased rather than sprawling +past this ticket's own scope — split into [ticket +31](tickets/31-triage-patient-raw-residual-2.md), which now replaces +ticket 27 as one of ticket 12's blockers. Full suite (599 passed, 1 +skipped), ruff, `ty check src/` all pass. Full detail: [ticket +27](tickets/27-triage-patient-raw-residual.md). + +**The frontier is now [Build a drill-down log analyzer for admins to +inspect a specific tracker file's errors/logs](tickets/16-log-analyzer-drill-down.md), +[Triage the residual product_units_received/product_units_released/product_received_from +raw-stage mismatches](tickets/24-triage-remaining-raw-column-residual.md), +[Triage the product_units_released cleaned-stage column +mismatches](tickets/25-triage-product-units-released-cleaned.md), [Triage +the residual patient cleaned-stage column +mismatches](tickets/29-triage-patient-cleaned-residual.md), [Triage the +patient pipeline's raw-stage column-existence +divergence](tickets/30-triage-patient-raw-column-divergence.md), and +[Triage the residual patient raw-stage column mismatches (round +2)](tickets/31-triage-patient-raw-residual-2.md) — six tickets, ticket 27 +replaced by its spawned residual ticket 31. Ticket 12 remains blocked, now +on `[20, 21, 22, 23, 24, 25, 26, 28, 29, 30, 31]`.** + +**Twenty-two tickets resolved.** [Triage the residual +product_units_received/product_units_released/product_received_from +raw-stage mismatches](tickets/24-triage-remaining-raw-column-residual.md) +is done: all 105 residual mismatches are now fully explained, none +requiring a pipeline code change. `product_units_received` (75) and 5 of +`product_received_from`'s 6 are a newly-identified pattern — R's readxl +infers a column's type from its majority values, so a lone Excel +date/time-formatted cell in an otherwise-numeric column (a genuine source +data-entry anomaly, verified against the real source Excel, e.g. Penang +General Hospital 2019 Apr19!E36) gets coerced to the column's numeric +type on R's side while Python's openpyxl honors the individual cell's own +format — a new `STRAY_DATE_CLASSIFIERS` classifier in +`src/a4d/migration/compare.py`, using openpyxl's own `from_excel` (not +hand-rolled epoch math) to also replicate the Excel 1900-leap-year serial +bug. 20 rows (`product_units_released` 19, `product_received_from` 1) were +plain float-precision representation differences, fixed by extending the +existing `normalize_numeric_column` wiring (ticket 22's precedent) to all +three columns. The remaining 5 `product_units_released` rows (2017-2019 +Mandalay files) are a wide-format comma/hyphen-split ambiguity on messy +human-entered source notes — verified R's value is a strict text prefix of +Python's in every case, i.e. Python is the more faithful extraction, not a +bug to fix toward R's more-truncated answer; a new +`WIDE_FORMAT_FRAGMENT_CLASSIFIERS` classifier documents it. The +date/time-coercion pattern is flagged as possibly systemic per the +ticket's own instruction — rather than spawning a new ticket, the lead and +the reusable classifier were noted as an addendum on the still-open +[ticket 27](tickets/27-triage-patient-raw-residual.md), which already +covers patient raw-stage residual triage. Full suite (592 passed, 1 +skipped), ruff, `ruff format --check`, `ty check src/` all pass; verified +end-to-end against the real 248-tracker drive data (`Product (cleaned)`, +`Patient (raw)`, and `Patient (cleaned)` reports confirmed unchanged by +this session's fix). Full detail: [ticket +24](tickets/24-triage-remaining-raw-column-residual.md). + +**Twenty-four tickets resolved.** [Triage the product_units_released +cleaned-stage column mismatches](tickets/25-triage-product-units-released-cleaned.md) +is done — and with it, **every product-arm column that had a ticket is now +explained and decided**. All 2,144 mismatches are the same row-order +divergence ticket 21 root-caused for this column's five siblings; the only +defect was in the comparison tool's column-to-registry map, where this +column carried the raw-stage-only wide-format classifier alone and so +reported every cleaned-stage row as `unclassified`. Python is confirmed +correct, R is not: R's output has `product_entry_date` null on every row of +the worst-affected group while the source Excel's "Entry Date" column is +fully populated, so R falls back to input order while Python sorts +chronologically on dates that really are in the file. Verified beyond the +classifier's own (loose) heuristic by two exhaustive checks against the real +248-tracker drive pair — per-group value multisets identical across all +2,283 groups, and `(units_released, released_to)` pair multisets identical +across all 11,649 groups, i.e. nothing added, lost, altered or re-partnered. +No pipeline change. Full detail: [ticket +25](tickets/25-triage-product-units-released-cleaned.md). + +**Gap found and ticketed: the product cleaned stage still carries 2,488 +`unclassified` rows across five columns that no ticket owned** — +`product_balance` (1,976), `product_sheet_name` (275), `product_entry_date` +(169), `file_name` (60), `product_units_received` (8). Each triage ticket +worked only the columns its own title named, and these fell between them. +Spawned as [ticket 36](tickets/36-triage-product-cleaned-unclassified-residual.md), +which inherits a concrete mechanism for the largest of them: `product_balance` +is *derived* on both sides (`_compute_running_balance`, step 2.15), so it +cannot travel with a row under a legitimate re-sort and the row-order +classifier's value-membership test cannot detect it in principle — which is +why it sat at 27.9% detection as ticket 21's "future work". + +**The frontier is now [Build a drill-down log analyzer for admins to inspect +a specific tracker file's errors/logs](tickets/16-log-analyzer-drill-down.md), +[Triage the residual patient cleaned-stage column +mismatches](tickets/29-triage-patient-cleaned-residual.md), [Triage the +patient pipeline's raw-stage column-existence +divergence](tickets/30-triage-patient-raw-column-divergence.md), [Triage the +residual patient raw-stage column mismatches (round +2)](tickets/31-triage-patient-raw-residual-2.md), [Re-audit every existing +cause classifier](tickets/32-audit-classifiers-against-decision-bar.md), +[Make the local pre-push check set actually match +CI](tickets/34-local-ci-parity-guard.md), [Resolve the Polars 2.0 +deprecation warnings](tickets/35-polars-2-deprecation-warnings.md), and +[Triage the product cleaned-stage mismatches no ticket +owns](tickets/36-triage-product-cleaned-unclassified-residual.md) — eight +tickets. The remaining triage work is now **patient-arm plus one product +residual**: 29, 30, 31 and 36 are ticket 12's direct blockers +(`blocked_by: [20, 21, 22, 23, 24, 25, 26, 28, 29, 30, 31, 36]`, all but +those four closed). Tickets 16, 32, 34 and 35 are independently takeable and +not wired as blockers.** + +**A pipeline change landed outside a ticket closure** (same session, after +ticket 25 closed, at the user's direction): the user asked whether +`_compute_running_balance` is itself wrong. It is not — it mirrors R's +`compute_balance` exactly, and both pipelines discard the tracker's recorded +per-row balance by design. What differs is only the order the ledger +accumulates in, downstream of step 2.7's chronological sort. Measured across +all 11,649 product groups: balances are wholly identical in 95.4%, closing +stock identical in 98.6%, and of the 158 differing endpoints 153 are float +noise where **Python is the cleaner side** (it rounds, R does not) and 5 are +**R corrupted by an Excel date serial** (2019 Penang General Hospital closes +at 43,572 in R vs 6.0 in Python). Python's closing stock is correct in every +group. The user decided to keep the current behaviour and add the missing +signal: `_compute_running_balance` now reports, per (sheet, product) group, +when the computed closing balance contradicts the tracker's own recorded +total — a new `balance_reconciliation` error code, firing on 113 groups +across 21 files (1.1%), with all 248 cleaned outputs verified byte-identical +so no production data moved. The granularity was picked by measurement (a +per-row check would have fired 15,301 times). Full detail: [ticket 36's +addendum](tickets/36-triage-product-cleaned-unclassified-residual.md#addendum-session-2026-08-12h-the-balance-mechanism-measured). + +**Twenty-five tickets resolved.** [Triage the product cleaned-stage +mismatches no ticket owns](tickets/36-triage-product-cleaned-unclassified-residual.md) +is done — see the Decisions-so-far entry below for the full account. **The +product arm's cleaned-stage triage is now complete**: 2,488 `unclassified` +rows down to 20, every one explained, and the 20 kept unclassified on +purpose as signals (11 = "the two ledgers disagree on closing stock", 9 = +positional-alignment residue). The session also landed a real pipeline +change — consistent whitespace trimming across both arms and the metadata +table — which recovered 72 rows of patient `sex` data both pipelines had +been discarding. + +**The frontier is now [Build a drill-down log analyzer for admins to inspect +a specific tracker file's errors/logs](tickets/16-log-analyzer-drill-down.md), +[Triage the residual patient cleaned-stage column +mismatches](tickets/29-triage-patient-cleaned-residual.md), [Triage the +patient pipeline's raw-stage column-existence +divergence](tickets/30-triage-patient-raw-column-divergence.md), [Triage the +residual patient raw-stage column mismatches (round +2)](tickets/31-triage-patient-raw-residual-2.md), [Re-audit every existing +cause classifier](tickets/32-audit-classifiers-against-decision-bar.md), +[Make the local pre-push check set actually match +CI](tickets/34-local-ci-parity-guard.md), and [Resolve the Polars 2.0 +deprecation warnings](tickets/35-polars-2-deprecation-warnings.md) — seven +tickets, down from eight. **All remaining triage work is patient-arm.** +Tickets 29, 30 and 31 are now ticket 12's only remaining triage blockers +(`blocked_by: [20, 21, 22, 23, 24, 25, 26, 28, 29, 30, 31]`, all but those +three closed); tickets 16, 32, 34 and 35 are independently takeable. + +**Note for the next session on ticket 32** (the classifier re-audit): it now +also inherits `derived_running_total_row_order`, +`python_future_date_sentinel`, `summary_residue_nulled`, `stray_date_zeroed` +and `r_validator_rejects_untrimmed`, all added this session. Each was +source-verified when written, and the map's Notes now record that a corrupt +source is itself a valid terminal answer — so ticket 32's bar should be read +in that light rather than as "every classifier must name a winner". + +**Twenty-six tickets resolved.** [Triage the residual patient cleaned-stage +column mismatches](tickets/29-triage-patient-cleaned-residual.md) is done for +the two priority columns its question named, plus the 999999-sentinel family +that ran through the whole report and two real Python bugs found while +sampling the tail. `insulin_total_units` (16,985) and the baseline-FBG pair +(10,197) are both **R defects, not Python divergences**: R's insulin-column +dedup grep deletes "TOTAL Insulin Units" from every 2024+ tracker before it +is read (0 non-null on all 81,859 R rows), and R's Patient List join renames +*both* sides on a name collision so its cleaning stage finds no unsuffixed +`fbg_baseline_mg` at all. R's 999999 sentinel (8,085 rows) is a deliberate +and correct Python representation choice -- all 8,085 were traced back to +Python's own raw stage and **not one** had a real number behind it. Two live +production bugs were fixed: `extract_regimen` lowercased every value its four +patterns did not match ("NPH" -> "nph"), and `validate_allowed_values` picked +the last of two config spellings that sanitize identically where R picks the +first. Verified by a full 248-tracker `a4d run patient --force` re-run and a +fresh comparison: cleaned-stage mismatches 99,408 -> 95,490, unclassified +55,670 -> **16,698 (-70%)**, `insulin_regimen` 1,348 -> 41, `status` 2,661 -> +50. The residual split into [ticket +37](tickets/37-triage-patient-cleaned-residual-2.md). Full detail: [ticket +29](tickets/29-triage-patient-cleaned-residual.md). + +**The canonical-label question was raised and settled in the same session, +reversing the recommendation it was raised with.** Checking the source rather +than the two pipelines showed the two spellings split by tracker generation: +2020-2023 trackers write "Active - Remote", 2024+ trackers write +"Active Remote", and the `Lookup List` dropdown the 2024 template introduced +defines only the latter (2022/2023 trackers have no such sheet at all). The +user decided: **one canonical label per status, aliases folded into it, and +the canonical form declared in config rather than implied by list order.** +`reference_data/validation_rules.yaml` (Python-only -- R reads +`data_cleaning.yaml`) now carries a `canonical -> [retired spellings]` alias +map, and `validate_allowed_values` raises on a silent collision instead of +resolving it. Production now emits one label (`Active Remote`, 2,776 rows); +the resulting 2,611-row divergence from R is deliberate and classified as +`python_canonical_label`, derived from that same config. + +**The frontier is now [Build a drill-down log analyzer for admins to inspect +a specific tracker file's errors/logs](tickets/16-log-analyzer-drill-down.md), +[Triage the patient pipeline's raw-stage column-existence +divergence](tickets/30-triage-patient-raw-column-divergence.md), [Triage the +residual patient raw-stage column mismatches (round +2)](tickets/31-triage-patient-raw-residual-2.md), [Re-audit every existing +cause classifier](tickets/32-audit-classifiers-against-decision-bar.md), +[Make the local pre-push check set actually match +CI](tickets/34-local-ci-parity-guard.md), [Resolve the Polars 2.0 +deprecation warnings](tickets/35-polars-2-deprecation-warnings.md), and +[Triage the residual patient cleaned-stage mismatches (round +2)](tickets/37-triage-patient-cleaned-residual-2.md) -- seven tickets, ticket +29 replaced by its spawned residual ticket 37. All remaining triage work is +still patient-arm; tickets 30, 31 and 37 are ticket 12's only remaining +triage blockers (`blocked_by: [20, 21, 22, 23, 24, 25, 26, 28, 30, 31, 37]`, +all but those three closed).** + +**Note for ticket 32** (the classifier re-audit): it now also inherits +`r_insulin_dedup_drop`, `r_join_suffix_collision` and +`r_numeric_error_sentinel`. All three were verified against the real drive +data rather than sampled, and the sentinel one exhaustively. + +**Twenty-seven tickets resolved.** [Triage the residual patient cleaned-stage +mismatches (round 2)](tickets/37-triage-patient-cleaned-residual-2.md) is +done for four of its five named leads -- and it was overwhelmingly a +bug-fixing session, not a labelling one. **Four real Python bugs** were found +and fixed, three of them silent data loss and one of them +non-determinism: `merge_headers` produced the unmappable header +`Updated 2022 Date` for every 2022 tracker (that template writes +`Updated 2022` in the upper header row instead of repeating the subject; R +carries an explicit fixup for it), losing **4,442 blood-pressure and 2,723 +education update dates**; `_apply_type_conversions` stripped a trailing time +component by splitting on the first space, so `"Jun 2006"` became `"Jun"`, +which dateutil completed **from today's date** -- destroying real diagnosis +dates and making the output depend on the day the pipeline ran; +`parse_date_flexible`'s month-name truncation deleted only the *fourth* +letter, so `"March"` became `"Marh"` and every full month name in the +trackers was unparseable and sentinelled (its own docstring examples never +worked); and its month-year branch accepted only 2-digit years. A +longest-parseable-prefix fallback was added to replace the free-text +truncation the space-split had been doing by accident +(`"16-Nov-2019 due to DKA"`). Three causes were named besides +(`r_extraction_gap` extended to two more columns over two separately-verified +R mechanisms, the new `r_ifelse_na_propagation` for R's three-valued-logic +NA propagation on `insulin_type`, and `buddhist_era_typo` made symmetric). +Unclassified cleaned-stage mismatches **16,698 -> 8,259 (-51%)**. The +date-column family did not converge and split into [ticket +38](tickets/38-triage-patient-cleaned-date-family.md). Full detail: [ticket +37](tickets/37-triage-patient-cleaned-residual-2.md). + +**The tracker set was refreshed to 254 files in the same session**, at the +user's direction, after the data analyst renamed every `06 ...` tracker to a +`2026_...` form and added new clinics. This is a change to the map's own +measuring instrument, so it is recorded here rather than only on the ticket: +the 41 stale `06 ...` local copies were **moved** (not deleted) to +`stale_06_trackers_2026-08-14/` on the drive -- left in place they would have +been processed alongside their renamed twins and double-counted every 2026 +patient; the frozen R output's 322 per-tracker files were renamed to match, +the mapping derived from the clinic folder rather than guessed, verified 1:1 +and saved at `output_r_rename_map_2026-08-14.json` so it is reversible; and +R's `file_name` column -- the only column carrying the old stem -- was +rewritten across 160 parquets / 40,701 rows, since renaming the files alone +left 33k phantom mismatches by construction. R was **not** re-run: the new +trackers have no R counterpart and show as Python-only, which the user +accepted. Post-refresh state, all four stages: patient cleaned 93,997 +mismatches / 8,259 unclassified; patient raw 27,921 / 14,844; product cleaned +22,718 / 20; product raw 118 / 0. **Every earlier count on this map was +measured against the 248-file set and should be read as historical.** + +**The frontier is now [Build a drill-down log analyzer for admins to inspect +a specific tracker file's errors/logs](tickets/16-log-analyzer-drill-down.md), +[Triage the patient pipeline's raw-stage column-existence +divergence](tickets/30-triage-patient-raw-column-divergence.md), [Triage the +residual patient raw-stage column mismatches (round +2)](tickets/31-triage-patient-raw-residual-2.md), [Re-audit every existing +cause classifier](tickets/32-audit-classifiers-against-decision-bar.md), +[Make the local pre-push check set actually match +CI](tickets/34-local-ci-parity-guard.md), [Resolve the Polars 2.0 +deprecation warnings](tickets/35-polars-2-deprecation-warnings.md), and +[Triage the patient cleaned-stage date-column family (round +3)](tickets/38-triage-patient-cleaned-date-family.md) -- seven tickets, +ticket 37 replaced by its spawned residual ticket 38. Tickets 30, 31 and 38 +are ticket 12's only remaining triage blockers (`blocked_by: [20, 21, 22, 23, +24, 25, 26, 28, 30, 31, 38]`, all but those three closed); tickets 16, 32, 34 +and 35 are independently takeable. Tickets 30 and 31 both had their premise +numbers refreshed against the new 254-file baseline -- neither needed +re-scoping.** + + +**Twenty-eight tickets resolved.** [Triage the patient cleaned-stage +date-column family (round 3)](tickets/38-triage-patient-cleaned-date-family.md) +is done, and it was a bug-fixing session rather than a labelling one. The +date path's missing-value handling had drifted from the numeric path's -- +`parse_date_flexible` knew four markers, `safe_convert_column` eleven -- so +every date cell recording an absence as `-`, `.`, `N/A`, `Nil`, `Unknown`, `?` +or the tracker template's own leftover placeholder text was stamped +9999-09-09 in production. The list is now declared once and shared, with two +date-scoped sets beside it; sentinel-stamped date cells fell 6,186 -> 1,994 +(-68%), the unrecognized-marker share of those 4,377 -> 0. R's side of the +same shape became the `r_date_error_sentinel` cause (5,707 rows), wired across +every column `get_date_columns()` returns, with Python verified as the correct +side against the source template's own "(Insert Date or NA)" sub-header. +Cleaned-stage unclassified 8,259 -> 6,652. **The ticket's own suggested +shortcut was rejected on measurement**: wiring `python_future_date_sentinel` +across the date columns would have labelled 1,209 cells as the future-date +guard when only 722 are -- the other 487 are Python failing to parse what R +parsed, which became [ticket +39](tickets/39-recover-dates-embedded-in-free-text.md). Two questions were +also closed by measurement: patient's row-alignment key stays an identity key +(0.10% of rows sit on a duplicated key), which settles ticket 31's copy of it; +and `t1d_diagnosis_date`'s largest population is a corrupt source file, not a +pipeline divergence. + +**The frontier is now [Build a drill-down log analyzer for admins to inspect a +specific tracker file's errors/logs](tickets/16-log-analyzer-drill-down.md), +[Triage the patient pipeline's raw-stage column-existence +divergence](tickets/30-triage-patient-raw-column-divergence.md), [Triage the +residual patient raw-stage column mismatches (round +2)](tickets/31-triage-patient-raw-residual-2.md), [Re-audit every existing +cause classifier](tickets/32-audit-classifiers-against-decision-bar.md), [Make +the local pre-push check set actually match CI](tickets/34-local-ci-parity-guard.md), +[Resolve the Polars 2.0 deprecation warnings](tickets/35-polars-2-deprecation-warnings.md), +and [Decide whether a date buried inside a clinical note should be recovered +or discarded](tickets/39-recover-dates-embedded-in-free-text.md) -- seven +tickets, ticket 38 replaced by its spawned residual ticket 39. **Tickets 30 +and 31 are now ticket 12's only remaining blockers** +(`blocked_by: [20, 21, 22, 23, 24, 25, 26, 28, 30, 31]`), and both are patient +raw-stage triage -- the cleaned stage of both arms is now fully owned. Ticket +39 is deliberately not wired as a blocker of ticket 12: it asks what Python +should do with a date buried in a clinical note, which needs no reference to +R's source. Every count on this map from before 2026-08-14 was measured +against an earlier tracker set or an earlier baseline run; the current +baseline is `output/comparison/2026-08-14T204031Z`. + +**Note for ticket 32** (the classifier re-audit): it now also inherits +`r_date_error_sentinel`, source-verified against the real 2020 Mahosot +workbook rather than sampled. + +**Session 2026-08-15, no ticket claimed — a living MR description, and four +real staleness findings.** The user asked for a merge-request description for +PR #2 and said it must be kept current after every session, so +[docs/archive/MR_DESCRIPTION.md](../archive/MR_DESCRIPTION.md) is now a +second session-end deliverable beside this map: treasure-map diagram, `a4d` +package overview + usage scenarios, the production Cloud Run Job setup, the +comparison harness and the triage loop it supports, the defects fixed, the +current per-stage counts, and the open-ticket list. Auditing deployment/CI for +drift while writing it turned up four things worth recording, all fixed +(`10bb0ae`): **CI never ran on PR #2** — the `pull_request` trigger listed +`[main, develop, migration]` where the default branch is `dev` and `develop` +does not exist, and its `paths:` filter covered only `src/`/`tests/` while ruff +runs repo-wide (the ticket-33 failure mode, now removed); **`just backup-bq` +snapshotted 4 of the 8 tables** `load_pipeline_tables` deletes and recreates, +so `clinic_data_static`, `logs`, `errors` and `tracker_metadata` had no +rollback point despite the recipe's own "keep in sync with `PARQUET_TO_TABLE`" +comment; **SETUP.md documented `A4D_DATA_ROOT=/tmp/data`** while the live job +sets only `A4D_MAX_WORKERS=8` and the container actually uses the image's +`/workspace/data` (verified against the deployed job); and stale +`a4d-python/`-era paths in SETUP.md/README.md. `PLAN.md` was deleted — it +planned the `a4d-python`-to-root move, long since done. Left for their owners: +`just hooks`/`hooks-run` are broken (no `.pre-commit-config.yaml`, though +`pre-commit` is a dev dependency) and `just ci` still differs from CI's marker +set and coverage gate — both [ticket 34](tickets/34-local-ci-parity-guard.md); +the `.github/workflows/*.bak` R workflows are +[ticket 12](tickets/12-retire-r-workspace.md)'s. + + +**Twenty-nine tickets resolved.** [Triage the patient pipeline's raw-stage +column-existence divergence](tickets/30-triage-patient-raw-column-divergence.md) +is done, and all 18,235 rows are accounted for: 17,132 are R's blank-header +artifact, 154 are name-representation only, 703 belong to columns that died out +before 2025, and the remaining 246 are R mapping gaps this map had already +decided plus five new 2026 fields. The session's real find was a **Python data +loss nobody had seen**: `filter_valid_columns` drops any column whose header +cell is blank, and in `2021_Kantha Bopha`'s `Mar21`/`Apr21` sheets the +insulin-regimen column has real data and an empty header -- 194 rows lost, R +losing them too (it keeps the values only as junk-named `na1`). A sweep of all +254 trackers found 26 files and 4,572 such values; most turned out to be the +2022 template's hidden merged-cell duplicate, which Python is right to drop +(measured: it never holds a value the visible column lacks, and where the two +differ the visible one is richer). The new `blank_header_with_data` error code +makes every site visible instead of silent, and `recover_blank_headers` gets +the data back wherever the workbook itself settles the answer -- a tracker's +month sheets share one layout, so a sheet that does label the column is +evidence. It recovers 17 sites across 5 trackers (including all 194 Kantha +Bopha rows) and leaves 201 reported. Full detail: [ticket +30](tickets/30-triage-patient-raw-column-divergence.md). + +**Two rejections in that session are worth not re-proposing, both settled by +measurement after being argued first.** The session initially rejected +cross-sheet recovery outright, on the claim it would import the 2022 +merged-cell duplicates -- wrong, since the rule only fires where a sibling +sheet names the position, and that column is blank in every sheet (316 of the +4,572 values have a donor, 4,256 have none). And **deriving one header set per +tracker and applying it to all sheets** -- the natural simplification, proposed +by the user -- is unsafe: of the 359 positions where a tracker's month sheets +disagree, 75 are pure renames the synonym file already absorbs, but **284 +across 37 trackers change the canonical column**. `2018_CDA`'s `Apr18` has no +`Insulin Regimen` column at all, shifting seven columns left for that month; +`2020_CDA` relabels `Baseline FBG` from `mmol/dL` to `mg/dL` in June over +unchanged values. Position -> meaning is not stable within a tracker, which is +why recovery requires unanimous donors and only ever fills a blank. + +**A new standing rule came out of that, set by the user: a tracker is one +workbook for one clinic-year and should not change shape mid-year; where it +does, flag it.** `find_layout_changes` now reports every position whose +canonical column is unstable across a tracker's month sheets, under the +`tracker_layout_changed` error code -- 37 trackers, 284 positions. It +immediately paid for itself: `2020_CDA`'s bad header edit means Jan-May's +baseline FBG is **dropped entirely**, because `Baseline FBG (mmol/dL)` has no +synonym entry. + +Following that thread produced a whole-set inventory of every FBG header +spelling (`output/fbg_header_inventory.xlsx`) and a question the pipeline +cannot answer alone, now [ticket +42](tickets/42-fbg-unit-headers-and-implausible-values.md): the malformed +`mmol/dL` spelling is genuinely mmol in two files and mg in a third, so it +cannot be blanket-aliased and being unmapped is the safe default rather than a +gap; a 2018 tracker ships a header that offers the reader a choice of units +(`Baseline FBG (mmol/L or mg/dL)`, 155 values dropped -- low priority under the +latest-template rule); and, the largest finding, **unit confusion runs in both +directions across the whole dataset**: 9,264 readings sit in mg/dL-labelled +columns but below 30 mg/dL, where a mmol/L number lands, against 122 in +mmol/L-labelled columns above 100 mmol/L +([assets/glucose_readings_by_unit.md](assets/glucose_readings_by_unit.md)). +Neither is a header defect, so no header fix reaches them. + +**The email to A4D's medical advisor went out 2026-08-15**, asking for a +credible range and what the pipeline should do outside it. Ticket 42 is open +but not workable until that reply lands -- it needs clinical judgement, not +more measurement. Two framings were tried and dropped on the way and should +not be retried: bucketing on an invented 35 mmol/L threshold, and comparing +mg-recording clinics against mmol-recording ones (different clinics, different +populations, no dual recording). The user is taking the clinical half to A4D's +medical advisor; nothing in the synonym file was changed meanwhile. + +**The user set a scoping rule this session that applies to the whole map: the +current tracker template is the golden rule.** A column that appears in one +year and is gone the next was very likely a test that did not survive; the +trackers are kept if the data is ever wanted, but triage effort goes to the +core fields that run through every year. This is what ruled 703 of ticket 30's +residual rows out of scope, and it should be applied to the remaining triage +rather than re-argued. + +**The user also asked for a deliverable the map did not have:** one Excel +listing every source-tracker defect with file, sheet, `patient_id`, row and the +exact finding, so the workbooks themselves can be corrected -- their stated +preference over inferring fixes in code. That is [ticket +40](tickets/40-source-defect-findings-report.md), which inherits a concrete +backlog of already-verified findings from tickets 24, 27, 30, 36 and 38, and +carries the open question of whether it and [ticket +16](tickets/16-log-analyzer-drill-down.md) are the same deliverable. The 2026 +template's five unmapped Patient List fields are [ticket +41](tickets/41-decide-2026-new-patient-list-columns.md), deliberately left for +the user, who wants to look at the new trackers before deciding. + +**[Triage the residual patient raw-stage column mismatches (round +2)](tickets/31-triage-patient-raw-residual-2.md) is now the only remaining +blocker of [Retire R from the workspace](tickets/12-retire-r-workspace.md)** -- +ticket 12's `blocked_by` list is otherwise fully closed. The frontier is ticket +31, [Build a drill-down log analyzer](tickets/16-log-analyzer-drill-down.md), +[Re-audit every existing cause +classifier](tickets/32-audit-classifiers-against-decision-bar.md), [Make the +local pre-push check set actually match CI](tickets/34-local-ci-parity-guard.md), +[Resolve the Polars 2.0 deprecation +warnings](tickets/35-polars-2-deprecation-warnings.md), [Decide whether a date +buried inside a clinical note should be recovered or +discarded](tickets/39-recover-dates-embedded-in-free-text.md), and the two new +tickets 40 and 41 -- eight tickets. Ticket 31 is the one on the critical path; +the rest are independently takeable. + +**Thirty tickets resolved.** [Triage the residual patient raw-stage column +mismatches (round 2)](tickets/31-triage-patient-raw-residual-2.md) is done, and +like ticket 30 before it, it was a bug-fixing session rather than a labelling +one. Its dominant column `complication_screening` (12,566 mismatches, 84% of +the stage's residual) had **two unrelated causes stacked on top of each other**, +and the ticket's own headline lead was neither. + +Underneath sat **a Python data loss nobody had seen**: +`ColumnMapper.rename_columns` kept only the *first* of several source columns +mapping to one canonical name, under a comment calling it "an edge case from +discontinued 2023 format". It is not an edge case -- the 2023 template splits +complication screening across B.P./Kidney/Eye/Foot/Lipids sub-columns that all +map to `complication_screening`, each independently populated. Measured by +instrumenting the real extraction over all 254 trackers: **2,489 recorded +values discarded across 27 trackers**. `KH_KB023` in `2023 Kantha Bopha`'s +`Jan'23` has `JAN` in the Kidney column and nothing else, and Python emitted +`null`. The fix merges the group instead -- comma-joining non-empty values, +which is what R's `tidyr::unite()` does and what the pipeline's own +`merge_duplicate_columns_data` already did for repeated *raw headers* -- and +the old misleading `invalid_tracker` warning becomes a +`duplicate_source_columns` one naming the group, so a new collision is visible +rather than silent (ticket 30's `blank_header_with_data` precedent). + +On top of it sat an R rendering artifact that *is* correctly a classifier: +R's `unite` leaves `na.rm` at `FALSE`, so an unscreened patient reads +`NA,NA,NA,NA,NA` and a January screening reads `JAN,NA,NA,NA,NA`. That is +`r_na_unite_padding`, source-verified against the real workbook and wired to +the only three canonical columns that form a duplicate group anywhere in the +254-tracker set -- derived by measurement, not hand-listed. **Patient +raw-stage unclassified fell 14,903 -> 1,879 (-87%)**; total mismatches barely +moved (28,092 -> 27,980), because the fix adds data on Python's side while the +artifact stays a mismatch until classified. The two halves must not be judged +by the same number. + +**Worth not re-proposing:** ticket 27's lead that this column was a Python +multi-select extraction gap (`2021_NPH`, Python holding a comma-joined list +where R holds only the first selection) is **2 rows out of 12,566**. And +`insulin_regimen` (235 residual rows) was deliberately *not* wired to +`r_extraction_gap` despite matching its shape: 194 rows are ticket 30's +source-verified blank-header recovery, but the other 41 are unverified, and a +whole-column classifier would have labelled them without deciding them. + +**Ticket 12 is still blocked, by one ticket.** The 1,879 remaining +unclassified rows across ~50 columns became [ticket +43](tickets/43-triage-patient-raw-residual-3.md), which replaces ticket 31 in +[Retire R from the workspace](tickets/12-retire-r-workspace.md)'s `blocked_by` +-- ticket 31 again had to read `r-archive/R/script1_read_patient_data.R` +directly to root-cause the divergence, so R's source is still a live reference +for the triage that remains. The current baseline run is +`output/comparison/2026-08-15T214447Z`; every count on this map measured +before it should be read as historical. + +**The frontier is [Build a drill-down log +analyzer](tickets/16-log-analyzer-drill-down.md), [Re-audit every existing +cause classifier](tickets/32-audit-classifiers-against-decision-bar.md), [Make +the local pre-push check set actually match +CI](tickets/34-local-ci-parity-guard.md), [Resolve the Polars 2.0 deprecation +warnings](tickets/35-polars-2-deprecation-warnings.md), [Decide whether a date +buried inside a clinical note should be recovered or +discarded](tickets/39-recover-dates-embedded-in-free-text.md), [Produce one +Excel of every source-tracker defect](tickets/40-source-defect-findings-report.md), +[Decide whether the 2026 template's five new Patient List fields enter the +pipeline](tickets/41-decide-2026-new-patient-list-columns.md), [Decide how FBG +unit headers are resolved](tickets/42-fbg-unit-headers-and-implausible-values.md), +and the new [round-3 patient raw +triage](tickets/43-triage-patient-raw-residual-3.md) -- nine tickets. Ticket 43 +is the one on the critical path, as ticket 12's only open blocker. Ticket 42 is +takeable but not workable: it waits on A4D's medical advisor's reply.** + +**Thirty-one tickets resolved.** [Decide how FBG unit headers are resolved, and +what to do about physiologically implausible mmol values](tickets/42-fbg-unit-headers-and-implausible-values.md) +is decided and implemented. **A4D's medical advisor replied 2026-08-17, and his +answer did not confirm the premise the ticket was built on.** The ticket asked +whether a reading below 30 mg/dL is impossible; he answered with *analytical* +limits (mg/dL ~2-5 to ~720-800, mmol/L ~0.1-0.3 to ~40-45), which make 5-30 +mg/dL possible but severe. So the headline "9,264 readings with the unit mixed +up" was never established. What he authorised is narrower: *"if it is clearly a +wrong unit mixed up, then correct it"* -- and defining "clearly" was the real +decision. + +It was settled by measurement, not a threshold. Across 372 file-column groups +the confusion is overwhelmingly **per column, and clusters by clinic**: 29 +groups have nine tenths or more of their readings in mmol territory under an +mg/dL header (`2025_Kantha Bopha II`: 99.8% of 951), while 300 groups are +clean. The user chose the split that follows: **correct at column level, flag +at row level**. `src/a4d/clean/glucose.py` moves a wholly mmol-recorded +column into its mmol sibling and rescales by 18 (29 columns, 21 files, reported +once per column because the defect is the header); a stray sub-30 reading in an +otherwise ordinary column is reported and **left exactly as recorded** (5,673 +rows, 34 files), because a genuine severe hypo and a mis-entered unit are +indistinguishable and converting one would hide it. Zero became null in both +units -- it is below both analytical floors, so it means "not measured". + +**The analytical limits are now enforced, and the top end had never been +looked at.** All four FBG columns are bounded at the permissive end of the +advisor's ranges, replacing an inherited 0-150 mmol/L bound from R that was +three times his ceiling and covering the three columns that had **no bound at +all**. 832 readings rejected -- including 383 above the mg/dL ceiling (max +2,013), which the original analysis never searched for because it only looked +downward. Every FBG value now sits inside the limits (mg max 800, mmol max +44.9). + +**The cost, recorded rather than hidden:** this is a deliberate divergence from +R, which has no unit resolution, so patient cleaned-stage mismatches rose +98,274 -> 114,509. The new `python_glucose_unit_corrected` classifier explains +11,189 of them on the three shapes the correction provably produces, but +deliberately never fires on an R-null cell, leaving 2,842 unclassified +(cleaned-stage unclassified 6,587 -> 9,427). 98.3% of those are measured to sit +in files whose column was swapped, so they are understood -- they are not +classified because the per-cell classifier has no file context and a blanket +rule would swallow ~193 unrelated cells. That is [ticket +44](tickets/44-triage-cleaned-fbg-r-null-residual.md), and it is the same +refusal ticket 31 made on `insulin_regimen`. + +Two of the ticket's own questions closed with it: `Baseline FBG (mmol/L or +mg/dL)` **stays unmapped** (2018 tracker; a template should never offer the +reader a choice of unit, so it is a source defect, not a synonym), and a +general per-column unit check was **not** built (the mechanism needs a column +pair in two units; HbA1c has none). + +**The frontier is [Build a drill-down log +analyzer](tickets/16-log-analyzer-drill-down.md), [Re-audit every existing cause +classifier](tickets/32-audit-classifiers-against-decision-bar.md), [Make the +local pre-push check set actually match CI](tickets/34-local-ci-parity-guard.md), +[Resolve the Polars 2.0 deprecation +warnings](tickets/35-polars-2-deprecation-warnings.md), [Decide whether a date +buried inside a clinical note should be recovered or +discarded](tickets/39-recover-dates-embedded-in-free-text.md), [Produce one +Excel of every source-tracker +defect](tickets/40-source-defect-findings-report.md), [Decide whether the 2026 +template's five new Patient List +fields enter the pipeline](tickets/41-decide-2026-new-patient-list-columns.md), +[round-3 patient raw triage](tickets/43-triage-patient-raw-residual-3.md), and +the new [cleaned-stage FBG +residual](tickets/44-triage-cleaned-fbg-r-null-residual.md) -- nine tickets. +Ticket 43 remains the one on the critical path, as [ticket 12](tickets/12-retire-r-workspace.md)'s +only open blocker. Ticket 40 now inherits a large, concrete backlog from this +session: 29 mislabelled columns and 5,673 suspect readings, each already +carrying the file, patient and value needed to correct the workbook -- and the +advisor's own closing instruction was to advise the data-entry staff.** + +**Thirty-two tickets resolved.** [Triage the residual patient raw-stage column +mismatches (round 3)](tickets/43-triage-patient-raw-residual-3.md) is done for +the two causes it settled, and like tickets 27, 30 and 31 before it, one of +them was a Python bug rather than a label. Patient raw-stage unclassified fell +**1,879 -> 1,409**; the new baseline run is +`output/comparison/2026-08-17T192144Z` and every raw-stage count on this map +measured before it should be read as historical. + +The larger half by count -- 434 rows across `fbg_baseline_mg.static`, +`fbg_baseline_mmol.static` and two `complication_screening_*_value` columns -- +was never a divergence at all: R writes `9.3000000000000007` where Python +writes `9.3`, and `normalize_numeric_column` has handled exactly that since +ticket 22. It never reached these four because the list of columns to +normalize was derived from the **cleaned** schema, which has no entry for the +Patient List join's `.static` copies and types a screening measurement as a +string because the same column can also hold "normal". The derivation was +pointed at the right place instead (`numeric_normalize_targets`, every raw +column bar the join keys), safe because non-numeric text passes through +untouched -- and measured before widening: of all 27,980 raw mismatches +exactly 434 had both sides parsing to the same float, all in those four +columns. + +**The Python defect was invented data, not lost data** -- the opposite of what +the ticket's evidence first suggested. `read_patient_rows` accepted a row if +*either* the row number or the patient_id was present, so the bare list of +patient IDs sitting below `2024_Vietnam National Children`'s `Jul24` data +block -- two non-empty cells per row, the ID repeated, nothing else across 40 +columns -- became 24 monthly records, which then picked up real-looking +demographics from the Patient List join. R never sees them: it bounds a +sheet's data block by the row-number column alone. **Adopting R's rule was +rejected on measurement**: a sweep of all 254 trackers found only 41 rows +Python takes past R's block, and one of them (`2024_Mahosot` `Jun24` +`LA-MH088`, 24 populated cells) is a complete patient record R loses. So the +rule now keeps an unnumbered row only when it carries a value beyond a repeat +of its own identifier -- deliberately not a cell-count threshold, which would +be a guess about layout. That file's unpaired Python rows went 24 -> 0. + +**A third finding was verified but deliberately left unlabelled.** All 86 +`hba1c_updated`/`fbg_updated_mg` mismatches in `2017_Yangon` are R holding +`8.8(20.9.16)` against Python's `8.8 (20.9.16)`; the source cell was read +directly and **contains the space**, so Python reproduces the workbook and R +drops it. R's mechanism was searched for and not found, so no classifier was +written -- naming a cause here would cement a guess, and the map's bar treats +an undecidable cause as an open question. It carries forward with the tail. + +**Ticket 12 is still blocked, by two tickets.** The residual split into [Give +the patient comparison an ordinal row +key](tickets/45-patient-row-alignment-duplicate-keys.md) -- 808 of the 1,409, +57%, are join fan-out in three files where `(patient_id, sheet_name)` is not +unique, which is ticket 17's product problem repeated on the patient arm -- +and [round-4 patient raw triage](tickets/46-triage-patient-raw-residual-4.md) +for the ~600 outside those files, blocked on 45 so the tail is triaged against +realigned rows rather than noise. [Retire R from the +workspace](tickets/12-retire-r-workspace.md)'s `blocked_by` swaps `43` for +`45, 46`, the same precedent applied when tickets 27 and 31 closed: ticket 43 +again had to read `script1_helper_read_patient_data.R` to establish how R +bounds a data block, so R's source is still a live reference. + +**Thirty-three tickets resolved.** [Give the patient comparison an ordinal row +key](tickets/45-patient-row-alignment-duplicate-keys.md) is done, and its open +question -- how widely to adopt positional alignment -- resolved to **nowhere**. +Product needed a positional key because it had no usable natural key; patient's +`(patient_id, sheet_name)` is unique in 249 of 254 files, so the same +`add_row_ordinal` mechanism is grouped by the *identity key itself* and only +breaks ties inside it. Identity is still checked everywhere, and in the files +whose key is unique every ordinal is 0, so nothing changes for them. The user +set the rule: `patient_id` is the main key -- if it does not match, something +is very wrong -- and `sheet_name` stays because monthly is the data's +granularity. + +Two things the ticket asserted turned out to need correcting, both caught by +measuring rather than trusting. **The three-file list is raw-only**: at the +cleaned stage there are five duplicate-key files, adding `2023_NPH` and +`2026_Surat Thani`, so the fix was applied to both patient stages rather than +raw alone. And **position is the wrong tie-break at the cleaned stage**: +brute-forcing every two-sided duplicate group showed occurrence order is +already optimal at raw (both pipelines emit source order) but pairs the wrong +copies in 9 cleaned groups, so duplicates are paired by minimum differing cells +instead -- with raw as the control proving that content matching reproduces +known-correct ordering exactly rather than merely minimising counts. Patient +raw unclassified fell **1,409 -> 601**, cleaned **9,392 -> 8,383**, with +`row_key_overlap` byte-identical before and after (no pairing gained or lost) +and product unchanged to the cell. Full detail: [ticket +45](tickets/45-patient-row-alignment-duplicate-keys.md). + +**A production bug surfaced that no comparison ticket could have found.** +Chasing the two extra cleaned-stage duplicate files found that in `2023_NPH`, +four distinct raw identities (`KH_NPH026`-`029`) arrive at the cleaned stage as +one `KH_NPH02`; across all 254 trackers, 4 files lose 9 identities this way, +with row counts preserved. **R does the same thing**, so it is not an R/Python +divergence at all -- agreement is precisely what the comparison tool is built +to stay quiet about. Spawned as [Four trackers where cleaning merges several +patients into one patient ID](tickets/47-patient-ids-merged-at-cleaning.md); it +needs no R, so it is not wired as a blocker of ticket 12. + +**Ticket 12 is now blocked by one ticket.** Its `blocked_by` drops `45`, +leaving [round-4 patient raw triage](tickets/46-triage-patient-raw-residual-4.md), +whose premise now names the exact 601-row residual and the new baseline run +`output/comparison/2026-08-17T202809Z`. + +**The frontier is [Build a drill-down log +analyzer](tickets/16-log-analyzer-drill-down.md), [Re-audit every existing +cause classifier](tickets/32-audit-classifiers-against-decision-bar.md), [Make +the local pre-push check set actually match +CI](tickets/34-local-ci-parity-guard.md), [Resolve the Polars 2.0 deprecation +warnings](tickets/35-polars-2-deprecation-warnings.md), [Decide whether a date +buried inside a clinical note should be recovered or +discarded](tickets/39-recover-dates-embedded-in-free-text.md), [Produce one +Excel of every source-tracker +defect](tickets/40-source-defect-findings-report.md), [Decide whether the 2026 +template's five new Patient List +fields enter the pipeline](tickets/41-decide-2026-new-patient-list-columns.md), +[cleaned-stage FBG residual](tickets/44-triage-cleaned-fbg-r-null-residual.md), +[round-4 patient raw triage](tickets/46-triage-patient-raw-residual-4.md), and +the new [merged patient IDs](tickets/47-patient-ids-merged-at-cleaning.md) -- +ten tickets. Ticket 46 is the one on the critical path, as ticket 12's last +remaining blocker; ticket 47 is the most serious in its own right, since a +merged identity attributes one patient's records to another in production +output. Ticket 40 gained three findings this session: the 23 patients listed +twice on one monthly sheet, and two trackers whose R output carries +`patient_id = "#REF!"`.** + +**Cleaned-stage counts recorded before this session no longer reproduce.** +Tickets 28, 29, 37 and 38 measured under the old key; their causes stand but +their numbers do not, and [ticket +44](tickets/44-triage-cleaned-fbg-r-null-residual.md) should re-measure against +the current baseline. + +**Thirty-four tickets resolved.** [Triage the residual patient raw-stage column +mismatches (round 4)](tickets/46-triage-patient-raw-residual-4.md) is done for +four of its causes, and like rounds 3, 2 and 1 before it, one of them was a +Python bug rather than a label. Patient raw-stage unclassified fell **601 -> +278 (-54%)**, cleaned **8,383 -> 8,148**; the new baseline run is +`output/comparison/2026-08-17T212151Z` and every count on this map measured +before it should be read as historical. + +**The bug was in production, not in the comparison.** A cell whose number +format is a date but whose content is a plain number -- a systolic 120 typed +into a `dd-mmm-yyyy`-formatted cell in `2025_Hat Yai`'s `Annual` sheet -- comes +back from openpyxl as `datetime(1900, 4, 29)`, because Excel's serial epoch is +1899-12-30 and 120 is simply the 120th day after it. R's readxl guesses a +column's type from its majority values and so reads the number, correctly, by +accident. Python's numeric conversion rejected the datetime and stamped 999999, +so **24 real blood-pressure readings were reaching BigQuery as the error +sentinel**. `read_patient_rows` now converts any pre-1903 datetime back to its +serial. The cut is a fact about the file format, not a threshold: a date +anyone could actually have typed stays a date, which is why +`2023_Chiang Mai`'s "Age at Diagnosis" cell (serial 20668, a 1956 date against +a 2009 D.O.B.) is left alone as a source defect rather than "recovered" into a +bogus age of 20668, which is exactly what R does with it. + +**The space ticket 43 could not explain is explained.** Its 86 rows of R +holding `8.8(20.9.16)` where Python and the source hold `8.8 (20.9.16)` were +about to become a classifier that recorded a verified fact with no mechanism. +The mechanism is in the workbook's own XML: where a cell carries mixed +formatting, xlsx stores it as a run sequence, and a space between two +differently-formatted fragments becomes a run holding nothing but that space. +openpyxl concatenates every run; readxl drops the whitespace-only one. Verified +in `xl/sharedStrings.xml` for two unrelated workbooks -- `2017_Yangon`'s glucose +column and `2022_Mahosot`'s free-text `observations` ("Unable to contact") -- +and the same text also exists as a plain shared string in the same column, +which is why only some cells diverge. That is `r_drops_richtext_space` (89 +rows); the two rows running the other way are `python_trims_merged_subvalue`, +ticket 31's merge trimming each fragment where `tidyr::unite` does not. + +**Worth not re-proposing:** a single "spaces-only difference" normalization +would have silenced both at once and hidden that they run in opposite +directions with different mechanisms. And `insulin_regimen` was finally wired +to `r_extraction_gap` (194 rows) only after re-reading `2021_Kantha Bopha`'s +`Mar21` column Q at source and confirming *both* header rows are empty -- +ticket 31's refusal was waiting for exactly that, and the ~40 unverified rows +it was protecting turned out to have been ticket 45's fan-out, now gone. + +**A second Python data loss was found and deliberately not fixed here.** In +`2021_Putrajaya`, the complication-screening block's row-1 header is merged +across columns 32-36, so Python maps neither `Results` nor `Date (mmm-yy)` and +files a screening selection under `observations`; R reads all three correctly. +~25 cells in one file, but the sweep it needs across all 254 trackers is a +session of its own -- [ticket +48](tickets/48-putrajaya-screening-columns-lost.md). + +**Also measured and left alone: product's own sub-1903 cells.** 18 +`product_units_received` and 2 `product_entry_date` cells carry the identical +shape (e.g. `1900-03-15`, i.e. 74 units) and currently land on +`openpyxl_date_typed_stray_cell`, whose verdict is "Python is faithful" -- +which this session's evidence contradicts for that sub-range. Product raw is +separate extraction code and already-closed triage scope, so it is [ticket +32](tickets/32-audit-classifiers-against-decision-bar.md)'s to re-audit. That +is the first concrete case ticket 32 was written for. + +**Thirty-five tickets resolved.** [Python drops complication-screening results +and dates where a merged header spans the +block](tickets/48-putrajaya-screening-columns-lost.md) is done, and like the +four triage rounds before it the ticket's one-file framing was too small: the +sweep found **693 sheets across 64 trackers** carrying a merged upper header. +Two independent mechanisms were damaging the same block -- forward-fill +resetting at a column blank in both header rows, and ticket 30's blank-header +recovery filing a screening selection under `observations` from a sibling sheet +laid out differently -- so a synonym entry could never have fixed it. + +**The sweep is what shaped the fix, twice.** Naive propagation of the merged +title looked obviously right and was measurably wrong: giving the bare title to +a column blank in both header rows would have put **290 sheets** into a state +where two columns compete for `complication_screening`, and would have +comma-joined **3,659** near-duplicate `Insulin Regimen` cells across 10 +trackers into single worse values -- on data where R and Python already agree. +The rule adopted instead is that a merged title *qualifies a sub-header* and +never names a column outright, which drops the collision count to 0 at no +measured cost. Verified against the real 248-tracker drive data: +`complication_screening_results` **11 -> 0**, `complication_screening_date` +**31 -> 0**, `observations` 8 unclassified -> **1**, patient raw unclassified +**278 -> 229**, cleaned **8,148 -> 8,141**, product byte-identical. The new +baseline run is `output/comparison/2026-08-17T222225Z`; counts measured before +it are historical. `2023_Chiang Mai`'s R output carries the same propagated +column names, confirming the fix reproduces R's own behaviour rather than +inventing one. + +**Ticket 12 is now blocked by one ticket.** Its `blocked_by` drops `48`, +leaving [round-5 patient raw triage](tickets/49-triage-patient-raw-residual-5.md), +whose premise and per-shape counts were refreshed against the new baseline -- +ticket 48 removed 49 of its 278, not the ~25 first estimated. + +**The frontier is [Build a drill-down log +analyzer](tickets/16-log-analyzer-drill-down.md), [Re-audit every existing +cause classifier](tickets/32-audit-classifiers-against-decision-bar.md), [Make +the local pre-push check set actually match +CI](tickets/34-local-ci-parity-guard.md), [Resolve the Polars 2.0 deprecation +warnings](tickets/35-polars-2-deprecation-warnings.md), [Decide whether a date +buried inside a clinical note should be recovered or +discarded](tickets/39-recover-dates-embedded-in-free-text.md), [Produce one +Excel of every source-tracker +defect](tickets/40-source-defect-findings-report.md), [Decide whether the 2026 +template's five new Patient List +fields enter the pipeline](tickets/41-decide-2026-new-patient-list-columns.md), +[cleaned-stage FBG residual](tickets/44-triage-cleaned-fbg-r-null-residual.md), +[merged patient IDs](tickets/47-patient-ids-merged-at-cleaning.md), and +[round-5 patient raw triage](tickets/49-triage-patient-raw-residual-5.md) -- +**ten tickets**, down from eleven. Ticket 49 is on the critical path as ticket +12's last remaining blocker; ticket 47 is the most serious in its own right, +since a merged identity attributes one patient's records to another in +production output. The rest are independent triage residuals and a separate +feature (ticket 16), takeable in any order.** + +**Thirty-six tickets resolved.** [Triage the residual patient raw-stage column +mismatches (round 5)](tickets/49-triage-patient-raw-residual-5.md) is done: +patient raw-stage unclassified went **229 -> 84** (63%), verified end-to-end +against the full 248-tracker drive data across two real comparison runs. This +round found **no Python defect** -- a first for the patient raw residual, and +the substantive result. Two of its four causes were the *comparison tool* +mis-measuring rather than the pipeline diverging: +`PATIENT_WHITESPACE_NORMALIZE_COLS` resolved through the *cleaned* schema's +`get_string_columns()`, so it never reached a raw-only column +(`dm_complications`) or one the cleaned schema types `Float64` +(`insulin_injections`, `hba1c_updated`) -- the identical defect ticket 43 fixed +for the numeric list, never applied to the whitespace one; and R's `FALSE` +against Python's `False` for a genuine Excel boolean, confirmed a pure +representation artifact by checking the cleaned stage, where both sides already +agree. `dm_complications` had been invisible for five rounds because DuckDB's +xlsx reader normalizes `\r\n` away, so the report showed R's +`Kidney \r\nDamage` and Python's `Kidney \nDamage` as byte-identical -- +**a shape that looks identical in the report is not evidence the values are +equal.** The other two causes are R limitations, each traced to a mechanism in +R's own source rather than wired on shape: `r_non_latin_header_miss` (44 rows) +-- `2022_Mukdahan` appends Thai translations to two headers, R's `sanitize_str` +strips `[^[:alnum:]]` whose class is Unicode-aware so the Thai survives, +`harmonize_patient_data_columns`' `match()` is exact, and R drops the whole +column; Python's `[^a-z0-9]` strips it and matches, recovering 44 real dates -- +and ticket 37's already-verified 2026 Annual-sheet gap reaching five columns +`r_extraction_gap` had not been registered for (41 rows, verified per column +against `2026 ISDFI, Annual!E32/H32/I32/AD32`). The residual 84 did not +converge and split into [ticket +50](tickets/50-triage-patient-raw-residual-6.md); ticket 12's `blocked_by` +swaps `49` for `50`, since closing 49 required reading and *running* +`r-archive/`'s own R source -- exactly the dependency ticket 12 waits on. Full +detail: [ticket 49](tickets/49-triage-patient-raw-residual-5.md). + +**Thirty-seven tickets resolved, and the patient raw stage is finished.** +[Triage the residual patient raw-stage column mismatches (round +6)](tickets/50-triage-patient-raw-residual-6.md) took the last 84 to **zero** +unclassified -- the first stage of either arm to reach it, ending a chain that +started at 14,903. Again no Python defect: three populations were R never +mapping the column at all (a header opening with thirteen spaces, a sheet whose +header merges have been deleted, a column with no header in either row), one was +a date typed into a numeric column that ticket 24's existing +`openpyxl_date_typed_stray_cell` already explained on the product arm, and one +was R keeping only the first selection of a multi-select screening block. The +two more interesting ones were **the comparison tool measuring wrong**, both now +fixed in `parse_date_flexible` itself: the month-name truncation required a word +boundary a following digit cannot provide, so a rich-text-damaged `July2014` +fell through to dateutil, which filled the day **from today** -- the comparison +was literally producing different numbers on different days; and the Excel-serial +ceiling of 100000 excluded the Buddhist-Era serials a BE year typed into a +Gregorian cell produces, so `241062` was read positionally as 24/10/62. +Correcting the ceiling removed 375 mismatches outright rather than labelling 12. +The cleaned stage is now the whole of the patient residual (7,967 across 27 +columns) and became [ticket +51](tickets/51-triage-patient-cleaned-residual-4.md); ticket 12's `blocked_by` +swaps `50` for `51` on the same standing precedent. Full detail: [ticket +50](tickets/50-triage-patient-raw-residual-6.md). + +**Fifty-one tickets resolved; the cleaned stage is now half explained.** +[Round 4](tickets/51-triage-patient-cleaned-residual-4.md) cut the patient +cleaned-stage in-scope residual from 5,031 to **2,538** (whole-stage 7,967 -> +5,474) on three causes, and for the first time the largest of them is squarely +**R's bug rather than a header or extraction gap**: `parse_date_string` +(`r-archive/R/script2_helper_dates.R`) asks lubridate for `ymd` before `dmy`, +so every source date written `D.M.YY` is read year-first -- `30.1.18` becomes +2030-01-18 -- and R's readings land 1,236 of the 1,714 affected cells in the +*future* relative to the tracker's own year. Verified in 2018 Yangon's own +cells, not inferred from shape. The second cause runs the other way and is a +deliberate Python divergence rather than a defect: Python's `_validate_dates` +sentinels a date past its tracker year and R has no future-date guard at all, +which surfaced a whole corrupt source column (2022 VNCH records every diagnosis +date as a 2023 one, for patients recruited in 2017) and one false docstring +claiming the guard "matches R pipeline behavior". The third closed a standing +fog patch: Python's accent-folding `sanitize_str` turns out to be *safer* than +suspected, not riskier -- it recovers the province `Thái Nguyễn` that R +discards, and cannot silently merge two provinces because +`validate_allowed_values` raises on any two allowed values that sanitize alike +(the 209-entry list has zero such pairs). + +What is left of the cleaned stage became [round +5](tickets/52-triage-patient-cleaned-residual-5.md): `t1d_diagnosis_age` (558) +and the numeric long tail, blood pressure (491), and ~533 date cells where +Python could not parse the source at all and R guessed badly. That last group +is deliberately not decided yet -- part of it is dates buried in clinical notes, +which is [ticket 39](tickets/39-recover-dates-embedded-in-free-text.md)'s open +HITL question. Ticket 12's `blocked_by` swaps `51` for `52` on the same standing +precedent: R cannot be retired while triage still has to read `r-archive/` to +root-cause a difference, which is exactly what this round did three times. + +**[Round 5](tickets/52-triage-patient-cleaned-residual-5.md) is now closed, and +it broke the pattern of the four rounds before it: its largest shape was not a +divergence to explain but two real Python bugs to fix.** A bare four-digit year +typed into a date cell -- 590 `dob` cells and 425 `t1d_diagnosis_date` cells +across nine trackers -- was read as an Excel serial and became a 1905 date, +which drove `age` to the 999999 sentinel and `t1d_diagnosis_age` to -95; and +the diagnosis-age derivation emitted negative ages where a source workbook +records a diagnosis before the birth date. Both are fixed and both were in +production BigQuery. The cleaned output now holds zero pre-1930 birth dates and +zero negative diagnosis ages, where it held 590 and 264. + +Two things worth carrying forward. First, **fixing the parser alone was not +enough and the map's own instrument said so**: it left one file (2019 Yangon +Children's, the only one whose D.O.B. column is date-formatted) unrecovered and +produced a 163-cell regression at the raw stage, where six earlier rounds had +reached zero. That regression is what located the extraction half of the bug; +the raw stage is back to 0 unclassified. Second, **the in-scope count can rise +when Python gets more correct** -- recovering the years made Python disagree +with R on two derived columns that had previously agreed on a shared wrong +answer -- so the number alone is not the progress signal, and a round that +raises it may be the round that did the most. + +What is left became [round 6](tickets/53-triage-patient-cleaned-residual-6.md): +2,278 in-scope cells, now led by `hospitalisation_date` (546) and the +parse-failure date family, blood pressure (491), the 298 diagnosis-age cells +that did *not* trace to a bare year, and the long tail. Ticket 12's +`blocked_by` swaps `52` for `53` on the same standing precedent. + +**[Round 6](tickets/53-triage-patient-cleaned-residual-6.md) is closed, and +like round 5 its work was fixing Python rather than explaining R.** The +in-scope cleaned residual went **2,278 -> 1,689** and the raw stage held at 0. +Two defects, both live in production output. + +The first is the third appearance of one root cause and was finally closed as +a class. `parse_date_flexible` has an alphabetic month-year branch precisely so +a month-and-year cell resolves to the first of the month instead of reaching +`dateutil`, which fills any absent field from `datetime.now()`. The branch's +pattern never matched the numeric (`10/2019`), comma-separated (`Mar, 2017`) or +apostrophe (`Jun'09`) spellings, so 42 production cells carried **the run +date's own day of month** — `parse_date_flexible("10/2019")` returned +2019-10-19 on the 19th. Ticket 50 had already closed one route into this same +fill. The fix stops enumerating spellings: the string is parsed twice against +two disjoint defaults, and any component that differs between the two parses is +one dateutil invented rather than read — an invented day becomes the 1st, an +invented month or year makes the cell unparseable. Checked by running the old +and new parser over all 5,185 distinct raw date strings: 36 differ, all of them +either a today-filled day becoming the correct 1st or a confidently wrong date +becoming the sentinel. + +The second is a plain data loss. All 491 in-scope blood-pressure cells had R +holding a plausible reading and Python holding the 999999 sentinel, and **every +affected source value has whitespace beside the slash** (`70 / 40`, `103/ 69`) +and nothing else wrong with it. `split_bp_in_sys_and_dias` splits on `/` and +left the padding on; R's `separate_wider_delim` does too, but R's `as.numeric` +ignores surrounding whitespace where Polars' cast fails on it. Trimming the +fragments restored 465 readings across 7 trackers and took both columns out of +the residual entirely (302 -> 13, 215 -> 13). + +**`hospitalisation_date` turned out not to be this round's work at all.** +Joining all 546 cells back to the raw stage and grouping by source value showed +the column is **100%** clinical notes — the "mangled date token and nothing +else" kind earlier rounds expected alongside them does not appear here. It is +therefore entirely [ticket +39](tickets/39-recover-dates-embedded-in-free-text.md)'s, and that ticket +gained a sub-question nobody had stated: 179 of the cells run the *other* way +(Python already recovers a date from the note and R sentinels), and 65 have +both pipelines finding a date and disagreeing about **which of several recorded +admissions** the cell means. The 489 cells were deliberately left +`unclassified` rather than labelled, so the open decision stays visible. + +**Round 7 closed the largest remaining column and one harness gap: 1,200 +in-scope -> 770 (36%), raw still 0, and nothing about the pipeline's behaviour +changed.** `t1d_diagnosis_age` (298 -> 0) was neither a Python defect nor the +bare-year effect round 6 predicted. **R never derives a diagnosis age at all**: +`fix_t1d_diagnosis_age` exists in R and is unit-tested against exactly the +strings the trackers carry, but its call site is commented out, so R only ever +passes the source column through `as.numeric` — a blank cell stays NA, a +word-written age (`11yr`, `At birth`, `4mth`) becomes 999999. Python derives +from `dob` and `t1d_diagnosis_date` and lands on a figure the source's own +words confirm. The remaining 16 are a date typed into the age column at two +clinics, where Python is right to null what R carries through as a serial. The +predicted bare-year framing was killed by measurement rather than argued away: +R and Python hold *identical* dates on the dominant Sarawak population, so +there is no bare-year mismatch to be downstream of. Separately, 132 cells were +a float-rounding artifact surviving cleaning because the schema types those two +screening columns as String — the one exception to ticket 22's "cleaning casts +its numerics" scoping — now normalized away rather than labelled. + +What is left became [round +8](tickets/55-triage-patient-cleaned-residual-8.md), and it is the first +cleaned-stage round to open with **two suspected Python defects at the head of +the queue** rather than R limitations: `height` (114), where Python emits +`0.069` metres into production from source cells R rejects outright, and +`fbg_updated_mg` (113), where R manufactures a reading of 140 from the text +"Lost follow up" and 200 from "HI" while Python sentinels — a case where R's +behaviour needs reading before Python's can be judged. `bmi` (22), +`insulin_subtype` (67, Python stamping "Undefined" over real brand names), the +nine smaller date columns (~440) and the long tail follow. Ticket 12's +`blocked_by` swaps `54` for `55` on the same standing precedent — round 7 read +R's commented-out call site directly, and round 8 opens by reading R's +`fix_fbg`. + + +**Round 8 closed both suspected defects at the head of its queue, and they were +real: 770 in-scope -> 448 (42%), raw still 0, three Python fixes shipped.** +`height` (114 -> 0) and `bmi` (22 -> 0) were one defect with two halves. +Python converted a height to metres above **2.3** where R converts above **50**, +so the 120 source cells sitting between the two units -- `2.31`, `2.43`, `6.9`, +`10.4`, `13.0` -- were divided by 100 and published as `0.069` and `0.0243` +metres rather than failing the [0, 2.3] bound as they do in R. And BMI was +derived at step 5.7, before range validation, so it was computed from that +impossible height and its result (10-12) passed the BMI bound; R sequences the +height cut first, which propagates the sentinel. Both halves are fixed and both +were live in production output. + +`fbg_updated_mg` (113 -> 0) was the alarming one and turned out to have **no +Python defect**. R's `fix_fbg` runs its CDC category words through `grepl` +without a word boundary, so 41 cells reading `Lost follow up` become a fasting +glucose of **140** -- "fol-low" contains "low" -- and 41 more reading +`SMBG 50-HI`, `129-HI`, `CBG 57-High` or bare `HI` become **200**, throwing away +the number the clinic did write. Python anchors the same patterns to the whole +string (`^(low|good|okay)$`) and sentinels everything else, which is the correct +side; it also reads `148 mg/dl (Mar-18)` where R's `as.numeric` fails on the +whole string. Two classifiers, no pipeline change. + +Two more results worth carrying. `insulin_subtype` recovered **56 rows** where +2024 Sarawak ticks the template's insulin boxes by writing the drug -- +`Novorapid`, `Glargine`, `Toujeo`, `Ryzodeg` -- which both pipelines discarded +by testing for `Y` exactly. And the date family gave up a third defect nobody +had suspected: `parse_date_flexible` accepted a year with a digit missing, so +`1/16/224` and `13-Mar-0202` reached production as `0224-01-16` and +`0202-03-13`; the parser now floors the calendar at 1900, mirroring the +beyond-tracker-year guard at the other end. + +**One attempted fix was measured and reverted, which is the round's other +lesson.** Making an unticked insulin row null instead of `Undefined` is +defensible in principle -- and created **17,418 new divergences**, because R +publishes `Undefined` there too and the two pipelines already agreed. The +question is real but it is a change to shared behaviour, not a residual to +triage, so it is recorded as fog rather than smuggled into a triage round. + +What is left became [round +9](tickets/56-triage-patient-cleaned-residual-9.md): 412 of the 448 are the nine +smaller date columns, and round 8 killed the obvious framings by measuring them +-- only 2 of 414 are a day/month swap and none is a year-only difference. The +residual is concentrated by tracker instead (2021 and 2020 Mahosot DC account +for ~150), and moves together across `hba1c_updated_date`, `fbg_updated_date` +and `bmi_date`, which points at the legacy path that lifts a date out of the +measurement cell rather than at nine separate causes. Ticket 12's `blocked_by` +swaps `55` for `56` on the same standing precedent -- round 8 read four of R's +own functions to reach its verdicts. + +**Round 9 closed the date family as a single mechanism: 448 -> 96 in-scope +(79%), raw byte-identical at 0, product untouched.** Its lead was half right +and half wrong, and measurement said which. Right: the nine date columns share +one cause. Wrong: the cause is not `extract_date_from_measurement` and the +Mahosot DC concentration is not where that path lives. R's `parse_dates` +shortens any word of four or more letters by deleting its **fourth letter** -- +`April-17` becomes `Aprl-17` -- and then hands the wreckage to a fixed order +list ending in `my` and `y`, which cannot fail loudly. Installing lubridate and +running that chain over the real source strings reproduced R's frozen output +exactly: a spelled-out month collapses to 1 January of its year, an unreadable +month makes R read the **day** as the month (`9-Dce-20` -> 2020-09-01), and a +day past 12 leaves R with no reading at all. Three classifiers now carry it and +Python is right in all three. + +The same investigation found **three Python gaps, all real data loss**: month +spellings Python did not know -- the Bahasa Malaysia `Mac`/`Mei`/`Okt` the +Malaysian clinics write in a column everyone else writes in English, all twelve +Thai abbreviations, plus the transposition `Dce` and the dropped letter in +`ug`; separator runs damaged by a stray keystroke (`26-05- 2007`, +`19-Jan_2023`, `02-Apr=-2026`), which R recovers because lubridate splits on +any non-alphanumeric run; and zero-width characters pasted in from another +application. Old parser against new over all **5,187** distinct raw date +strings: 32 changed, every one sentinel-to-real-date, none of the already +parsing values altered. **The narrowness was forced by measurement, not +caution**: the first, lubridate-like version of the separator repair turned the +*range* `11-15 /01/2019` into a single date of 2001-11-15, so the repair now +runs only when at most three numbers remain. + +`insulin_subtype`'s 11 cells were traced and deliberately left unclassified: +R's derivation is a chain of `ifelse(x == "Y", ...)`, so an all-`-` row yields +`""` and becomes null while an all-null row yields `NA` and becomes +`Undefined`. **R's null-vs-`Undefined` split is NA propagation, not a designed +distinction** -- which strengthens the standing fog entry rather than +answering it, and means R's null there is not evidence of intent. + +What is left became [round +10](tickets/57-triage-patient-cleaned-residual-10.md): 96 cells in five groups +that share no mechanism -- the 2017/2018 measurement-cell dates where R's +`extract_date_from_measurement` genuinely does live and Python has no +equivalent (~27), the undecidable `25-Ma4-2025` (18), glued digit groups +where R's own answer is not stable (~19), a numeric tail where two FBG cells +sit *inside* the analytical bounds and should not have been sentinelled (14), +and the 13 `Undefined` cells the fog owns. Ticket 12's `blocked_by` swaps `56` +for `57` on the same standing precedent -- this round could not have been +closed without running R. + +**Round 10 ends the round chain: 96 -> 19 in-scope, and no round 11.** The +last real Python defect in the patient cleaned stage was a single character -- +R's `extract_date_from_measurement` makes the closing parenthesis optional and +Python's did not, so a measurement cell written `180(May-2017` (25 of that +group's 30 source cells) lost its date. 24 cells recovered. The remaining 19 +are not unexplained: 13 are the standing `Undefined` fog and 6 are six Kantha +Bopha readings whose cause is a *column-level* property (the ticket-42 unit +swap) that no per-cell classifier can see -- a widening that would have caught +them was written, measured, and backed out for claiming far too much. Everything +else resolved to source defects, now on ticket 40. + +**Ticket 12's blocking changes kind rather than contents.** For ten rounds it +swapped round N for round N+1 on one standing precedent: R cannot be retired +while triage still needs to read it. With the rounds finished, `blocked_by` +drops `57` and gains `[32, 39, 44, 47]` -- the four open tickets that inherit +that precedent (the classifier re-audit reads R's source by definition; ticket +39's 478 clinical-note cells and ticket 44's 2,936 FBG cells are the two +largest unexplained populations left; ticket 47 is a correctness question whose +evidence includes R's row handling). Letting R retire now, treating those four +as residuals of closed scope, was the alternative -- rejected because this +ticket was already unblocked prematurely once on that reasoning and the +correction cost a session. It is a one-line edit to reverse per ticket. + +**[Merged patient IDs](tickets/47-patient-ids-merged-at-cleaning.md) is closed, +and it was the first ticket on this map to fix a defect in *identity* rather +than in a measurement.** The pipeline was publishing `KH_NPH02` -- an ID no +tracker contains -- with four patients' September records under it. It no +longer truncates: an ID that fails the format is recovered from the tracker's +own correct spelling where exactly one candidate sits an edit away, and +sentinelled where none does. The user set that policy explicitly ("C as first +rule and B as fall back"), and the NOGH case shows why the fallback matters -- +its Patient List writes the malformed IDs too, so there is nothing to recover +against and three patients stay merged under `Undefined` until the clinic fixes +the workbook. + +The fix did **not** restore the recovered rows' demographics -- extraction +joins the Patient List on the unfixed ID, which is now [ticket +58](tickets/58-patient-list-join-uses-unfixed-id.md). + +**The comparison was re-run at the user's request, in the same session.** The +patient arm was regenerated against the real 254-tracker set and diffed against +the frozen `output_r/`. The result is worth carrying forward as a shape, not +just a number: **this divergence never reaches a cell comparison.** Patient's +row-alignment key is `patient_id` + `sheet_name`, so R's `KH_NPH02` rows and +Python's recovered `KH_NP026`-`KH_NP029` rows simply stop pairing -- row-key +divergence 6 files/121/15 -> 7 files/125/19, the whole delta being +`2023_NPH`'s 4+4, while cell mismatches went *down* 114,373 -> 114,371 and no +column or cause moved beyond two FBG cells. Patient raw: zero delta. A future +session scanning the `cell_mismatches` sheet will not find this there, which is +why it is written here; no classifier was added, since the cause registry +classifies cell mismatches and there is no cell mismatch to classify. + +**That last point turned out to be a tooling gap, not just a caveat, and the +user caught it**: the comparison reported unmatched rows only as *counts*, so +the four keys this fix introduced could be named only by querying the parquets +by hand -- which is exactly what this session did. `RowKeyOverlap` now carries +the unmatched keys themselves and the report gained a `row_key_unmatched` +sheet (file, side, key, surplus rows). Its first run named a population nobody +had looked at: **144 unmatched rows across 7 patient cleaned files and 130 at +the raw stage, of which only 8 are ticket 47's** -- the largest being 98 R-only +rows in `2026_Preah Kossamak`. Product is 0 at both stages, resolved by ticket +17's ordinal key. That population is structurally invisible to every triage +ticket run so far, because a row with no partner never reaches a cell +comparison; it became [ticket +59](tickets/59-triage-unmatched-row-keys.md), the same shape of gap +`compare_columns` was before ticket 26. + +**The frontier is nine tickets.** Ticket 47's closure drops it out and ticket +58 replaces it, so the count holds. Ticket 12's `blocked_by` loses `47` and +keeps `[32, 39, 44]` -- ticket 58 is **not** wired as a blocker, because unlike +those three it does not need `r-archive/`'s source to answer: its evidence is +Python's own join and the source workbooks. Of the nine, [the classifier +audit](tickets/32-audit-classifiers-against-decision-bar.md), [dates in +clinical notes](tickets/39-recover-dates-embedded-in-free-text.md) and [the FBG +R-null residual](tickets/44-triage-cleaned-fbg-r-null-residual.md) are on the +route to the destination; the rest are standing decisions, the source-defect +report, a separate feature (ticket 16) and ticket 58. + +**[Dates buried in clinical notes](tickets/39-recover-dates-embedded-in-free-text.md) +is closed, and it was the first ticket on this map whose pattern set was built +from a measurement the user asked for rather than from the ticket's own +examples.** The scan of the whole column -- 745 distinct strings over 7,847 +cells in 240 trackers -- changed three of the answers, and one of its rows +decided the central question outright: **48 cells carry digits and no date** +(`3 month come back meet Doctor`, `on stamlor 5mg`), and `dateutil`'s fuzzy mode +does not decline on any of them. The header was read from the workbook rather +than assumed -- "Hospitalisation due to diabetes emergency or glucose control +(Include Date)" -- which is why this column and no other carries prose. + +Python now reads the date out of the note: `hospitalisation_date`'s sentinels +fell **673 -> 156**, and its R/Python mismatches went **478 -> 0** unclassified +across three new causes. The recovery is auditable rather than silent: 1,223 +error-table entries under `date_recovered_from_text`, `date_multiple_in_cell` +and `date_year_inferred`, the middle one feeding +[ticket 40](tickets/40-source-defect-findings-report.md), which also gained the +template's own placeholder text found sitting in ~170 data cells. + +Two things worth carrying forward. First, **the ticket's own largest claim was +false and cost nothing to check**: it named `t1d_diagnosis_date` (559 cells) as +part of this population, and every one of those sentinels traces to a clean +source date rejected by the future-date guard -- round 4's corrupt 2022 VNCH +column. Second, **the session's three real defects were all found by reading +measured output, not by testing**: a bare range with no prose around it was +being dated from its first number as a *year* (`6-12 Nov 2020` published as +2006-11-12), the prefix walk was beating the explicit recogniser to exactly +those cells, and `_is_range` was testing `match.lastgroup`, which names an +inner group and so never fired. + +**Ticket 12's `blocked_by` drops `39`, leaving `[32, 44]`** -- the classifier +re-audit and the FBG R-null residual, the last two open tickets that still need +to read `r-archive/`. The frontier is nine tickets; of them, those two are what +stands between this map and retiring R. + +**[The FBG R-null residual](tickets/44-triage-cleaned-fbg-r-null-residual.md) +is closed, and it ends the patient cleaned-stage triage: 2,955 unclassified +cells become 16, every one of the 16 already owned by a fog patch.** The +ticket's own framing was the thing that had to give. It described one +population (R null, Python present) and named a file-level obstacle -- a +per-cell classifier cannot see whether a file's glucose column was +unit-swapped. Scanning the whole column first, per this map's standing +preference, found the decisive fact is **row-level and much stronger**: neither +pipeline reads the mmol column from the workbook at all. Both derive it from +the mg cell beside it, so an mmol divergence is an mg divergence restated -- +and all 2,935 cells, without exception, sat beside an mg cell that already +carried a named cause. + +That reframing also surfaced **110 cells running the other way, which no ticket +had ever described**: Python null, R holding a value. Each is exactly +`mg_r / 18` -- R dividing a number `fix_fbg` manufactured out of text (140 from +"Lost follow up") or one far past the analytical ceiling (2013 mg/dL) and +publishing the quotient as a measurement. `2023_CDA` gets **111.8 mmol/L** that +way, above the level A4D's medical advisor called impossible. A further 28, +also unlooked-at, run the third way: 2018 CDA writes `148 mg/dl (Mar-18)`, +R's `as.numeric` fails on the whole string and Python reads it. Python is the +correct side in all three shapes, each already argued under the mg cell's own +cause -- so `mmol_derived_from_mg_sibling` names the cascade rather than +re-deciding it, bounded so it cannot claim an mmol cell carrying a reading its +mg sibling does not account for. + +**The file-level context the user chose (option 1) still earned its keep, just +not where the ticket expected**: `column_unit_swapped`, read back from the +run's own `glucose_unit_swapped` error records, lets +`_is_python_glucose_unit_corrected` judge a reading against the unit the column +*holds* rather than the one its name claims -- which closes the standing fog +patch **"Whether a column-level finding can be classified at all"** and the six +2020 Kantha Bopha cells round 10 could only write out in prose +(`fbg_baseline_mg` unclassified 6 -> 0). Verified against the real 254-tracker +drive data across two full runs: `per_column` counts are byte-identical and the +patient raw, product raw and product cleaned snapshots are unchanged, so +nothing was suppressed -- only named. + +**Ticket 12's `blocked_by` now reads `[32]` alone.** [The classifier +re-audit](tickets/32-audit-classifiers-against-decision-bar.md) is the single +remaining open ticket that needs `r-archive/`'s source, and it needs it by +definition. **One decision now stands between this map and retiring R** -- and +the re-audit is properly last, since it audits every classifier this map has +written, including the two added here. The frontier is eight tickets; the other +seven are standing decisions, the source-defect report, a separate feature +(ticket 16), and tickets 58 and 59. + +**[The classifier re-audit](tickets/32-audit-classifiers-against-decision-bar.md) +is closed, and the thing that decided the session was measuring its own +premise.** The ticket said every classifier in `compare.py` was written before +the map's *triage means deciding* bar. Dating all 50 causes by first commit put +**12** before the bar and **38** after it, each of the 38 argued in its own +triage ticket — so the audit is 12 causes, not 50, and the other 38 are +explicitly out of scope rather than pending. The four the ticket named as +known-weak are resolved: `off_by_one_day` deleted (its ten cleaned rows are +consecutive daily entries knocked one position out of step by R's null-date sort +fallback — `row_order_divergence` wearing another name, winning only on registry +merge order), `ce_typo` deleted (it fired on `normalize_date_column`'s own 9999 +sentinel), `sentinel_null` renamed `python_sentinel_r_extraction_gap` after +tracing it to Sarawak's 2023 tracker genuinely holding December **2024** dates in +its `Dec23` sheet, and `r_value_missing` bounded — it is right about R for 11,436 +of its 11,468 rows and was speaking for 32 more where Python publishes `0202-06-20` +or a Buddhist-era year. + +**Both of the session's pipeline defects were found by pulling a thread from a +classifier, not by testing.** `ce_typo`'s two rows led to +`_validate_entry_dates` exempting every year `>= 2400` so Buddhist-era dates +could flow — which also let Excel serials 1,339,576 and 411,384 reach BigQuery +as `5567-08-19` and `3026-04-30`. The exemption is now the tracker's own BE band, +keeping all 22 genuine Thai BE dates and sentinelling the 3 corrupt ones under a +new `implausible_era_date` code. Emitting that code then revealed the second +defect: **`a4d run`, the production entry point, published an errors table +containing the patient arm only** — the patient arm writes it from inside +`run_patient_pipeline` and nothing wrote the product arm's, so BigQuery's +`errors` table has never held a `balance_reconciliation` row (116 exist). +63,295 -> 97,326 records. That matters beyond this ticket: +[the source-defect report](tickets/40-source-defect-findings-report.md) is built +on the premise that the errors table is the source of truth. + +**Ticket 12's blocking was re-derived, not swapped, and it grew.** `blocked_by` +is now `[59, 60]`: [the second half of the +audit](tickets/60-audit-remaining-pre-bar-classifiers.md) needs `r-archive/` by +definition, and [rows that pair with nothing](tickets/59-triage-unmatched-row-keys.md) +is **added** on a change of judgement — explaining 98 R-only rows is a question +about R's row handling, not Python's output. So R does not retire this session; +two tickets stand between the map and retiring it, where the previous session +expected one. The frontier is nine tickets. Also spawned, and deliberately not a +blocker: [whether a Thai clinic's Buddhist-era date is published as 2567 or +converted to 2024](tickets/61-decide-buddhist-era-date-conversion.md) — today the +product table carries entry dates 543 years in the future for 22 rows, which R +does not convert either. + +**Ticket 61's question is already decided, and measuring it doubled its size.** +The user's answer (2026-08-22): where a date is clearly not Gregorian and its +intended value is unambiguous, **convert it in the cleaned stage, not in raw**, +auditably, like any other recovery. The ticket asked for the patient arm to be +measured first — done, and **patient is the larger half and is losing data +rather than publishing it oddly: 381 cells across 95 distinct values, every one +clobbered to the 9999-09-09 sentinel** because a Buddhist year reads as a future +date. Nothing had surfaced them, since R sentinels them too and so the +comparison has no mismatch to show. The product fix's band test looks right for +patient as well: 2022 Hat Yai's `2560-01-01` converts to 2017-01-01, the date +[ticket 40](tickets/40-source-defect-findings-report.md) had separately +established from that patient's own D.O.B. and recruitment. **Implementation was +deliberately left to a fresh session** — it moves 381 patient cells plus 22 +product rows and needs both guards, tests, a full 254-tracker run and a +comparison. + +**That session ran, and the conversion is live on both arms.** [Whether a Thai +clinic's Buddhist-era date is published as 2567 or converted to +2024](tickets/61-decide-buddhist-era-date-conversion.md) is closed: **375 +patient cells recovered** (the 381 measured, minus 6 that decode to nothing) +and **22 product rows converted**, with `product_data` now holding zero entry +dates past year 2400. The user settled the one question the previous session +left open — the two `hospitalisation_date` cells naming a BE date *range* stay +unconverted, because the range is a problem [ticket +39](tickets/39-recover-dates-embedded-in-free-text.md) declined and this +decision did not overturn; they become source-defect findings instead. Every +divergence the change created is named: two new causes +(`python_buddhist_era_converted`, `buddhist_era_conversion_row_order`), and +**unclassified did not move on either arm** (16 patient, 21 product). Full +detail: [ticket 61](tickets/61-decide-buddhist-era-date-conversion.md). + +**The frontier is eight tickets.** [Ticket +12](tickets/12-retire-r-workspace.md) is unchanged at `blocked_by: [59, 60]` +— this ticket never needed `r-archive/`, so closing it neither adds nor +removes a reason to keep R. [Rows that pair with +nothing](tickets/59-triage-unmatched-row-keys.md) and [the eight pre-bar +causes](tickets/60-audit-remaining-pre-bar-classifiers.md) remain the two +tickets standing between this map and retiring R. + +**[Rows that pair with nothing](tickets/59-triage-unmatched-row-keys.md) is +closed, and it was the first ticket on this map to work a population the +comparison could count but never show.** A row with no partner never reaches +`compare_cells`, so the ten-round patient triage chain — which worked entirely +off `cell_mismatches` — was structurally blind to it. What sat there turned out +to be five separate things, and the keys alone sorted them before any code was +read: two whole month sheets missing from Python, one patient missing from R +eleven times, and two already-closed decisions resurfacing under a key that +cannot pair. + +**The largest Python defect this map has found by row count was a single +space.** `2022_Children's Hospital 2`'s `Oct22` sheet has its first patient's +row number cleared, leaving `' '` in `A70`. `find_data_start_row` scans for a +*numeric* cell, so it started at 71, read the header rows from 70 and 69, and +row 70 is data — the patient-ID header became the literal string `VN_CH001`, +harmonization found no `patient_id`, and the sheet was skipped whole. Fifteen +patients' October records, gone. **The same residue costs R a patient in the +opposite direction**: R's hardcoded `+1` for 2022+ trackers assumes exactly one +leading empty row, and every `2024_Mandalay Children's` sheet except `Jan24` +holds a space in row 1, so R drops `MM_MD001` from 11 of 12 sheets. Python is +correct there and R is not being repaired. + +**The 254-tracker scan is what made the fix narrow.** Adopting R's rule — first +non-empty cell — would have been the obvious repair and would have destroyed 14 +sheets whose column A holds a stray `'m'`/`'f'`/`'n'` in row 1 (2026 Gensan, +2025/2026 VNCH). Python now extends the numeric block back over a whitespace-only +cell *directly abutting* it, one row only; measured across the corpus, that moves +exactly one sheet. + +**The third finding is the one with a decision in it, and the user made it.** +`read_all_patient_sheets` drops any row whose patient ID starts with `#` — a +broken formula — and did so with no error record at all. The rows are not empty: +`2026_Preah Kossamak`'s `May26` is 98 of them, carrying 50 ages and updated FBG +readings, 47 baseline HbA1c / weight / height / BMI, 45 insulin regimens. The +user chose **keep dropping, start reporting**, on the grounds that `Undefined` +is a bucket rather than an identity and keeping them would pool 98 people under +one group key. That is compatible with [ticket +47](tickets/47-patient-ids-merged-at-cleaning.md) keeping a *misspelled* ID +under the same sentinel: a misspelling is an identifier a clinic could +reconcile, `#REF!` is not one at all. The new `excel_error_patient_id` code +fires on **120 rows across 9 trackers** — 17 more than the comparison could +ever have shown, since it only saw the rows R also kept. + +**Ticket 12's `blocked_by` drops `59`, leaving `[60]` — one ticket now stands +between this map and retiring R.** The frontier is seven; of them, only [the +eight pre-bar causes](tickets/60-audit-remaining-pre-bar-classifiers.md) is on +the route to the destination. The other six are standing decisions, the +source-defect report, a separate feature (ticket 16) and ticket 58. No ticket +was spawned: the residual unmatched population (129 cleaned / 115 raw) is +entirely accounted for by decisions now made. + +**[The eight pre-bar causes](tickets/60-audit-remaining-pre-bar-classifiers.md) +is closed, and for the second session running the thing that decided it was +measuring a classifier's own claim rather than the difference it names.** +`buddhist_era_typo` asserted a Buddhist-era typo on twelve cells; opening the +two source workbooks showed years of 3035 and 5025, which are neither +Gregorian-plausible nor 543 off their tracker's. Ticket 61 is what made the old +`year >= 2400` test wrong rather than merely loose: once genuine BE dates are +converted in cleaning, anything still above the threshold is by construction +not one. + +**The session's second finding came from its own fix**, which is worth carrying +forward as a habit: bounding one classifier dropped five rows onto +`r_parse_order_cannot_read_cell`, a *post-bar* cause and so out of ticket 32's +scope — but the label was false (R never failed to parse; it carried the Excel +serial `1141523` through as a string, and the comparison's own date +normalization is what produced the sentinel). Ticket 60 owned it because its +own change put them there. The fix was to wire ticket 32's existing +`python_absurd_excel_serial` to the patient date columns, where its +Buddhist-era band had never been able to apply at all — no patient stage sets +`tracker_year_col`, so it now reads the year off the sheet name like the other +patient date causes. + +The measurement that makes those two safe to believe: every per-cause count in +all four stages, before and against after. **Exactly four entries moved**, and +`unclassified` held at 37 — so nothing was quietly pushed into the unexplained +bucket, and all twelve rows went from a false label to a true one. + +**Two more causes had their docstrings corrected without a single count +moving**, which is the shape ticket 32 predicted: the label was right and the +explanation under it was incomplete. `r_extraction_gap` spans thirteen columns +against the six mechanisms it documented, and one of the three newly traced +mechanisms **corrects an explanation already sitting in the registry** — the +comment above the 2026 ISDFI cells blames R for reading nothing from the Annual +sheet, when R reads 102 of that file's 120 `edu_occ` rows and the actual cause +is three rows whose column-A row number is blank. That is the same residue as +ticket 59's cleared row number, on a different sheet type. +`r_validator_rejects_multivalue` turned out to name two R defects, the +documented CSV rejection plus NA propagation through `ifelse`/`paste`. + +**Ticket 12's `blocked_by` moves from `[60]` to `[62]`, so R still does not +retire, and the count of tickets standing between this map and retiring it is +unchanged at one.** [Finishing the audit](tickets/62-finish-the-pre-bar-classifier-audit.md) +inherits the two causes ticket 60 did not reach plus `r_extraction_gap`'s 98% +bulk (`recruitment_date` 28,009 and `edu_occ_updated` 2,770, both left +unmeasured) — all three are questions about R's own code, so `r-archive/` stays. +The frontier is seven; of them only ticket 62 is on the route to the +destination. + +**[Finishing the audit](tickets/62-finish-the-pre-bar-classifier-audit.md) is +closed, and with it the twelve-cause audit ticket 32 scoped. It is the first +ticket in that chain to find no defect at all** — Python is the correct side in +all three populations and no pipeline code changed. What it found instead is +that every one of the three *explanations* was wrong, which is the failure mode +the audit exists for: a cause can carry the right label over the wrong reason +indefinitely, because the count never moves. + +**The largest correction is that a mechanism blamed on R's code was really a +difference between two Excel readers.** `r_category_lookup_miss`'s docstring +said R's join is case- and whitespace-sensitive. Bucketing all 866 rows by the +exact bytes showed 652 of them never give the two sides the same string to +compare: the reference workbook stores embedded line breaks as a bare `\n` (its +Stock_Summary sheet XML holds 29 LF and **zero** CR bytes) while the trackers +store `\r\n`, readxl returns each faithfully, and openpyxl folds CRLF to LF. The +reference sheet already contains all 26 names verbatim — R should have matched +them. Only the remaining 214 rows, over three names, are genuine case +(`(singles)` vs `(Singles)`, `ACCU-CHEK` vs `Accu-Chek`). + +**The second correction runs the other way: a classifier claiming Python is the +faithful side is backwards for two-thirds of its population.** +`openpyxl_date_typed_stray_cell`'s 100 rows are three shapes, not one. Fourteen +are a real date in a quantity column, where Python is right. Sixty-six are a +quantity (5-200) or a plain zero sitting in a date-formatted cell — there R's +serial *is* the number the clinician typed and Python's `1900-05-29` is the +misleading rendering. It costs nothing only because the divergence dies at +cleaning: both sides publish identical cleaned values for every affected group, +verified against the real data, so the pipeline was deliberately left alone. The +last 20 are the patient arm the docstring never mentioned, and they are the +strongest Python-is-right case on the map — five source cells where Excel +silently auto-converted a typed value (`10/60` becoming Oct-1960 in a blood +pressure column, `1-2` becoming 1-Feb in "Testing Frequency (per day)", a +diagnosis date in "Age at Diagnosis"), all five re-opened and confirmed +date-formatted, and R publishes them as **diagnosis ages of 20,668 and 42,859 +years** where Python nulls them. + +**The third is the sharpest mechanism the map has found for an R gap.** +`r_extraction_gap`'s `recruitment_date` bulk was described as R failing "for the +large majority of patients". R populates 64% of its rows, and the loss is +perfectly file-level: 91 trackers where it reads **zero**, 148 with no +divergence, and **zero** files where it reads some and misses others. A trailing +space in the header cell forces Excel to write ``, and +`openxlsx` — which is what R reads headers with — folds the attribute into the +header text, so `make.names` emits `xmlspacepreservedateofrecruitmentmmmyy` and +no synonym matches. The correlation is exact: 91/91 affected files carry that +column in R's own raw parquet, 0/148 of the rest do. The same defect eats an +insurance-status column nobody had noticed. + +**Ticket 12 is *not* unblocked, and this time the check was run before saying +so.** Re-deriving the open tickets that still need the R *source* — the map's +Notes make that list derived, not inherited — leaves exactly one: [monthly rows +with a misspelled ID](tickets/58-patient-list-join-uses-unfixed-id.md), whose +question 3 is literally "Check R". Tickets 6, 9, 16, 34, 35, 40 and 41 were each +checked and need none. So `blocked_by` moves `[62]` -> `[58]` and the count of +tickets standing between this map and retiring R stays at one. **The frontier is +six**: ticket 58 is now the only one on the route to the destination, and the +other five are standing decisions, the source-defect report and a separate +feature (ticket 16). One fog patch was added — whether Python's category join +should normalise line endings explicitly, rather than relying on openpyxl to do +it — and no ticket was spawned. + +**[Monthly rows with a misspelled ID](tickets/58-patient-list-join-uses-unfixed-id.md) +is closed, and it is the largest data recovery this map has made.** Both +whole-tracker joins in extraction — the `Patient List` demographics and the +`Annual` sheet — keyed on the raw `patient_id`, while the identity fix that +makes a respelled ID usable runs much later, in cleaning. Where a month sheet +spells a patient differently from the Patient List in the same workbook, the +join could not match and the row kept its measurements while losing every +static column. Both joins now key on a *derived* normalized ID; the raw +`patient_id` column is untouched, so the comparison's row-alignment key is +unaffected and row-key divergence did not move on either stage. + +**The blast radius was a hundred times the known case, and it is one clinic.** +The ticket knew of 4 NPH rows. Measured across all 254 real trackers: **695 +rows in 6 files**, of which **672 are the 2023 and 2024 Mahosot trackers** — +their month sheets write `LA-MH056` and up with a hyphen against a Patient List +spelled `LA_MH056`, 38 patients in 2024 and 27 in 2023. **680 rows recovered, +carrying 6,863 Patient List cells** across 12–13 columns, plus 8 rows / 40 +cells from the Annual join. The remaining 15 rows are source defects: NPH's +stray `H`, and `2024_Mandalay General`'s `MM_YG013_MG`, which that workbook's +Patient List does not contain under any spelling. + +**R has the identical gap, so the comparison was structurally blind to all of +it.** R's own frozen cleaned `2024_Mahosot` carries the same null block +starting at `LA_MH056`. Both sides null means no cell mismatch — this is the +third population on this map (after [ticket +59](tickets/59-triage-unmatched-row-keys.md)'s unmatched rows and [ticket +42](tickets/42-fbg-unit-headers-and-implausible-values.md)'s unit swap) that +the tool can never show, found only by measuring the workbooks directly. + +**Two things the session got right by measuring rather than reasoning.** The +fan-out risk — two static entries folding to one key would silently duplicate a +patient's months — is zero across all 192 Patient-List-bearing trackers, which +is what made the choice cheap. And the new cause was first wired to every +patient column on the argument that its own gate would bound it; the run showed +it over-claiming on the three columns the month sheets *also* carry +(`t1d_diagnosis_age` 1,069 against a join-miss population of 679, `bmi` 144, +`age` 1), so it was backed out to an explicit list. + +**The divergence the fix creates is only half named, deliberately.** New cause +`r_static_join_misses_respelled_id` fires on 5,959 raw-stage cells across 16 +columns and takes raw `unclassified` to **zero**. It cannot work on the cleaned +stage: its discriminator is the unnormalized spelling, and cleaning removes +exactly that on both sides. Cleaned `unclassified` is therefore **16 → 3,593**, +and that is [ticket 63](tickets/63-name-the-cleaned-stage-static-join-divergence.md). + +**Ticket 12's `blocked_by` swaps `58` for `[63]`, so the count stands at one — +but the previous session's reason was wrong and is corrected rather than +dropped.** Ticket 58's "check R" was answered from R's frozen *output*, which +the map's Notes put outside this ticket's scope; it never needed `r-archive/`. +Ticket 63 does: its question 2 asks what mechanism leaves R null on `bmi`, +`age` and `t1d_diagnosis_age` for these patients, which is a question about R's +code. **The frontier is six**, and ticket 63 is the only one on the route. + +**[The cleaned stage's 4,949 unnamed cells](tickets/63-name-the-cleaned-stage-static-join-divergence.md) +is closed, and the first thing it found was that its own headline number was +wrong.** The ticket said 3,593; five consecutive comparison runs say **4,949**. +The fix is the cheap one — the affected identities are derived per file from +the run's own *raw* output, which is the last place the two ID spellings +coexist, and threaded to the cleaned stage as a `CellMismatch` flag. No +production code moved. + +**Wiring it naively would have been the session's mistake, and measuring is +what caught it.** The prepended cause immediately claimed 9,302 cells across +**43 files**, against a join miss that touches 6. 2024 Preah Kossamak reads +**zero** `recruitment_date` in 863 rows and 2024 Yangon General zero in 1,057 +— that is `r_extraction_gap`'s file-level `xml:space` header defect, which +[ticket 62](tickets/62-finish-the-pre-bar-classifier-audit.md) established +nulls a column for every patient at once — while 2024 Mahosot reads 672 of +1,068, so *its* nulls really are per-patient. A new `r_column_empty_in_file` +flag makes the cause decline the first case, keeping 2,336 cells off a label +whose reason is R's header rather than R's key. + +**The strongest evidence the reassignment is right is that two independent +discriminators now agree.** Four existing causes were quietly holding +join-miss cells on the cleaned side — +`recruitment_date|r_extraction_gap` (671), +`fbg_baseline_mg|r_join_suffix_collision` (677), `edu_occ` and +`edu_occ_updated` (224 each) and the three blood-pressure columns (9 each) — +and every one of those figures now matches the **raw** stage's count for the +same column cell-for-cell, though the two stages compute the discriminator in +completely different ways. + +**`age` turned out not to be a wiring question at all.** Its 113 cells are a +cascade: `_fix_age_from_dob` overrides the sheet's Age cell with the age +derived from `dob`, and R's `dob` is null for **all 275 rows** of the affected +patients, so R publishes the clinic's typed figure or nothing. LA_MH060's +recovered D.O.B. is 2009-01-16, so `Jan23` is 14 — Python's value; R says 13, +the clinic's own, a month stale. New cause `r_age_not_derived_without_dob`. +Of the ticket's other two over-claim columns, `bmi` has **zero** cells on +these identities and `t1d_diagnosis_age`'s 1,078 keep their causes, whose +reason was right but incomplete — a docstring correction with no count moving, +which is [ticket 62](tickets/62-finish-the-pre-bar-classifier-audit.md)'s +shape a third time. + +**Ticket 12's `blocked_by` is now empty, and R can retire.** Re-derived rather +than inherited: tickets 6, 9, 16, 34, 35, 40 and 41 were each grepped for a +question about R's code and none has one, and nothing in the codebase reads +`r-archive/` at runtime — the only references outside `docs/` are two lint +exclusions and eight docstring citations. Those citations are the one open +question recorded on ticket 12, not a blocker: deleting the archive makes the +evidence behind the cause registry unverifiable, and whoever takes the ticket +decides what to do about that. **The frontier is six** — ticket 63 leaving it +and ticket 12 joining it — and [retiring +R](tickets/12-retire-r-workspace.md) is now the only one on the route to the +destination. The other five are standing decisions, the source-defect report +and a separate feature (ticket 16). + +**[Retiring R from the workspace](tickets/12-retire-r-workspace.md) is closed, +and `r-archive/` is gone — 156 tracked files, 1.9M, deleted outright.** It is +recoverable through the annotated tag `r-archive-removed`, whose `^` is the last +commit containing it, so a citation can still be checked with one `git show` and +nobody has to know a SHA. The ticket had waited through eleven rounds of triage +on a single derived rule — *R retires only when nothing still needs to read it* +— and that rule finally ran out of claimants when [ticket +63](tickets/63-name-the-cleaned-stage-static-join-divergence.md) closed. Four +config/doc sites moved with it: `CLAUDE.md`'s "R Archive" section and its "do +not modify" instruction (the conflict this ticket was written to resolve), +`docs/CLAUDE.md`'s `reference_data/` note, `SETUP.md`'s pointer, and +`pyproject.toml`'s ruff exclusion — plus two dead ignore rules the ticket's own +inventory did not know about. + +**The one thing that made the decision was measuring the ticket's own headline +number, which was wrong by a factor of four.** Its inventory said the only +non-doc references to R were two lint exclusions and eight docstring citations +in `compare.py`. That figure came from grepping the literal string `r-archive`, +which misses every citation naming an R file without its path. Grepping +`script[0-9]_[a-z_]*\.R` instead finds **35 citations across 12 modules** — +`compare.py` has 17, but `clean/patient.py` has 10, and six other production +modules carry the rest. That killed the option of inlining the cited excerpts +before deleting, and it is why the residue is a ticket rather than a footnote. + +**The destination moved closer in one respect and further in another.** Ticket +12 was the last open blocker of [promoting `migration` into +`dev`](tickets/06-promote-migration-to-dev.md), so for the first time every +ticket in that list is closed. It was re-blocked in the same session on the one +ticket this session spawned: [rewriting every docstring and doc that explains +the code by what R did](tickets/64-documentation-overhaul-drop-r-framing.md). +The user's framing is that "to match R" was fine as *working* documentation and +is now simply wrong — it explains the code by something that no longer exists — +and promotion is the act that turns this branch's documentation into the +project's. Round 4's precedent is why that is a rewrite and not a +find-and-replace: `_validate_dates` carried a docstring claiming its future-date +guard "matches R pipeline behavior" when R has no such guard at all, so a stale +R citation can also be *wrong about R*. **The frontier is six** — ticket 12 +leaving it, ticket 64 joining — and ticket 64 is now the only one on the route +to the destination. + +**[The documentation overhaul](tickets/64-documentation-overhaul-drop-r-framing.md) +is closed, and with it the map reaches the state its destination describes: the +promotion ticket is genuinely unblocked, with no replacement blocker.** The +session measured its population before editing anything — 353 R references in +`src/`, 88 in `tests/`, 65 in markdown — and the shape of the fix came from +sampling the hardest sites first rather than from the count. Four kinds turned +up, and only one was a real rewrite: pure provenance carrying no information +("like R does", where the actual reason already sat in the same sentence); R +standing in for evidence that now exists; a deliberate divergence documented +against R, which had to be restated as a positive claim; and an R *library* +explaining a Python choice. + +**The exemption was argued, not assumed.** `src/a4d/migration/compare.py` holds +179 of the 353 — more than every other module combined — because R divergence is +literally its subject, and stripping R from it would leave it documenting +nothing. The user chose to mark it historical rather than merely exempt it, so +its module docstring now states that the tool will never run again, that it is +kept because its cause registry is the written evidence behind 56 tickets of +triage, and how to reach the R source. The script itself is untouched. + +**`docs/migration/` became `docs/archive/`, and the archive was corrected before +it was frozen.** Freezing a document while its facts are wrong preserves the +error, so each was brought to the true current state first: `MIGRATION_GUIDE`'s +"Phases 0-9" status became the real final state, `MR_DESCRIPTION`'s "240 commits +/ 664 tests" became 285 / 981, and `PYTHON_IMPROVEMENTS` now says in its own +header that [the completeness audit](tickets/07-pipeline-completeness-audit.md) +found it undercounting. `docs/VALIDATION_SUMMARY.md` was **deleted** rather than +banner-ed: a 174-tracker R-vs-Python verdict superseded by the 254-tracker +comparison is stale as well as R-framed. + +**The guard is a test, not a lint hook**, so CI enforces it without waiting on +[ticket 34](tickets/34-local-ci-parity-guard.md). It derives its file list from +the tree rather than hardcoding one, exempts the migration package and itself, +and assembles its patterns from parts so it cannot match its own source. Proven +non-vacuous by introducing a violation and watching it name the file and line. + +**The one thing the session declined to fix is the sharpest finding in it.** +`extract/product.py` publishes `function_name="read_product_data_step1"` — the +only one of twenty `function_name` values that is not the emitting Python +function — and four sites publish `script="script1"`/`"script3"` into a column +whose default is the stage name `"clean"`. These are **data in the BigQuery +logs table**, not comments, so renaming them changes published output. Recorded +as [ticket 65](tickets/65-logs-table-r-named-values.md) rather than changed +silently, and deliberately not wired as a promotion blocker: ticket 64's gate +was argued on documentation being *wrong*, and this is inconsistency rather than +falsehood. + +**The frontier is seven, and [promoting `migration` into +`dev`](tickets/06-promote-migration-to-dev.md) is on it** — the destination +itself, and the only frontier ticket on the route. The other six are standing +decisions, the source-defect report, a separate feature (ticket 16) and the logs +residue. + +## Decisions so far + +- [Rewrite every docstring and doc that explains the code by what R + did](tickets/64-documentation-overhaul-drop-r-framing.md) -- decided and + implemented. Every module outside `src/a4d/migration/` now explains itself in + its own terms: **`src/` 174 -> 0, `tests/` 46 -> 0, user-facing markdown 65 -> + 0**, with the comparison package's 179 + 42 exempt by decision. The + replacement shape is *what the rule does, the source evidence that forced it, + and what breaks without it* -- so the cm-to-m threshold cites the 120 cells + published as 0.069 metres rather than citing R, and the optional closing + parenthesis cites the 25 of 30 source cells written `180(May-2017`. + `src/a4d/migration/` is **marked historical rather than merely excused**: its + docstring says the tool will never run again, that it is kept because its + cause registry is the evidence behind the migration's conclusions, and how to + reach the R source. `docs/migration/` moved to **`docs/archive/`** with a + README index, and every document was brought to the true current state + *before* being frozen (`MR_DESCRIPTION`'s "240 commits / 664 tests" -> 285 / + 981; `PYTHON_IMPROVEMENTS` now states in its own header that [ticket + 7](tickets/07-pipeline-completeness-audit.md) found it undercounting). + `docs/VALIDATION_SUMMARY.md` **deleted** -- a 174-tracker verdict superseded + by the 254-tracker comparison, stale as well as R-framed. A CI guard + (`tests/test_docs_have_no_r_framing.py`) derives its file list from the tree, + exempts the migration package and itself, and was proven non-vacuous by + introducing a violation. Suite **1,080 passed** (980 + the guard's 100 cases), + 86% coverage. Residue spawned as [ticket + 65](tickets/65-logs-table-r-named-values.md): two values naming R scripts are + **published into the BigQuery logs table**, so they were recorded rather than + silently changed. + +- [Retire R from the workspace once the pipeline is fully verified + Python-only](tickets/12-retire-r-workspace.md) -- decided and executed. + `r-archive/` **deleted outright** (156 tracked files, 1.9M), recoverable via + the annotated tag `r-archive-removed` (`r-archive-removed^` is the last commit + containing it), which is now the recipe `CLAUDE.md` carries in place of its + "R Archive / do not modify" section. `docs/CLAUDE.md`, `SETUP.md`, + `pyproject.toml`'s ruff `extend-exclude`, `.dockerignore` and `.gitignore` + all updated; the last three were absent from the ticket's own inventory and + were found by re-deriving it. Inlining the cited R excerpts first was + **rejected on measurement**: the citation count is **35 across 12 modules**, + not the 8 in `compare.py` the ticket claimed (its grep matched only the + literal path `r-archive`, missing every bare `script2_*.R` citation), so + inlining meant editing production cleaning code at 35 sites to entrench a + framing that is being removed. Moving the archive to its own repo was + rejected as a second copy that can rot. Suite green (980 passed, 1 skipped, + 86% coverage), ruff and `ty check src/` clean with the exclusion gone. + Spawned [ticket 64](tickets/64-documentation-overhaul-drop-r-framing.md) for + the 35 citations, and wired it as the sole remaining blocker of [ticket + 6](tickets/06-promote-migration-to-dev.md). + +- [Monthly rows with a misspelled ID silently lose their Patient List + demographics](tickets/58-patient-list-join-uses-unfixed-id.md) -- decided and + implemented. Both extraction-side joins (`Patient List` and `Annual`) now key + on a derived normalized ID; the raw `patient_id` column is unchanged. + **680 rows / 6,863 Patient List cells recovered across 6 trackers**, 672 of + them the 2023/2024 Mahosot hyphen spelling. R has the same gap and the + comparison could never show it. New cause + `r_static_join_misses_respelled_id` (raw stage only, 5,959 cells, raw + `unclassified` -> 0); the cleaned-stage half is [ticket + 63](tickets/63-name-the-cleaned-stage-static-join-divergence.md). + +- [The cleaned stage has 4,949 cells with no cause, because the ID spelling that + explains them is gone by + then](tickets/63-name-the-cleaned-stage-static-join-divergence.md) -- decided + and implemented. The surviving discriminator is derived per file from the + run's own **raw** output (`static_join_missed_ids`), comparison-side only, so + production output is untouched. The ticket's own figure was understated: + measured, cleaned `unclassified` was **4,949, not 3,593**, and it is now + **16** -- the pre-session baseline -- with raw still 0 and the four-stage + totals unchanged. Two guards were needed before the cause was true: a column + R lost **file-wide** is `r_extraction_gap`'s `xml:space` defect, not this one + (2024 Preah Kossamak reads zero `recruitment_date` in 863 rows against 2024 + Mahosot's 672 of 1,068), which kept 2,336 cells off the wrong label; and + `age` is *derived*, so it got its own cause + `r_age_not_derived_without_dob` (113 cells) keyed on the row's recovered + D.O.B. **Four existing causes were silently holding join-miss cells** and + handed back 671 + 677 + 224 + 224 + 27 -- each now matching the raw stage's + count for the same column cell-for-cell. + +- [Audit the eight pre-bar causes the first classifier pass did not + reach](tickets/60-audit-remaining-pre-bar-classifiers.md) -- decided and + implemented. Six of eight audited, Python correct in all six. + **`buddhist_era_typo` was naming twelve cells that are not Buddhist-era + dates**: its whole test was `year >= 2400`, which since [ticket + 61](tickets/61-decide-buddhist-era-date-conversion.md) converts genuine BE + dates is exactly backwards. Both populations are ordinary year typos verified + in the workbook (`3035-03-01`, 2025 CDA `Mar25!O219`; `5025-05-19`, 2025 Surat + Thani `May25!O70`). Bounding it exposed a *second* false label -- five rows + fell to `r_parse_order_cannot_read_cell`, which asserts R's parse orders + failed when R had simply carried the Excel serial `1141523` through as a + string -- so ticket 32's `python_absurd_excel_serial` is now wired to every + patient date column, with its Buddhist-era band reading the sheet name + because no patient stage sets `tracker_year_col`. Measured before and after + across all four stages: **exactly four per-cause entries moved**, and + `unclassified` held at 37, so all twelve went from a false label to a true + one. Two docstrings corrected without moving a count: `r_extraction_gap` + spans **thirteen** columns against the six mechanisms it documented (three + more traced to source -- a blank `Q85` header costing 194 2021 Kantha Bopha + `insulin_regimen` rows, a blank row number on the 2026 `Annual` sheet costing + 7, and Thai text in a Nakornping header costing 42 -- and the second of those + **corrects an explanation already in the registry**, which blamed R for + reading nothing from a sheet it reads 102 of 120 rows from), and + `r_validator_rejects_multivalue` covers **two** R defects, the documented CSV + rejection plus NA propagation through `ifelse`/`paste` that rejects 575 + single-tick rows. `row_order_divergence`'s verdict written, not re-evidenced; + `excel_formula_error` confirmed 100% one-directional and raw-only. + `r_category_lookup_miss`, `openpyxl_date_typed_stray_cell` and + `r_extraction_gap`'s 98% bulk not reached -- split to [ticket + 62](tickets/62-finish-the-pre-bar-classifier-audit.md). + +- [Rows that pair with nothing on the other side, which no ticket has ever + triaged](tickets/59-triage-unmatched-row-keys.md) -- decided and implemented. + The 144 cleaned / 130 raw unmatched rows were **five populations, not one**, + and Python is the correct side of all five. A Python defect fixed: a cleared + patient row number leaves a whitespace-only cell in column A, which pushed + `find_data_start_row` one row past the data, made the first patient's row + serve as the header row, and cost `2022_Children's Hospital 2` its **entire + `Oct22` sheet** -- 15 patients' records. The rule was chosen by scanning all + 254 trackers, which killed the obvious fix (R's "first non-empty cell" would + start 14 sheets at row 1 on a stray letter); the narrow abutting-cell rule + moves exactly one sheet in the corpus. An R defect documented, not fixed: R's + hardcoded `+1` offset for 2022+ trackers assumes one leading empty row, and + every `2024_Mandalay Children's` sheet but `Jan24` holds a space there, so R + drops that tracker's first patient 11 times. And a source defect with a + behaviour decision the user made: 120 rows across 9 trackers carry `#REF!` + where the patient ID belongs and were being dropped **silently** -- they stay + dropped (`Undefined` is a bucket, not an identity, so keeping them would pool + 98 people's measurements under one group key) but now emit the new + `excel_error_patient_id` code. Verified end-to-end on the real 254-tracker + set: row-key divergence 144 -> 129 and 130 -> 115, cell mismatches +22 onto + existing causes, `unclassified` unmoved on all four stages. + +- [Decide whether a Thai clinic's Buddhist-era entry date is published as 2567 + or converted to 2024](tickets/61-decide-buddhist-era-date-conversion.md) -- + decided and implemented. Converted, in the cleaned stage on both arms, under + a new `buddhist_era_converted` error code so the shift is auditable; raw + keeps what the workbook says. 375 patient cells that `_validate_dates` was + destroying are recovered, and 22 product rows no longer publish a stock + movement dated 543 years ahead. Patient converts on the upper bound alone + (a diagnosis date may predate its tracker by decades), product inside the + band it already computed; 6 cells that decode to no plausible year stay + sentinelled, and the two `hospitalisation_date` cells naming a date *range* + stay out. Two comparison causes added, no new unclassified cells. + +- [Decide whether a date buried inside a clinical note should be recovered or + discarded](tickets/39-recover-dates-embedded-in-free-text.md) -- decided and + implemented. Recovered, by an explicit anchored pattern set that refuses what + it does not recognise; a note naming several dates publishes the first and + reports the discard; an absent day becomes the 1st and an absent year comes + from the tracker; everything else keeps the sentinel. Wired into + `parse_date_flexible` for all 18 date columns, with three new warning codes + (`date_recovered_from_text`, `date_multiple_in_cell`, `date_year_inferred`). + `hospitalisation_date`: 673 sentinels -> 156, and 478 -> **0** unclassified + R/Python mismatches. Also fixed a pre-existing misreading in which a bare + range was dated from its first number as a year (`6-12 Nov 2020` published as + 2006-11-12). + +- [Four trackers where cleaning merges several patients into one patient + ID](tickets/47-patient-ids-merged-at-cleaning.md) -- decided and implemented. + The mechanism was `fix_patient_id`'s R-inherited truncation, not the + preprocessing regex the premise suspected: a non-conforming ID over 8 + characters was cut to its first 8, so `2023_NPH`'s `KH_NPH026`-`KH_NPH029` + became one `KH_NPH02` -- an identifier present in **no** source workbook, + holding four different people's September records. The workbook is defective + (only its `Sep23` sheet carries the stray `H`; its Patient List and three + other month sheets spell the same four patients correctly), but truncation + turned a recoverable typo into a false merge. Truncation is now dropped: a + malformed ID is **recovered** against the well-formed IDs the same tracker + carries -- edit distance 1, unique candidate only -- and **sentinelled** + otherwise, so the pipeline never publishes an identity no tracker contains. + Measured by re-cleaning all 254 raw parquets: exactly one file changes, the + four patients each gain their September row, and zero non-conforming IDs + remain corpus-wide. Ticket 45's "4 files, 9 identities" turned out to be + three different mechanisms, two of them the pipeline working correctly + (hyphen-vs-underscore spellings of the same patient in Mahosot DC and Surat + Thani) and one a source defect nothing can repair (`2026_NOGH`'s `MM_NO97/98/ + 99`, spelled that way in its own Patient List, with no `MM_NO097` to recover + to -- three patients still sharing `Undefined`). Five source-defect findings + went to [ticket 40](tickets/40-source-defect-findings-report.md). Spawned + [ticket 58](tickets/58-patient-list-join-uses-unfixed-id.md): extraction + joins the Patient List on the *unfixed* ID, so the recovered rows have their + identity back but not their demographics. The comparison was re-run against + the regenerated output: the divergence lands entirely as **row-key + non-overlap** (7 files/125/19, up from 6/121/15, all of it `2023_NPH`'s 4+4) + rather than as cell mismatches, which fell 114,373 -> 114,371. + +- [Triage the residual patient cleaned-stage mismatches (round + 10)](tickets/57-triage-patient-cleaned-residual-10.md) -- the in-scope cleaned + residual falls 96 -> **19** (80%), raw stays byte-identical at 26,172, product + untouched, and **the ten-round chain ends here**: all 19 carry written + verdicts (13 the standing `Undefined` fog, 6 a named tooling limitation), so + there is no round 11. One real Python defect, found by disproving this + ticket's own premise: `_extract_date_from_measurement` existed all along, but + R makes the closing parenthesis optional (`[)]?`, and R's own test suite + covers an unclosed cell) where Python required it -- so `180(May-2017`, which + is how 25 of that group's 30 source cells are written, lost its date + entirely. 24 cells recovered. Matching R *exactly* was measured and rejected: + R's greedy prefix leaves `196(` as the value and then fails its own numeric + cast, so Python strips the stray `(` and keeps the 196 R throws away. The two + FBG cells the ticket suspected of being wrongly sentinelled are correctly + sentinelled -- 2020 Kantha Bopha's baseline column is 90.1% sub-30, so it is + read as mmol/L and 47.8 is past the mmol ceiling, not inside the mg/dL range + the header implies. The remaining 37 date cells are all source defects (R + invents a reading from a damaged token; its answer for `10/1023` is not even + stable between its frozen output and its own parser), reported to ticket 40. + Four classifiers added; a fifth was written, measured and backed out. + +- [Triage the residual patient cleaned-stage mismatches (round + 9)](tickets/56-triage-patient-cleaned-residual-9.md) -- the in-scope cleaned + residual falls 448 -> **96** (79%), the raw stage stays byte-identical at 0 + and product is untouched. The nine-column date family turned out to be **one + R mechanism, not nine causes**, and it was established by installing + lubridate and *executing* R rather than by reading it. `parse_dates` deletes + the fourth letter of any word of four or more letters -- `April-17` becomes + `Aprl-17` -- and then walks a fixed order list ending in `my` and `y`, so a + spelled-out month collapses to 1 January of its year (`r_month_name_truncated + _to_year`, 88), an unreadable month makes R read the **day** as the month + (`9-Dce-20` -> 2020-09-01, `r_reads_day_as_month`, 143), and a day past 12 + leaves R with no reading at all (`r_parse_order_cannot_read_cell`, 107). + Every reproduction matched R's frozen output. Python is the correct side of + all three. The same investigation exposed three Python gaps, all real data + loss and all fixed: month spellings Python did not know (the Bahasa Malaysia + `Mac`/`Mei`/`Okt`, all twelve Thai abbreviations, plus `Dce` and `ug`); + separator runs damaged by a stray keystroke (`26-05- 2007`, `19-Jan_2023`, + `02-Apr=-2026`), repaired on a second attempt only and only when at most + three numbers remain -- a guard measurement forced, after the first version + turned the *range* `11-15 /01/2019` into 2001-11-15; and zero-width + characters pasted in from another application. Checked the way round 6 + checked its parser change: old against new over all **5,187** distinct raw + date strings, 32 changed, every one from the sentinel to a real date and no + already-parsing value altered. `insulin_subtype`'s 11 cells were traced to + their mechanism and deliberately left unclassified -- R's null-vs-`Undefined` + split is `ifelse` NA propagation, not a designed distinction, which makes + them the standing fog entry on unticked insulin rows rather than a residual. + What is left became [round + 10](tickets/57-triage-patient-cleaned-residual-10.md). + +- [Triage the residual patient cleaned-stage mismatches (round + 8)](tickets/55-triage-patient-cleaned-residual-8.md) -- the in-scope cleaned + residual falls 770 -> **448** (42%) on three Python defects fixed and one + R limitation explained, with the raw stage holding at 0. `height`'s cm-to-m + threshold was 2.3 where R's is 50, so a source value between the two -- 120 + cells reading `2.43`, `6.9`, `13.0` -- was divided by 100 and published as + `0.069` metres instead of being rejected; and BMI was derived before range + validation, so it came from that impossible height (`60 / 2.43^2` = 10.16) + and passed its own bound. Both fixed, both were live in production. + `fbg_updated_mg` turned out to have no Python defect at all: R's `fix_fbg` + matches its CDC category words as substrings, so `Lost follow up` becomes a + glucose of **140** ("fol-low" contains "low") and every `SMBG 50-HI` / + `129-HI` / bare `HI` becomes **200**, discarding the number the clinic wrote; + Python already anchors the same patterns to the whole string and sentinels + instead, and separately reads `148 mg/dl (Mar-18)` where R cannot. + `insulin_subtype` recovered 56 rows where 2024 Sarawak ticks the insulin + boxes by writing `Novorapid`/`Glargine`/`Toujeo`/`Ryzodeg`, which both + pipelines were discarding. And a third defect surfaced in the date family: + `parse_date_flexible` accepted a year with a digit missing, so `1/16/224` and + `13-Mar-0202` reached production as dates in antiquity -- now floored at + 1900, mirroring the existing guard at the other end of the calendar. What is + left is almost entirely the date family and became [round + 9](tickets/56-triage-patient-cleaned-residual-9.md). + +- [Triage the residual patient raw-stage column mismatches (round + 6)](tickets/50-triage-patient-raw-residual-6.md) -- the patient raw stage + reaches **zero** unclassified (84 -> 0), on six mechanisms and no Python + defect. Three header defects R cannot survive and Python does, all + `r_extraction_gap` (38 rows): a hospitalisation header opening with thirteen + spaces, two sheets whose header merges have been deleted so "Date" is no + longer qualified, and a column unheaded in both header rows. A date typed into + a numeric column (20), wired to ticket 24's existing + `openpyxl_date_typed_stray_cell` -- each serial decodes to exactly Python's + value. R keeping only the first selection of a merged multi-select block (2), + the new `r_duplicate_header_selection_dropped`, proven against R's own junk + column. And two harness defects fixed in `parse_date_flexible`: the month-name + truncation needed a word boundary a digit cannot give, so `July2014` (readxl + drops the rich-text space) had its day filled **from today** -- the comparison + was non-deterministic across run dates (11); and the Excel-serial ceiling + excluded Buddhist-Era serials, so `241062` was read as 24/10/62 rather than as + 2560-01-01 (12, plus 363 rows `buddhist_era_typo` had been absorbing). The + ticket's own load-bearing question -- whether Python stamps the date sentinel + at the raw stage -- was answered no: Python's raw holds the verbatim + `on stamlor 5mg`, and the `9999-09-09` was the harness's own normalization. + Six source defects reported to [ticket + 40](tickets/40-source-defect-findings-report.md); cleaned-stage residual + spawned as [ticket 51](tickets/51-triage-patient-cleaned-residual-4.md). +- [Python drops complication-screening results and dates where a merged header + spans the block](tickets/48-putrajaya-screening-columns-lost.md) -- a merged + upper header now names every column its span covers, but **only where that + column has a sub-header of its own to qualify**. Two independent mechanisms + were damaging the same block: forward-fill resets on a column blank in both + header rows, so a merged title never reached the sub-headers past the gap + (leaving `Results`/`Date (mmm-yy)` bare, though the synonym map already knew + their qualified names); and ticket 30's sibling-donor recovery then filed a + screening selection under `observations`, because Putrajaya's month sheets do + **not** share one layout. The sweep that sized it (693 sheets, 64 files) + killed the naive version twice: propagating the bare title into columns with + no sub-header would have created 290 sheets' worth of standard-name + collisions on `complication_screening`, and comma-joined 3,659 near-duplicate + `Insulin Regimen` cells that R and Python already agree on. Measured against + the real drive data: `complication_screening_results` 11 -> 0, + `complication_screening_date` 31 -> 0, patient raw unclassified **278 -> 229**. +- [Triage the residual patient raw-stage column mismatches (round + 4)](tickets/46-triage-patient-raw-residual-4.md) -- four causes settled, one + of them a Python bug that was destroying real readings in production: + `read_patient_rows` now converts a pre-1903 datetime back to its Excel + serial, because Excel's 1899-12-30 epoch makes such a "date" a small number + that inherited a date format (24 systolic readings were reaching BigQuery as + 999999). The `8.8(20.9.16)` space ticket 43 left unexplained is explained: + readxl drops a whitespace-only **rich-text run**, found in two workbooks' + own `sharedStrings.xml`, so it is now `r_drops_richtext_space` (89) with + `python_trims_merged_subvalue` (2) for the opposite direction. Kantha + Bopha's blank-header `insulin_regimen` is wired to `r_extraction_gap` (194) + after re-verifying the header at source. Raw unclassified 601 -> 278, + cleaned 8,383 -> 8,148. Spawned [ticket + 48](tickets/48-putrajaya-screening-columns-lost.md) (a second, unrelated + Python data loss) and [round 5](tickets/49-triage-patient-raw-residual-5.md). +- [Give the patient comparison an ordinal row + key](tickets/45-patient-row-alignment-duplicate-keys.md) -- patient keeps + `patient_id` + `sheet_name` as its identity key and gains a tie-break inside + it, rather than adopting product's positional key anywhere: the key is unique + in 249 of 254 files, so `add_row_ordinal` is grouped by the identity key + itself. The tie is broken by **content, not position** -- measured, since + occurrence order is optimal at raw but pairs the wrong copies in 9 cleaned + groups. Both patient stages, since the duplicate files differ by stage (3 raw, + 5 cleaned). Raw unclassified 1,409 -> 601, cleaned 9,392 -> 8,383, + `row_key_overlap` unchanged, product untouched. Surfaced a production bug + neither pipeline's disagreement could reveal ([ticket + 47](tickets/47-patient-ids-merged-at-cleaning.md)) and two source-defect + findings for [ticket 40](tickets/40-source-defect-findings-report.md). +- [Triage the residual patient raw-stage column mismatches (round + 3)](tickets/43-triage-patient-raw-residual-3.md) -- two causes settled, + patient raw-stage unclassified 1,879 -> 1,409. 434 rows were a + float-to-string rounding artifact reaching four columns the normalize list + could not name, because it derived from the *cleaned* schema (no entry for + the Patient List join's `.static` copies; screening measurements typed as + strings) -- fixed by deriving from each raw frame's own columns. 24 rows + were a real Python defect: `read_patient_rows` turned a bare list of + patient IDs left below `2024_Vietnam National Children`'s `Jul24` data + block into invented monthly records. A 254-tracker sweep proved adopting + R's stricter rule would have lost a genuine record (`2024_Mahosot` + `LA-MH088`), so the fix keeps unnumbered rows that carry actual data. The + 808-row duplicate-key fan-out became [ticket + 45](tickets/45-patient-row-alignment-duplicate-keys.md) and the long tail + [ticket 46](tickets/46-triage-patient-raw-residual-4.md). +- [Decide how FBG unit headers are resolved, and what to do about + physiologically implausible mmol values](tickets/42-fbg-unit-headers-and-implausible-values.md) + -- decided and implemented on A4D's medical advisor's reply (2026-08-17), + which gave analytical limits rather than confirming the ticket's "below 30 + mg/dL is impossible" premise. Unit confusion is per column, not per row, so + the pipeline corrects a wholly mmol-recorded column (29 columns, 21 files) and + only flags a stray low reading (5,673 rows), since a severe hypo and a + mis-entered unit look identical. All four FBG columns now carry the advisor's + analytical bounds; 832 readings rejected. Divergence from R classified by + `python_glucose_unit_corrected`; the R-null residual is [ticket + 44](tickets/44-triage-cleaned-fbg-r-null-residual.md). + +- [Triage the residual patient raw-stage column mismatches (round + 2)](tickets/31-triage-patient-raw-residual-2.md) -- decided and implemented. + `complication_screening`'s 12,566 mismatches were two stacked causes. A real + Python data loss: `ColumnMapper.rename_columns` kept only the first of + several source columns sharing a canonical name, discarding 2,489 recorded + values across 27 trackers (the 2023 template's B.P./Kidney/Eye/Foot/Lipids + screening sub-columns); it now merges them like R's `tidyr::unite` and the + pipeline's own raw-header merge. And an R rendering artifact: R's `unite` + pads absent sub-columns with the literal string `NA`, now the + source-verified `r_na_unite_padding` classifier. Patient raw-stage + unclassified 14,903 -> 1,879 (-87%). Residual spawned as [ticket + 43](tickets/43-triage-patient-raw-residual-3.md). + +- [Triage the patient pipeline's raw-stage column-existence + divergence](tickets/30-triage-patient-raw-column-divergence.md) -- decided + and implemented. R's `make.names(header_cols, unique = TRUE)` renders a blank + header cell as the *string* `"NA."`, so R's own "delete columns without a + header" line never fires and every spacer column survives as `na`, `na1`, ... + `na10064` (17,132 rows, 94% of the stage's divergence) -- verified by running + R, not by reading it. Python drops blank-header columns and correctly has no + equivalent. A further 154 rows are the same column under two name spellings + (R sanitizes, Python keeps the literal source header), now both named by the + new `classify_column_divergence` and shown in a `cause` column on the + `column_divergence` sheet. **Found on the way, and the more important half:** + dropping blank-header columns silently loses real data where a clinician left + a header cell empty -- 194 insulin-regimen rows in `2021_Kantha Bopha`, + confirmed against the source workbook. Not fixed by inference (see Rejected); + instead `find_dropped_data_columns` reports every site under the new + `blank_header_with_data` error code, and `recover_blank_headers` names the + column from the sibling month sheets that do label it -- unanimous donors + only, blanks only, never overriding a stated header. 17 sites across 5 + trackers recovered (the 194 Kantha Bopha rows among them, `insulin_regimen` + 1,273 -> 1,467 non-null on that file), 201 left reported for source + correction. Of the 4,572 values the full-tracker sweep found, most are the + 2022 template's hidden merged-cell column that Python is right to drop, + measured rather than assumed. Spawned [ticket + 40](tickets/40-source-defect-findings-report.md) and [ticket + 41](tickets/41-decide-2026-new-patient-list-columns.md). + + +- [Triage the patient cleaned-stage date-column family (round + 3)](tickets/38-triage-patient-cleaned-date-family.md) -- decided and + implemented. The date sentinel was being stamped on cells that recorded an + **absence**: `parse_date_flexible` recognized four missing markers where the + numeric path's `safe_convert_column` already recognized eleven, so `-`, `.`, + `N/A`, `Nil`, `Unknown`, `?` and the tracker template's own leftover + instruction text ("Insert Date or NA") all became 9999-09-09 -- the claim "a + date was recorded and it is invalid" about a cell that recorded nothing. The + marker list is now declared once in `clean/date_parser.py` and **shared** + with `converters.py`, which had been carrying the copy that drifted; two + date-scoped sets were added beside it for absence-written-as-a-word and for + the template placeholder. R's side of the same shape became the + `r_date_error_sentinel` cause, wired across every column `get_date_columns()` + returns, with Python the verified-correct side: the source template's own + sub-header reads "(Insert Date or NA)". Sentinel-stamped date cells in + Python's own output 6,186 -> **1,994 (-68%)**, of which the + unrecognized-marker share went 4,377 -> **0**; cleaned-stage unclassified + 8,259 -> **6,652**. Also settled, by measurement rather than judgement: + patient's row-alignment key stays an identity key (84 of 85,325 rows, 0.10%, + sit on a duplicated `patient_id` + `sheet_name`) -- which settles [ticket + 31](tickets/31-triage-patient-raw-residual-2.md)'s copy of the question too. + `t1d_diagnosis_date`'s largest population is a **corrupt source, not a + pipeline difference**: one 2022-named, 2022-sheeted tracker whose Patient + List holds 2023 diagnosis dates for all 43 patients, internally consistent + with its own D.O.B. and age columns. Spawned [ticket + 39](tickets/39-recover-dates-embedded-in-free-text.md). + +- [Triage the residual patient cleaned-stage mismatches (round + 2)](tickets/37-triage-patient-cleaned-residual-2.md) -- decided and + implemented, four real Python bugs fixed: the 2022 template's + `Updated 2022` header left `blood_pressure_updated`/`edu_occ_updated` + unmapped on every 2022 tracker (7,165 values recovered); the cleaned + stage's time-stripping split on the first space and destroyed genuine + space-separated dates like `"Jun 2006"`, which dateutil then completed from + *today*, making the output non-deterministic and sentinelling real dates; + `parse_date_flexible`'s month-name truncation dropped only the fourth + letter (`"March"` -> `"Marh"`), breaking every full month name; and its + month-year branch handled only 2-digit years. Three causes named: + `r_extraction_gap` extended over two verified R mechanisms (R's own 2022 + fixup defeated by a leading space in the source cell, and R reading nothing + from the 2026 template's new `Annual` sheet), the new + `r_ifelse_na_propagation` (R's `FALSE | FALSE | NA` is NA, so a row whose + analog-insulin columns plainly read Y loses its `insulin_type`), and + `buddhist_era_typo` made symmetric. Unclassified 16,698 -> 8,259 (-51%). + 13 rows left deliberately unclassified as a duplicate-row-key alignment + signal rather than given a false cause. Residual split into [ticket + 38](tickets/38-triage-patient-cleaned-date-family.md). Same session, after + closure: the tracker set was refreshed to 254 files and the frozen R + baseline renamed, `file_name` column included, to keep the comparison + pairing -- see the ticket's addendum. + +- [Triage the residual patient cleaned-stage column + mismatches](tickets/29-triage-patient-cleaned-residual.md) — decided: + `insulin_total_units` is deleted by R's case-sensitive insulin-dedup grep + before R ever reads it; the baseline-FBG columns vanish because R's + Patient List join suffixes *both* sides on a name collision; R's 999999 + sentinel marks cells that never held a usable number (all 8,085 traced to + source, none a real value Python dropped). Two live Python bugs fixed: + `extract_regimen`'s column-wide lowercasing and + `validate_allowed_values`'s last-wins config lookup. Unclassified rows + 55,670 -> 16,698 (-70%) against the real 248-tracker pair. Also settled the + canonical-label question the source (not R) decides: one label per status, + aliases declared in `validation_rules.yaml`, collisions now a loud config + error. Residual split into [ticket + 37](tickets/37-triage-patient-cleaned-residual-2.md). + +- [Diagnose and fix why CI is red at migration HEAD](tickets/04-fix-migration-ci.md) + — decided and implemented: CI was red because GitHub Actions sets + `GITHUB_ACTIONS=true` for every job, and Typer reads that at import time + to force full-color `--help` rendering, which splits options like + `--file` into separate ANSI spans and breaks the tests' plain substring + assertions — regardless of the `NO_COLOR`/`COLUMNS` overrides the tests + already set. The original lead (`cli.py`'s module-level `console` object) + was investigated and ruled out: Rich re-reads `COLUMNS` live rather than + caching it, and `--help` is rendered by Typer's own internal console, not + `cli.py`'s. Fix: `tests/conftest.py` on `product-pipeline` sets Typer's + own `_TYPER_FORCE_DISABLE_TERMINAL` escape hatch before `typer.rich_utils` + is first imported. Pushed as `b970cf6`; PR #6's CI is green + (all 428 tests), coverage gate holds, PR #6 is `mergeable: MERGEABLE`. + Full detail: [ticket 4](tickets/04-fix-migration-ci.md). + +- [Does product-pipeline's test suite meet the same cell-by-cell rigor as + patient's?](tickets/01-product-pipeline-test-rigor.md) — superseded: R-parity + isn't the goal (source trackers are the arbiter, not R's output); + R-vs-Python comparison is analysis, not pytest; neither pipeline's + completeness has actually been audited yet. Split into tickets 7 and 8; + ticket 2 now owns the analysis-report side. +- [Is the product pipeline (and patient's own claimed completeness) actually + complete and sound, audited against R's product logic and patient's + structure?](tickets/07-pipeline-completeness-audit.md) — decided: R-logic + coverage is essentially complete function-for-function on both pipelines + (one diagnostic-only R gap on product); product has real structural test + gaps (no integration/e2e tests, no `wide_format.py` coverage, no + `test_tables/test_product.py`, group-granularity-only source-vs-output + check); patient's "174 trackers validated" claim has genuine test + infrastructure but no committed record of an actual passing run (CI + excludes it, USB-drive-gated). Full inventory and gap lists: + [research/07-pipeline-completeness-audit.md](research/07-pipeline-completeness-audit.md). +- [Does the pytest suite reach unit/integration/e2e/regression parity between + patient and product, excluding any R-comparison/USB-drive-dependent + tests?](tickets/08-pytest-suite-parity.md) — decided: parity = 85%+ coverage + enforced in CI, product gets the three missing integration/e2e files built + on patient's existing fixture/skip-if-missing convention plus + `test_tables/test_product.py`; `test_r_validation.py` leaves pytest + entirely with no product equivalent. "Regression test" was reframed + mid-session to mean golden-master/snapshot testing (not R-comparison or + edge-case testing) and split off into + [Add golden-master/snapshot regression tests for patient and + product](tickets/09-snapshot-regression-tests.md), deferred until both + pipelines' other test suites are green. +- [Merge product-pipeline (PR #6) into migration](tickets/03-merge-product-pipeline.md) + — decided and implemented: ticket 8's tests written (85%+ coverage + narrowed to product-only code, per a mid-session user correction — + repo-wide coverage was 73%, not close to 85%, and most of the gap is + patient/CLI code unrelated to product parity), `test_r_validation.py` + removed, repo-wide ruff/ty cleanup (blocking, since CI runs both + unscoped), an implementation review that found and fixed two real + logging-parity gaps (product never populated `TrackerResult.data_errors` + or created a logs/errors table, unlike patient — both now mirror patient + exactly), and doc alignment verified with no changes needed. PR #6's + merge conflicts resolved; it is now `mergeable: MERGEABLE`. CI still + fails on 7 `--help`-rendering tests unrelated to product code — handed to + ticket 4 with a concrete lead rather than fixed here. + +- [Define and execute the real GCP production verification run](tickets/05-production-verification-run.md) + — decided and executed: ran the combined patient + product pipeline for the + first time as one execution, via the existing `a4d-pipeline` Cloud Run Job + against real production GCS/BigQuery (execution `a4d-pipeline-8mxls`, + succeeded). Preceded by a `just backup-bq` snapshot as the rollback point. + Verified with a new script (`scripts/verify_production_run.py` + + `src/a4d/gcp/verify.py`, unit-tested) comparing row counts, distinct clinic + counts, and schema against that snapshot — not against R, which the user + decided has no role in this ticket's check (R-vs-source comparison stays + ticket 2's job). All four tables grew cleanly (51 -> 53 clinics), no + anomalies. Confirmed along the way that Cloud Scheduler isn't enabled on the + project yet. Full detail: [ticket 5](tickets/05-production-verification-run.md). + +- [Audit and update all dependencies and library versions before + rollout](tickets/13-dependency-audit.md) — decided and implemented: + `uv lock --upgrade` moved every dependency to current latest (three + majors — `pandera` 0.26->0.32, `pytest` 8->9, `typer` 0.19->0.27, `rich` + came along transitively 14->15); resolved all 19 known vulnerabilities + `pip-audit` found in the prior lock; `Dockerfile`'s `python:3.14-slim` + floating tag checked and already current, no change needed. `ty` (dev + type checker) jumped 0.0.1a23 -> 0.0.69 and surfaced two real `src/` + gaps the old alpha missed — `gcp/storage.py`'s unguarded `blob.name` + (`str | None` per stubs) and `validate/common.py`'s `emit_finding` + `error_code` param (typed `str` behind a dead mypy-style ignore `ty` + never honored) — both fixed. Full suite (488 tests), ruff, `ty check + src/`, and the product-only coverage gate all pass; pushed as `c4721ad`. + This unblocks [ticket 10](tickets/10-performance-profiling.md) (its + other blocker, ticket 3, was already closed). Full detail: [ticket + 13](tickets/13-dependency-audit.md). + +- [Profile the combined pipeline's performance against the R baseline before + rollout](tickets/10-performance-profiling.md) — decided and executed: R + comparison dropped (already known slower); reframed as a `pyinstrument` + function-level profile of the Python pipeline against the full 177-tracker + real dataset. Found and fixed an O(n^2) bug in `find_data_start_row` + (per-row `.cell()` calls each re-parse a read-only worksheet's XML from + row 1) — 6.6x speedup on the full patient arm (145.8s -> 22.0s), pushed as + `97479f8` with a regression test. Full combined run: 73.86s wall, ~556MB + peak RSS. Also surfaced 4 product trackers failing outright, spawned as + [ticket 14](tickets/14-product-column-detection-failures.md). Full detail: + [ticket 10](tickets/10-performance-profiling.md). + +- [Fix product pipeline's "unable to find column product" failures on 4 real + trackers](tickets/14-product-column-detection-failures.md) — decided and + implemented: confirmed legitimate (pre-product-tracking tracker years for + 2 clinics, verified directly against the real files), not a synonym gap or + regression; fixed a real bug where `clean_product_data` crashed on the + columnless raw frame extraction correctly produces for these trackers, + instead returning an empty schema-conformant result. Full detail: [ticket + 14](tickets/14-product-column-detection-failures.md). + +- [Retire the PDF/notebook analysis docs for an automated, script-based + report](tickets/02-documentation-strategy.md) — decided: comparison baseline + is the already-frozen `output_r/` on the USB drive (no R re-run needed, + ever); the comparison script is decoupled from pipeline execution, runs + four layered checks (shape, totals, columns, cell-by-cell) with an + extensible cause-classifier registry, outputs an HTML report, and lives at + `scripts/` + a `just` recipe (not `a4d.cli`, since it's migration-only + tooling). The unrelated dashboarding PDF was deleted; the parity- + presentation PDF is kept until the new script supersedes it; + `PYTHON_IMPROVEMENTS.md`'s dead notebook citation was fixed. + `test_r_validation.py` is already gone from pytest. Building the script and + triaging the actual flagged differences was deferred to [ticket + 15](tickets/15-build-and-run-comparison-script.md), which now also carries + the `blocked_by` role ticket 2 used to hold on [ticket + 6](tickets/06-promote-migration-to-dev.md) and [ticket + 12](tickets/12-retire-r-workspace.md). Full detail: [ticket + 2](tickets/02-documentation-strategy.md). + +- [Decide what CLI/TUI UX and error-log observability improvements + admins/developers need before rollout](tickets/11-cli-ux-observability.md) + — decided and implemented (CLI/UX half only): `run-pipeline` never rendered + the rich per-arm summary tables `process-patient`/`process-product` already + had, and aborted the entire run on any single patient tracker failure + before the product arm even ran. Fixed by adding a combined patient+product + run summary (both ok / patient-only failed / product-only failed / lost + entirely, plus merged per-file error counts) and switching the patient arm + to soft-fail-and-continue like product already does. Also fixed a + test-isolation bug the soft-fail change exposed: two tests weren't mocking + `run_product_pipeline` and were silently processing real local tracker + files. Pushed as `33694b4`. The observability half (per-file drill-down) + split off into [ticket 16](tickets/16-log-analyzer-drill-down.md). Full + detail: [ticket 11](tickets/11-cli-ux-observability.md). + +- [Build and run the R/Python output comparison script, then triage every + flagged difference](tickets/15-build-and-run-comparison-script.md) — + decided and partially executed: built `src/a4d/migration/compare.py` (four + unit-tested layers + seeded cause classifier + HTML report renderer) and + `scripts/compare_outputs.py` + `just compare-outputs` per ticket 2's + design; ran a fresh Python pipeline pass against `a4dphase2_upload` onto + the USB drive (174/174 both arms) and diffed it against the frozen + `output_r/`. Found patient's row-alignment key sound but product's broken + (null `product_entry_date` collapses the join key, causing fan-out + inflation and hiding `product_entry_date` from classification entirely). + Split the fix + actual triage into [ticket + 17](tickets/17-fix-product-row-alignment-and-triage.md). Same session, + after closure: tool substantially hardened (HTML dropped for Excel, + `compare_id_overlap`/`compare_categorical_overlap`/`compare_row_key_overlap` + added, raw-vs-cleaned staging, consistent "X divergence" naming) — see the + ticket's addendum. Full detail: + [ticket 15](tickets/15-build-and-run-comparison-script.md). + +- [Persist comparison run history and show run-over-run + deltas](tickets/19-compare-run-history-deltas.md) — decided and + implemented: `snapshot_from_summary()`/`compute_deltas()` reduce/diff + `build_summary_rows()`'s counts; the CLI persists a timestamped JSON + snapshot per stage on every run and prints + writes an Excel delta against + the most recent prior one. Validated against three live runs on the real + drive data. Supports ticket 18's triage rather than being part of it. Full + detail: [ticket 19](tickets/19-compare-run-history-deltas.md). + +- [Fix the product comparison's row-alignment key, then triage every flagged + R/Python difference](tickets/17-fix-product-row-alignment-and-triage.md) — + decided and implemented (row-alignment-key half only): replaced the broken + equi-join key with `add_row_ordinal()`'s ordinal-position-within-group key, + computed at comparison time rather than stored (R's frozen baseline can't + be re-run). Validated directly: cleaned-product row-key match jumped from + near-0% to 97.2%; the remaining 2.8% is a single clinic where R's frozen + output has a `clinic_id` typo (`"NGH"` for North Okkalapa General + Hospital, correct value `"NOH"`). `product_sheet_name`'s mismatch count + (201) now reproduces the parity-presentation PDF's number exactly. Actual + per-column triage (both arms) didn't converge — split into [ticket + 18](tickets/18-triage-comparison-flagged-differences.md), which also + inherits the still-open 189-vs-155-tracker discrepancy (confirmed + unaffected by this fix). Full detail: [ticket + 17](tickets/17-fix-product-row-alignment-and-triage.md). + +- [Triage every flagged R/Python difference for both arms, and resolve the + 189-vs-155-tracker discrepancy](tickets/18-triage-comparison-flagged-differences.md) + — decided and partially executed: the 189-vs-155 gap is unreconcilable (no + drive snapshot, including the current Python run, reaches 189 trackers — + the PDF's baseline predates everything that exists now), so per-column + counts are judged by pattern rather than exact reproduction. + `product_category` (13,638 mismatches) and most of `product_entry_date` + (10,211 of 10,424) are both root-caused to genuine R limitations — a + case/whitespace-sensitive category-lookup join in `read_product_data.R`, + and a plain R date-extraction gap confirmed against real source Excel — + and classified accordingly in `src/a4d/migration/compare.py` + (`r_category_lookup_miss` added; `typo_rescue` renamed `r_value_missing` + since most cases aren't typos). Raw-stage `product_entry_date` was found + to be ~99% a serial-vs-parsed representation artifact, not real + divergence. Remaining product columns, remaining raw-stage columns, and + the entire patient arm didn't converge — split into [ticket + 20](tickets/20-fix-raw-entry-date-representation.md), [ticket + 21](tickets/21-triage-remaining-product-columns.md), [ticket + 22](tickets/22-triage-product-raw-columns.md), and [ticket + 23](tickets/23-triage-patient-arm.md). Full detail: [ticket + 18](tickets/18-triage-comparison-flagged-differences.md). + +- [Normalize the raw-stage product_entry_date comparison so it stops + flagging near-universal false mismatches](tickets/20-fix-raw-entry-date-representation.md) + — decided and implemented: `normalize_date_column()` reuses the cleaning + stage's flexible date parser to parse both sides' raw `product_entry_date` + to a common `date` before diffing, wired in for the `Product (raw)` stage + only. Verified against the real drive data: raw-stage `product_entry_date` + mismatches dropped from 65,743 to 91 (99.86% was the representation + artifact); 46 of the residual land in existing seeded classifiers, and the + other 50 (including a confirmed real bug — a stray `"\n"`-dated row in + Python's raw extraction for one Sarawak sheet) were folded into [ticket + 22](tickets/22-triage-product-raw-columns.md)'s scope rather than left + untracked, since the destination requires every difference explicitly + decided. Full detail: [ticket + 20](tickets/20-fix-raw-entry-date-representation.md). + +- [Triage the remaining product raw-stage column + mismatches](tickets/22-triage-product-raw-columns.md) — decided and + implemented: a real Python bug (`remove_header_rows` missing blank rows + where one cell held `""` instead of `None`) and two comparison-tool + normalizations (`normalize_numeric_column` for float-string formatting, + `normalize_whitespace_column` for readxl's whitespace-trim and + `\r\n`-vs-`\n` defaults) together cut raw-stage product mismatches from + 2,007 to 105 (95%), verified against the real drive data. Fully resolved + `product`, `product_balance`, `product_entry_date`, `product_remarks`, + `product_released_to`, `product_units_returned`. Residual split into + [ticket 24](tickets/24-triage-remaining-raw-column-residual.md). Full + detail: [ticket 22](tickets/22-triage-product-raw-columns.md). + +- [Triage the remaining product cleaned-stage column + mismatches](tickets/21-triage-remaining-product-columns.md) — decided and + implemented: five of six columns (`product_balance`, + `product_received_from`, `product_released_to`, `product_remarks`, + `product_units_received`) share one root cause — R's per-(clinic, sheet, + product) row sort falls back to raw input-row order whenever + `product_entry_date` fails to parse, near-universally so for several + major clinics; Python correctly sorts chronologically instead, using the + identical documented rank algorithm. Since the row-alignment key is + purely positional, this legitimate order difference cascades into + value-level mismatches on every column compared through it (verified: the + underlying data is unaffected — 98.3% of affected groups still land on + the same end-of-group balance). A new `row_order_divergence` classifier + explains 4 of these 5 columns fully or almost fully (100%, 100%, 100%, + 97.4%); `product_balance` shares the cause but under-detects via simple + value-membership (27.9% caught), left as future work. `product` (the + sixth column) had a different, single cause — an embedded `\r\n`-vs-`\n` + line break surviving cleaning unnoticed — resolved fully (652 -> 0) by + extending ticket 22's whitespace normalization to the cleaned stage. + Discovered `product_units_released`'s cleaned-stage mismatches (2,144) + were never assigned to any ticket; spawned [ticket + 25](tickets/25-triage-product-units-released-cleaned.md). Full detail: + [ticket 21](tickets/21-triage-remaining-product-columns.md). + +- [Triage every flagged R/Python difference for the patient + arm](tickets/23-triage-patient-arm.md) — decided and implemented + (raw-stage dominant cause only): patient raw-stage mismatches were + dominated by the same date-representation artifact ticket 20 fixed for + product — `normalize_date_column` extended to `Patient (raw)` via the + cleaned schema's `get_date_columns()` helper. Verified: raw-stage + mismatches dropped 564,096 -> 46,788 (91.7%) across 70 -> 67 columns; the + cleaned stage (120,639 mismatches, 61 columns) confirmed unaffected. This + unblocks [ticket 12](tickets/12-retire-r-workspace.md) — its last + remaining blocker. Raw residual and cleaned-stage triage split into + [ticket 27](tickets/27-triage-patient-raw-residual.md) and [ticket + 28](tickets/28-triage-patient-cleaned.md). Full detail: [ticket + 23](tickets/23-triage-patient-arm.md). + +- **Destination redrawn**: performance re-profiling, CLI/UX + observability, + retiring R from the workspace, and a dependency/library version audit are + all in this map's scope, not a separate effort — the user confirmed each + belongs to "are we really ready to roll out" (2026-08-09, mid-ticket-5 + session). Spawned [ticket 10](tickets/10-performance-profiling.md), [ticket + 11](tickets/11-cli-ux-observability.md) (graduated from fog), [ticket + 12](tickets/12-retire-r-workspace.md), and [ticket + 13](tickets/13-dependency-audit.md); all four now block [ticket + 6](tickets/06-promote-migration-to-dev.md). `CLAUDE.md`'s current "do not + modify `r-archive/`" instruction is a known conflict ticket 12 will need to + resolve, not before. Ticket 13 was also wired ahead of ticket 10 + (`blocked_by: [3, 13]`) since profiling against dependencies that are about + to change would produce stale numbers. + +- [Triage the patient cleaned-stage column mismatches](tickets/28-triage-patient-cleaned.md) + — decided and partially implemented: root-caused and resolved three of the + five dominant columns. `t1d_diagnosis_age` (25,968 mismatches) was a real + Python bug — `_fix_t1d_diagnosis_age` discarded a real recorded diagnosis + age whenever `dob`/`t1d_diagnosis_date` failed to parse, wrongly believing + this matched R (R's own equivalent function is dead code, never called); + fixed to prefer the raw recorded value, falling back to date-based + calculation only when genuinely missing. Verified with a real 248-tracker + pipeline re-run: 25,968 -> 4,807 (81.5% drop). `recruitment_date` (28,512) + and `insulin_subtype` (15,724) are genuine Python-correct divergences — + confirmed against the real source Excel (recruitment_date) and already + documented in code (insulin_subtype's R validator/typo bug) — given new + classifiers rather than "fixed". Total cleaned-stage mismatches: 120,639 -> + 99,478 (17.5%). `insulin_total_units`, `fbg_baseline_mg`, and the other 56 + columns didn't converge — split into [ticket + 29](tickets/29-triage-patient-cleaned-residual.md). Full detail: [ticket + 28](tickets/28-triage-patient-cleaned.md). + +- [Triage the product pipeline's column-existence and dtype + divergence](tickets/26-triage-product-column-divergence.md) — decided and + implemented: added a `column_divergence` report sheet (per-file + only-in-R/only-in-Python/dtype-mismatch rows) closing the "CLI-only, never + written to a report" gap; root-caused every named divergence against the + real 248-tracker drive data plus real source Excel for the raw-stage gap. + `product_table_month`/`product_table_year`/`product_unit_capacity` + (Float64-vs-Int32) and `product_remarks` (Boolean-vs-String, all-null + columns) are confirmed harmless representation artifacts, left + undisturbed in `compare_columns()` per an existing test's "flag every + dtype difference" policy — documented, not normalized. + `orig_product_released_to` confirmed an expected Python-only helper + column. `product_returned_by`/`product_units_returned` (raw stage, + only-in-Python) confirmed a genuine R extraction gap — R's raw output + drops a column literally present in the source Excel header + ("Units Returned") that Python correctly extracts. Patient's cleaned + stage is essentially clean; patient's raw stage (18,783 divergence rows, + two large unexplained patterns) didn't converge — split into [ticket + 30](tickets/30-triage-patient-raw-column-divergence.md). Full detail: + [ticket 26](tickets/26-triage-product-column-divergence.md). + +- [Triage the residual product_units_received/product_units_released/ + product_received_from raw-stage + mismatches](tickets/24-triage-remaining-raw-column-residual.md) — + decided and implemented: all 105 residual mismatches fully explained, + none requiring a pipeline fix. 80 rows (product_units_received, + product_received_from) are a newly-identified, possibly-systemic + pattern — R's readxl coerces a lone Excel date/time-formatted cell in an + otherwise-numeric column to that column's numeric type, while Python's + openpyxl honors the individual cell's own format, verified against real + source Excel; a new `STRAY_DATE_CLASSIFIERS` classifier handles it, + using openpyxl's own `from_excel` to replicate the Excel 1900-leap-year + serial bug. 20 rows were float-precision representation differences, + fixed by extending `normalize_numeric_column` (ticket 22's precedent) to + all three columns. The remaining 5 rows (product_units_released, + 2017-2019 Mandalay files) are a wide-format comma/hyphen-split ambiguity + on messy source notes where Python's value is verified more faithful + than R's (a strict prefix relation) — a new + `WIDE_FORMAT_FRAGMENT_CLASSIFIERS` classifier documents it rather than + chasing R's more-truncated answer. The date/time-coercion pattern's + possible patient-arm implications were noted as a lead on the + already-open [ticket 27](tickets/27-triage-patient-raw-residual.md) + rather than spawning a new ticket. This unblocks one of [ticket + 12](tickets/12-retire-r-workspace.md)'s remaining blockers. Full detail: + [ticket 24](tickets/24-triage-remaining-raw-column-residual.md). + +- [Triage the residual patient raw-stage column mismatches after date + normalization](tickets/27-triage-patient-raw-residual.md) — decided and + implemented, including a **real pipeline change**: extended + `normalize_numeric_column` (derived via `get_numeric_columns()`) and + `normalize_date_column` (hand-extended with the schema-absent + `meter_received_date`) to patient's raw stage; added `buddhist_era_typo` + (a clinician-entered Thai Buddhist-Era year in a Gregorian date cell, + verified against real source Excel, harmless since the cleaned stage's + future-date guard already reconciles it). Then, after the user challenged + the first close: `clean_excel_errors` was removed from all four + extraction call sites in **both arms** — extraction had been silently + nulling the source trackers' own `#DIV/0!`/`#NUM!` strings, so the raw + layer misreported the source file — replaced by + `normalize_excel_formula_errors` at the cleaning stage, which nulls them + (deliberately `null`, not the `999999` "recorded but invalid" sentinel: + a calculation whose input was never entered is absent, not invalid) and + logs each under a new `source_formula_error` code. Verified by a full + both-arm re-run against the real 248-tracker dataset: **cleaned-stage + output byte-identical in both arms** (no production data moved), raw + stage now faithful, new log entries across 145 tracker files. Raw-stage + patient mismatches 46,788 -> 18,813 on the tool fixes, then 28,033 once + Python became more faithful than R (readxl inconsistently nulls error + cells depending on its column-type guess) — all classified, with + `unclassified` flat at 14,981. `complication_screening` (12,566) and ~50 + smaller columns didn't converge — split into [ticket + 31](tickets/31-triage-patient-raw-residual-2.md). Full detail: [ticket + 27](tickets/27-triage-patient-raw-residual.md). + +- **Standing bar set: triage means deciding, not labelling** (user, 2026-08-12g, + after [ticket 27](tickets/27-triage-patient-raw-residual.md) had to be + reopened for exactly this failure). Explaining a difference and naming a + cause is only half the job; every flagged difference must also carry an + explicit verdict on whether Python is doing the right thing. "Python has A + where R has B" plus a classifier is not a decision in favour of A. Recorded + in the map's Notes, propagated into every open triage ticket's body, and + spawned [ticket 32](tickets/32-audit-classifiers-against-decision-bar.md) + to re-audit all nine existing classifier registries against it — + `off_by_one_day` (labels a one-day date gap without ever deciding which + side is right) and `r_value_missing` (11,468 cleaned product rows, + generalized from a single verified instance, knowingly lumping two distinct + causes) are the clearest failures. + +- [Fix red CI — ruff format --check fails on Python snippets inside markdown + docs](tickets/33-fix-red-ci-ruff-format-markdown.md) — decided and + implemented: `docs/archive` added to ruff's `extend-exclude` in + `pyproject.toml`, next to the existing `r-archive` entry. The user + overrode this ticket's own recommendation (which had been to accept the + reformatting) on the grounds that `MIGRATION_GUIDE.md` is a working spec + document with nothing in it for ruff to validate — its fenced Python is + illustrative prose, never imported or executed. Scoped to the whole + directory rather than the one failing file, since both markdown files in + the repo carrying Python fences live there and the second would have + re-broken CI on any future edit. Every CI step reproduced locally and + passing. The guard the ticket asked for — so local and CI check sets + cannot drift again — was **not** built here and is spawned as [ticket + 34](tickets/34-local-ci-parity-guard.md): `just ci` already exists but + still doesn't match CI (no coverage gate, different pytest markers) and + nothing makes anyone run it. Full detail: [ticket + 33](tickets/33-fix-red-ci-ruff-format-markdown.md). + +- **CI had been red on `migration` since 2026-08-09** — found 2026-08-12g + when the user asked to fix CI before more pipeline work. Not a regression + of [ticket 4](tickets/04-fix-migration-ci.md)'s fix: the sole cause is + `ruff format --check` wanting to reformat Python snippets inside + `docs/archive/MIGRATION_GUIDE.md`, so the step fails in ~15s and the + test suite never runs at all. Every triage ticket closed since was + verified against a green *local* suite while CI itself was red — local + and CI check sets had silently diverged. Fixed the same session — see the ticket-33 entry above; ticket 33's + blocker on [ticket 6](tickets/06-promote-migration-to-dev.md) is + released, now that CI green is restored. + +- **CI is green again on `migration`** (run `31644262255`, 2026-08-12g) — + first success since 2026-08-09, and the first run in that window to get + past the formatting step and actually execute the test suite. With its + output readable again, the suite's **17 Polars 2.0 deprecation warnings** + (three source sites: `empty_as_null` on `str.split` in `clean/product.py` + and its mirror in `validate/source_vs_output_product.py`, plus a + String->Date cast in `clean/converters.py`) were surfaced and ticketed as + [ticket 35](tickets/35-polars-2-deprecation-warnings.md). Not silencing + work: each warning is a real behaviour question (does an empty split + fragment become `null` or a phantom empty-string product row?), to be + decided against real tracker data per the map's triage bar. + +- [Triage the product_units_released cleaned-stage column + mismatches](tickets/25-triage-product-units-released-cleaned.md) — decided + and implemented (comparison tooling only, no pipeline change): all 2,144 + mismatches are ticket 21's row-order divergence, and **Python is correct** + — R's date-extraction gap leaves `product_entry_date` null on every row of + the affected groups (source Excel confirmed fully populated), so R keeps + raw input order while Python sorts chronologically. Established by + exhaustive multiset checks rather than by the classifier's own loose + membership heuristic: per-group values identical across all 2,283 groups, + `(units_released, released_to)` pairs identical across all 11,649 groups. + Fixed the actual defect — `product_units_released` mapped to the + raw-stage-only wide-format registry alone — taking the column from 2,144 + `unclassified` to 0, with a regression test pinning that *every* + positionally-aligned product column carries the row-order classifier. + Surfaced that 2,488 product cleaned-stage rows across five columns belong + to no ticket; spawned [ticket + 36](tickets/36-triage-product-cleaned-unclassified-residual.md), which also + inherits the newly-explained reason `product_balance` resists detection + (it is derived, so it cannot follow a row under a re-sort). Full detail: + [ticket 25](tickets/25-triage-product-units-released-cleaned.md). + +- [Triage the product cleaned-stage mismatches no ticket + owns](tickets/36-triage-product-cleaned-unclassified-residual.md) — + decided and implemented, **and with it every product-arm cleaned-stage + mismatch is now explained**: 2,488 `unclassified` -> 20. `product_balance` + (1,976) needed order-independent evidence, since a derived running total + cannot travel with its row under a re-sort — a new `group_endpoint_matches` + diagnostic on `CellMismatch` plus a `derived_running_total_row_order` + classifier: 2,729 of 2,740 balance mismatches sit in groups whose closing + balance agrees exactly (2,281 of 2,283 groups). `product_entry_date` (169) + was mixed-type source cells — readxl nulls the text-formatted dates in a + column it guessed as date, dropping R back to input-order sorting — plus + two verified Python-correct behaviours (`python_future_date_sentinel`, + `summary_residue_nulled`). `product_units_received` (8) was ticket 24's + stray-date cause reaching the cleaned stage, where Python zeroes and logs + `type_conversion` while R carries the raw serial into its ledger. The + remaining 20 are deliberate signals, not labels. **A real pipeline change + landed**: patient cleaning now strips string whitespace before validation + and `tables/metadata.py` strips `tracker_path.stem`, on the user's + decision that end-whitespace never carries meaning — which recovered 72 + rows of patient `sex` that R's validator and Python both used to lose to + the "Undefined" sentinel (source-verified: Kantha Bopha 2019, KH_KB023, + cell reads `'F '`). Also replaced ticket 25's hand-written + `positional_columns` test list with one derived from the product schema — + that list had itself omitted `product_entry_date`, which is why 169 rows + sat unclassified. Full detail: [ticket + 36](tickets/36-triage-product-cleaned-unclassified-residual.md). + +- **Standing bar refined: a corrupt source is a valid final answer** (user, + 2026-08-13, mid-ticket-36). Triage still may not stop at a label, but + deciding whether Python or R is *right* is not this phase's job — a clear + understanding of the mechanism is. Where the mechanism turns out to be a + corrupt source file, "this tracker needs human inspection" is the + conclusion, and a valuable finding rather than a failure to converge. + Recorded in the map's Notes. + +- [Triage the residual patient raw-stage column mismatches (round 5)](tickets/49-triage-patient-raw-residual-5.md) + — patient raw-stage unclassified 229 -> 84, with no Python defect in it: two + comparison-tool defects fixed (the whitespace target list was derived from + the *cleaned* schema, so it never reached `dm_complications`, `insulin_injections` + or `hba1c_updated`; and R's `FALSE` vs Python's `False` for an Excel boolean, + confirmed absent at the cleaned stage) and two R limitations root-caused and + classified (`r_non_latin_header_miss` — R's Unicode-aware sanitizer keeps a + Thai translation appended to a header, so its exact `match()` drops the whole + column, 44 rows; and ticket 37's 2026 Annual-sheet gap reaching five more + columns, 41). Residual split into [ticket 50](tickets/50-triage-patient-raw-residual-6.md). + +- [Triage the residual patient cleaned-stage mismatches (round 4)](tickets/51-triage-patient-cleaned-residual-4.md) + — cleaned-stage in-scope unclassified 5,031 -> 2,538 (whole-stage 7,967 -> + 5,474), on three causes each with an explicit verdict that **Python is + right**. `r_ymd_first_misparse` (1,714): R's `parse_date_string` orders + `ymd` before `dmy`, so a source date written `30.1.18` is read year-first as + 2030-01-18 — verified against 2018 Yangon's own cells (`Jan18!N92` = + `223(30.1.18)`), and R's reading puts 1,236 of the 1,714 in the future + relative to the tracker's own year while Python's puts none there. + `python_rejects_beyond_tracker_year` (722): Python's `_validate_dates` + sentinels a date past its tracker year and R has no such guard at all — the + triggering data is a real source defect, 2022 VNCH recording every diagnosis + date as a 2023 one for patients recruited in 2017. `r_unicode_sanitizer_rejects_accent` + (57): R's `[^[:alnum:]]` keeps accents and Python's `[^a-z0-9]` folds them, + so Python recovers the province `Thái Nguyễn` -> `Thái Nguyên` that R + discards as "Undefined". **A false docstring claim was corrected**: + `_validate_dates` said its future-date guard "matches R pipeline behavior"; + it does not, and now says so. Residual split into [ticket + 52](tickets/52-triage-patient-cleaned-residual-5.md). + +- [Triage the residual patient cleaned-stage mismatches (round + 5)](tickets/52-triage-patient-cleaned-residual-5.md) — in-scope residual + 2,538 -> 2,278, and the largest shape turned out to be **two real Python + bugs, not a labelling job**. A bare four-digit year typed into a date cell + was read as an Excel serial and became a 1905 date: 590 `dob` cells across + four Yangon Children's trackers, 425 `t1d_diagnosis_date` cells across five + more clinics, driving `age` to the 999999 sentinel and `t1d_diagnosis_age` + to -95. Fixed in **both** places the misreading happens — the bare-year + window in `parse_date_flexible`, and `_IMPOSSIBLE_DATE_BEFORE` raised + 1903 -> 1906 in `read_patient_rows` for the one file whose column is + date-formatted. Python is right on the source's own evidence: Sarawak + General writes the same patients' diagnoses as real 1-January dates in its + 2024 workbook and as bare years in 2025/2026. Separately, + `_fix_t1d_diagnosis_age` emitted **negative** ages (25 cells) where the + source records a diagnosis before the birth date — the workbook's own + formula says `#NUM!` — now nulled, and a ticket 40 finding. Two classifiers + landed (`python_reads_bare_year` 1,166, `python_age_from_bare_year` 992), + the second on a new row-level `row_has_bare_year_date` flag rather than on + the derived values' shape. Cleaned output now holds zero pre-1930 birth + dates and zero negative diagnosis ages. Residual split into [round + 6](tickets/53-triage-patient-cleaned-residual-6.md). +- [Triage the residual patient cleaned-stage mismatches (round + 6)](tickets/53-triage-patient-cleaned-residual-6.md) — 2,278 -> **1,689** + in-scope (26%), raw held at 0, on two Python defects fixed rather than + labelled. `parse_date_flexible` let `dateutil` complete an absent **day from + today** for every month-year spelling the alphabetic branch misses + (`10/2019`, `Mar, 2017`, `Jun'09`) — 42 production cells whose value changed + with the run date; closed as a *class* by parsing twice against two disjoint + defaults, so an invented day becomes the 1st and an invented month or year + makes the cell unparseable. `split_bp_in_sys_and_dias` left the padding on + each fragment, and Polars' cast fails where R's `as.numeric` tolerates it, so + every blood pressure written `70 / 40` reached the output as the 999999 + sentinel — 465 cells across 7 trackers restored, both BP columns out of the + residual (302 -> 13, 215 -> 13). `hospitalisation_date` (489) was measured + and found to be **100%** [ticket + 39](tickets/39-recover-dates-embedded-in-free-text.md)'s clinical-note + population — deliberately left unclassified. Residual split into [round + 7](tickets/54-triage-patient-cleaned-residual-7.md). + +- [Triage the residual patient cleaned-stage mismatches (round + 7)](tickets/54-triage-patient-cleaned-residual-7.md) -- 1,200 -> **770** + in-scope (36%), raw held at 0, on one R limitation and one harness gap, with + no pipeline behaviour changed. `t1d_diagnosis_age` (298 -> 0) resolved into + two causes: **R never derives a diagnosis age at all** -- its + `fix_t1d_diagnosis_age` is unit-tested against exactly the strings the + trackers carry ("At birth", "5y", "10y10m") but its call site is commented + out (`script2_process_patient_data.R:251`, read directly), so a blank cell + stays NA and a word-written age becomes R's 999999, while Python derives from + `dob` and `t1d_diagnosis_date` and lands on a figure the source's own words + confirm (MM_MD010 reads `11yr`, Python derives 11) -- and + `source_date_in_diagnosis_age` (16), a date typed into the age column at two + clinics, verified in both workbooks, where Python is right to null what R + carries through as an Excel serial. The ticket's own hypothesis, that this + was a downstream face of `python_reads_bare_year`, was **killed by + measurement**: R and Python hold identical dates on the dominant Sarawak + population, so no bare-year mismatch exists to be downstream of. Separately, + 132 cells across two screening-measurement columns were pure float-rounding + representation (`4.8600000000000003` vs `4.86`) surviving into the cleaned + stage because the schema types those columns as **String** -- the one + exception to ticket 22's "cleaning casts its numerics" argument -- fixed with + `string_numeric_normalize_targets`, derived from each frame's own dtypes, so + they stop being mismatches at all (total 114,712 -> 114,580, exactly the + 132). Three ticket 40 source-defect findings. `height`, `fbg_updated_mg`, + `bmi` and `insulin_subtype` were each measured to their mechanism but + deliberately not decided -- split into [round + 8](tickets/55-triage-patient-cleaned-residual-8.md). + +- [Classify the cleaned-stage FBG cells where R has nothing and Python has a + corrected reading](tickets/44-triage-cleaned-fbg-r-null-residual.md) -- the + mmol column is **derived** from the mg one on both sides, so all 2,935 cells + are an already-named mg divergence restated next door; one cascade classifier + bounded by the ÷18 identity, plus the file-level swap fact read from the + run's own error records, take the cleaned stage's whole unclassified + population **2,955 -> 16**, all 16 already owned by fog. +- [Re-audit every existing cause classifier — is Python actually right, or was + the diff merely labelled?](tickets/32-audit-classifiers-against-decision-bar.md) + — the audit is scoped to the **12** causes that predate the two-bar standard, + not all 50 (dated by first commit; the other 38 were argued under the bar). + Of the four known-weak ones: `off_by_one_day` and `ce_typo` deleted as labels + that named no mechanism, `sentinel_null` renamed + `python_sentinel_r_extraction_gap`, `r_value_missing` bounded so it stops + speaking for 32 cells where Python is the questionable side. Two pipeline + defects fixed underneath them — the `>= 2400` era exemption was publishing + corrupt Excel serials as dates in the year 5567, and `a4d run` was publishing + an errors table holding the patient arm only (63,295 -> 97,326 records). Eight + pre-bar causes split to [ticket 60](tickets/60-audit-remaining-pre-bar-classifiers.md). +- [Finish the pre-bar classifier audit — the two causes and the one bulk + population it did not reach](tickets/62-finish-the-pre-bar-classifier-audit.md) + -- decided; **the twelve-cause audit ticket 32 scoped is now complete**. + Python is the correct side in all three populations and **no pipeline change + was needed** — the first ticket in the chain to find no defect. All three + *explanations* were wrong, so three docstrings are rewritten against measured + mechanisms. `r_category_lookup_miss` (866) is two causes, and the larger is + not about R's join at all: **652 rows are CRLF-vs-LF** (the reference workbook + stores `\n` — zero CR bytes in its sheet XML — where the trackers store + `\r\n`; readxl is faithful to both, openpyxl folds them), leaving 214 that are + genuine case. `openpyxl_date_typed_stray_cell` (100) states one verdict for + three shapes and is **backwards for 66 of them** — R's serial is the quantity + the clinician typed and Python's 1900 date is the misleading rendering, + harmless only because both sides publish identical cleaned output; the 20 + patient rows it never documented are the opposite, with R publishing diagnosis + ages of **20,668 and 42,859 years**. `r_extraction_gap`'s + `recruitment_date` bulk (28,009) does not "fail for the large majority of + patients" — R populates 64% — it fails for **91 whole trackers and no partial + file**, because a trailing space in the header forces + `` and `openxlsx` folds the attribute into the column + name (`xmlspacepreservedateofrecruitmentmmmyy`, present in 91/91 affected + files and 0/148 others). Verified by a second full comparison showing zero + per-cause movement in all four stages. + +## Assumptions in force + +- **A date whose year is past 2400 and which decodes to no later than its + tracker's year is a Buddhist-era date.** That rule is what converts 375 + patient cells and 22 product rows, and it is an inference from the year + alone -- nothing in the workbook says which calendar a cell is in. It is + strongly corroborated (every affected tracker is Thai; each product cell's + month matches its sheet; 2022 Hat Yai's `2560-01-01` converts to the + 2017-01-01 that patient's own D.O.B. and recruitment independently give) + but it is still an inference. Resting on [ticket + 61](tickets/61-decide-buddhist-era-date-conversion.md); overturned by a cell + in that range that means something else -- a mistyped Gregorian year, or a + serial corruption that happens to decode plausibly -- which would make the + conversion a silent rewrite rather than a recovery. + +- **A year-less date in a note belongs to its tracker's own year.** + `recover_date_from_text` (clean/date_parser.py) fills an absent year from the + row's `tracker_year`, which publishes 39 cells whose source states no year at + all. The data argues against it: `26 Jun (ceton urine high)` appears in all 12 + months of the 2020 *and* the 2021 VNCH tracker for patient `VN_VC051`, so one + event is published twice with two different years. The user accepted it on the + reasoning that a patient not seen within a year would not still be active. + Resting on [ticket 39](tickets/39-recover-dates-embedded-in-free-text.md); + overturned by a tracker where a year-less note is demonstrably carried forward + from an earlier year rather than re-recorded -- the VNCH case is a candidate + and was not chased. + +- **The glucose limits the pipeline enforces are the right ones.** A4D's + medical advisor gave the analytical limits of the machines in use (mg/dL + ~2-5 to ~720-800, mmol/L ~0.1-0.3 to ~40-45) and confirmed above 100 mmol/L + is impossible; the permissive end of each is now a hard bound on all four FBG + columns, and 832 readings are rejected on it. This is an expert statement, + not something this repo can verify, and his reply carried one internal + inconsistency (readings below 0.1 mmol/L described as possible while his own + floor is 0.1-0.3 -- immaterial, since no reading sits there). Resting on + [ticket 42](tickets/42-fbg-unit-headers-and-implausible-values.md); overturned + by a clinic confirming its meters read outside these ranges, which would make + the rejections data loss rather than data cleaning. + +- **A column that is >=90% sub-30 mg/dL was recorded in mmol.** The threshold + is not the advisor's -- he only authorised correcting what is "clearly" a unit + mix-up. It rests on the measured distribution being strongly bimodal (29 + groups above 0.9, 300 at 0.0) and on 951 consecutive sub-30 readings at one + clinic being clinically impossible. Resting on [ticket + 42](tickets/42-fbg-unit-headers-and-implausible-values.md); overturned by a + clinic that genuinely records in mg/dL and whose patients really do run that + low, or by a swapped column whose source workbook turns out to say mmol in + the header after all -- which is why every swap is reported for source + correction rather than silently applied. + +- **Python's baseline-FBG copy is the right one to keep.** Where a tracker + records baseline FBG on both the monthly sheet and the Patient List, + Python keeps the monthly value and R keeps neither (its join suffixes both + sides). The two copies agree on 92.4% of ~10,000 rows, so the choice + rarely matters — but on the 663 rows where they genuinely differ, nothing + establishes which sheet is authoritative; R's dropping of monthly + `hba1c_baseline` hints its author thought the Patient List was, for that + column at least. Resting on [ticket + 29](tickets/29-triage-patient-cleaned-residual.md); overturned by evidence + (a clinic's own convention, or a source-Excel pattern) that the Patient + List copy is the intended baseline, which would make Python's monthly-wins + a silent data choice rather than a harmless one. + +- **The words that count as "no date recorded" are the right ones.** + `DATE_ABSENCE_MARKERS` (clean/date_parser.py) treats `nil`, `nill`, `no`, + `unknown`, `unknwon`, `uncertain` and `?` as an absence rather than as an + unusable value. `nil`/`unknown`/`uncertain`/`?` are unambiguous; `unknwon` + and `nill` are typo spellings observed in this dataset and may be + over-fitting to it. Resting on [ticket + 38](tickets/38-triage-patient-cleaned-date-family.md); overturned by a + tracker where one of these words carries meaning other than absence, or by + the numeric path being brought in line (see **Not yet specified**), which + would need the same list to survive a second review. + +(The one assumption this map originally carried, patient's completeness being +unverified, was confirmed rather than overturned by +[the completeness audit](tickets/07-pipeline-completeness-audit.md) and is now +folded into Decisions so far above.) + +## Not yet specified + +- Whether **R's inability to read a slash-separated month/year** deserves its + own cause. One `bmi_date` cell (`0ct/19`, typo-rescued to `OCT/19`) is + unclassified: Python reads October 2019 and is right, R's parse order list + cannot express the spelling. The obvious home is a widened + `r_parse_order_cannot_read_cell`, whose own docstring says the day>12 + signature is what keeps it from over-claiming -- so widening it needs the same + measurement round 10 did before backing out a similar widening. One cell is + not enough to judge it on. Surfaced by [ticket + 39](tickets/39-recover-dates-embedded-in-free-text.md). + +- Whether the **product category join should normalise line endings** + explicitly. [Ticket 62](tickets/62-finish-the-pre-bar-classifier-audit.md) + found Python resolves 26 product names carrying an embedded line break purely + because openpyxl folds CRLF to LF on read — its own join normalises case and + nothing else. A reference entry typed with CRLF against a tracker spelled with + LF would miss on Python exactly as it misses on R today, and the comparison + would be silent, since both sides would be null. Nothing in the current + 254-tracker set triggers it, so nobody has measured how exposed the join + actually is or whether an explicit normalisation would change any current + output. Not sharp enough to ticket until that is measured; it moves a + production join. + +- Whether the **unaccented** spellings of a province should be recovered too. + Resolved for accents by [ticket 51](tickets/51-triage-patient-cleaned-residual-4.md): + Python's ASCII-folding `sanitize_str` is safe (the 209-entry province list + has zero accent-strip collisions, and `validate_allowed_values` *raises* on + any two allowed values that sanitize alike), and it recovers `Thái Nguyễn` + where R stamps "Undefined". What is left is the spelling with no accents at + all: the same VNCH trackers also write `Thai Nguyen` and `Thai nguyen`, which + sanitize to `thainguyen` on **both** sides and so are lost in both pipelines. + The comparison is silent on it — a shared limitation, not a divergence — so + nobody has measured how many provinces across how many trackers are written + this way, or whether an `aliases` entry is the right fix. Not sharp enough to + ticket until that is measured. + +- Whether an insulin row that **ticks nothing** should publish `Undefined`. + Both pipelines do today, on 17,418 rows, so the comparison is silent on it -- + which is why round 8 measured the alternative rather than assuming it: making + Python null those rows created 17,418 new divergences and resolved none, so + it was reverted. The claim is still false in the same way `999999` is on a + numeric absence (below): the clinic recorded no subtype, and `Undefined` + says it recorded an unrecognised one. Not sharp enough to ticket until + someone has said whether the downstream consumer distinguishes them. + Surfaced by [ticket 55](tickets/55-triage-patient-cleaned-residual-8.md). + **Sharpened by [round 9](tickets/56-triage-patient-cleaned-residual-9.md)**, + which found 11 more cells reaching the same question from the other side: a + row ticking `-` in all five columns, where R holds null and Python + `Undefined`. R's apparent distinction between the two is not a design -- + `ifelse(x == "Y", ...)` yields `""` for `-` and `NA` for null, and only the + second survives to become `Undefined` -- so R's null cannot be cited as + evidence for what the output should say. +- Whether the **numeric** conversion path should treat absence-written-as-a- + word (`Nil`, `Unknown`, `?`) as missing, the way the date path now does + (ticket 38). Correct in principle -- 999999 makes the same false claim there + that 9999-09-09 made on dates -- but it moves production numeric output and + would invalidate ticket 29's exhaustive verification of + `r_numeric_error_sentinel`, so it was deliberately not reopened in the same + session that depended on it. Not sharp enough to ticket until someone has + measured how many numeric cells it actually moves. **Sharpened by [ticket + 44](tickets/44-triage-cleaned-fbg-r-null-residual.md)**: the last 2 + unclassified `fbg_updated_mmol` cells (2025 Kantha Bopha II, KH_KB119) are + R-null against Python's `999999` where *neither side published a reading* -- + the cascade classifier declines them deliberately, because there is no + measurement to cascade. Together with the 11 `insulin_subtype` and 2 + `remote_followup` cells above, this patch and the `Undefined` one now own + 15 of the 16 cells the cleaned stage has left. +- Whether a **multi-select screening block should keep more than its first + selection**. `2021_Putrajaya` records four complication-screening selections + per row in four adjacent columns under one merged header; both pipelines keep + only the first, so the comparison stays silent (R parks the rest in unmapped + suffixed columns, Python drops them, neither reaches mapped output). Found + while closing [ticket 48](tickets/48-putrajaya-screening-columns-lost.md) and + deliberately not fixed there -- it is a shared limitation, not a divergence, + and nobody has yet said whether the extra selections are wanted. Not sharp + enough to ticket until someone has measured how many trackers lay a block out + this way and what the downstream consumer expects. +- What **"official migration" communication or cutover** means once `migration` + reaches `dev`/`main` — who is told, what the A4D-facing announcement says, + whether anything outside this repo still runs R. Both repo-side threads have + now graduated and closed: `r-archive/` is deleted ([ticket + 12](tickets/12-retire-r-workspace.md)), and `src/a4d/migration/compare.py`'s + fate is settled — kept, exempt, and marked historical ([ticket + 64](tickets/64-documentation-overhaul-drop-r-framing.md)). What remains is + organisational and is likely a separate map. +- Cloud Scheduler / production scheduling cutover (mentioned in the Migration + Guide's state-management open item) — not yet sharp enough to ticket; may + turn out to be a separate map entirely once `dev` is reached. Confirmed + during [ticket 5](tickets/05-production-verification-run.md) (executed: + `gcloud scheduler jobs list` fails with `SERVICE_DISABLED`) that the Cloud + Scheduler API isn't even enabled on `a4dphase2` yet — the Migration Guide's + claim that this is still open is accurate, not stale. +- Patient's own gaps from the completeness audit (no committed record of an + actual 174-tracker passing run; `PYTHON_IMPROVEMENTS.md` undercounting + known divergences; no `pipeline/patient.py` unit test) aren't ticketed yet + — they don't block the merge/promotion path the way product's gaps do, but + will need a home before the map can call itself done. + +## Out of scope + +- **Columns that appear in one tracker year and are gone the next.** 703 of + ticket 30's residual divergence rows, across 68 columns last seen in 2024 or + earlier (`TSH`, `tTG-IgA`, `Home Visit 1`, `Age (year)`, + `INSTANT Meter Received Date`). Ruled out by the user 2026-08-15: the current + template is the golden rule, these were very likely tests that did not + survive, and the trackers are kept if the data is ever wanted. Closed with + [ticket 30](tickets/30-triage-patient-raw-column-divergence.md). + +### The route actually walked + +Sessions top to bottom, oldest first. `spawned` and `closed` are causal — what a +decision did to the rest of the map — and are where the real structure lives. + + +```mermaid +flowchart TB + subgraph S2026_08_08["Session 2026-08-08"] + direction LR + U1["1
Does product-pipeline's
test suite meet the same
cell-by-cell rigor as
patient's?"] + U7["7
Is the product pipeline
(and patient's own
claimed completeness)
actually complete and
sound, audited against
R's product logic and
patient's structure?"] + U8["8
Does the pytest suite
reach unit/integration/e
2e/regression parity
between patient and
product, excluding any
R-comparison/USB-drive-
dependent tests?"] + end + subgraph S2026_08_08b["Session 2026-08-08b"] + direction LR + U3["3
Merge product-pipeline
(PR #6) into migration"] + end + subgraph S2026_08_09["Session 2026-08-09"] + direction LR + U4["4
Diagnose and fix why CI
is red at migration HEAD"] + end + subgraph S2026_08_09b["Session 2026-08-09b"] + direction LR + U5["5
Define and execute the
real GCP production
verification run"] + end + subgraph S2026_08_09c["Session 2026-08-09c"] + direction LR + U13["13
Audit and update all
dependencies and library
versions before rollout"] + end + subgraph S2026_08_09d["Session 2026-08-09d"] + direction LR + U10["10
Profile the combined
pipeline's performance
against the R baseline
before promoting to dev"] + end + subgraph S2026_08_09e["Session 2026-08-09e"] + direction LR + U14["14
Fix product pipeline's
unable to find column
product failures on 4
real trackers"] + end + subgraph S2026_08_09f["Session 2026-08-09f"] + direction LR + U2["2
Retire the PDF/notebook
analysis docs for an
automated, script-based
report"] + end + subgraph S2026_08_09g["Session 2026-08-09g"] + direction LR + U11["11
Decide what CLI/TUI UX
and error-log
observability
improvements
admins/developers need
before rollout"] + end + subgraph S2026_08_10["Session 2026-08-10"] + direction LR + U15["15
Build and run the
R/Python output
comparison script, then
triage every flagged
difference"] + end + subgraph S2026_08_11["Session 2026-08-11"] + direction LR + U17["17
Fix the product
comparison's row-
alignment key, then
triage every flagged
R/Python difference"] + U18["18
Triage every flagged
R/Python difference for
both arms, and resolve
the 189-vs-155-tracker
discrepancy"] + U19["19
Persist comparison run
history and show run-
over-run deltas"] + end + subgraph S2026_08_12["Session 2026-08-12"] + direction LR + U20["20
Normalize the raw-stage
product_entry_date
comparison so it stops
flagging near-universal
false mismatches"] + U21["21
Triage the remaining
product cleaned-stage
column mismatches
(balance, received_from,
released_to, remarks,
units_received, product)"] + U22["22
Triage the remaining
product raw-stage column
mismatches"] + end + subgraph S2026_08_12b["Session 2026-08-12b"] + direction LR + U23["23
Triage every flagged
R/Python difference for
the patient arm (raw and
cleaned)"] + end + subgraph S2026_08_12c["Session 2026-08-12c"] + direction LR + U28["28
Triage the patient
cleaned-stage column
mismatches"] + end + subgraph S2026_08_12d["Session 2026-08-12d"] + direction LR + U26["26
Triage the product
pipeline's column-
existence and dtype
divergence (Column
divergence)"] + end + subgraph S2026_08_12e["Session 2026-08-12e"] + direction LR + U24["24
Triage the residual prod
uct_units_received/produ
ct_units_released/produc
t_received_from raw-
stage mismatches"] + end + subgraph S2026_08_12f["Session 2026-08-12f"] + direction LR + U27["27
Triage the residual
patient raw-stage column
mismatches after date
normalization"] + end + subgraph S2026_08_12g["Session 2026-08-12g"] + direction LR + U33["33
Fix red CI — ruff format
--check fails on Python
snippets inside markdown
docs"] + end + subgraph S2026_08_12h["Session 2026-08-12h"] + direction LR + U25["25
Triage the
product_units_released
cleaned-stage column
mismatches"] + end + subgraph S2026_08_13["Session 2026-08-13"] + direction LR + U36["36
Triage the product
cleaned-stage mismatches
no ticket owns
(product_balance,
sheet_name, entry_date,
units_received,
file_name)"] + end + subgraph S2026_08_13b["Session 2026-08-13b"] + direction LR + U29["29
Triage the residual
patient cleaned-stage
column mismatches"] + end + subgraph S2026_08_14["Session 2026-08-14"] + direction LR + U37["37
Triage the residual
patient cleaned-stage
mismatches (round 2)"] + end + subgraph S2026_08_14b["Session 2026-08-14b"] + direction LR + U38["38
Triage the patient
cleaned-stage date-
column family (round 3)"] + end + subgraph S2026_08_15["Session 2026-08-15"] + direction LR + U30["30
Triage the patient
pipeline's raw-stage
column-existence
divergence"] + end + subgraph S2026_08_15b["Session 2026-08-15b"] + direction LR + U31["31
Triage the residual
patient raw-stage column
mismatches (round 2)"] + end + subgraph S2026_08_17["Session 2026-08-17"] + direction LR + U42["42
Decide how FBG unit
headers are resolved,
and what to do about
physiologically
implausible mmol values"] + end + subgraph S2026_08_17b["Session 2026-08-17b"] + direction LR + U43["43
Triage the residual
patient raw-stage column
mismatches (round 3)"] + end + subgraph S2026_08_17c["Session 2026-08-17c"] + direction LR + U45["45
Give the patient
comparison an ordinal
row key, so duplicated
patient IDs stop faking
mismatches"] + end + subgraph S2026_08_17d["Session 2026-08-17d"] + direction LR + U46["46
Triage the residual
patient raw-stage column
mismatches (round 4)"] + end + subgraph S2026_08_17e["Session 2026-08-17e"] + direction LR + U48["48
Python drops
complication-screening
results and dates where
a merged header spans
the block"] + end + subgraph S2026_08_18["Session 2026-08-18"] + direction LR + U49["49
Triage the residual
patient raw-stage column
mismatches (round 5)"] + end + subgraph S2026_08_18b["Session 2026-08-18b"] + direction LR + U50["50
Triage the residual
patient raw-stage column
mismatches (round 6)"] + end + subgraph S2026_08_19["Session 2026-08-19"] + direction LR + U51["51
Triage the residual
patient cleaned-stage
mismatches (round 4)"] + end + subgraph S2026_08_19b["Session 2026-08-19b"] + direction LR + U52["52
Triage the residual
patient cleaned-stage
mismatches (round 5)"] + end + subgraph S2026_08_19c["Session 2026-08-19c"] + direction LR + U53["53
Triage the residual
patient cleaned-stage
mismatches (round 6)"] + end + subgraph S2026_08_19d["Session 2026-08-19d"] + direction LR + U54["54
Triage the residual
patient cleaned-stage
mismatches (round 7)"] + end + subgraph S2026_08_19e["Session 2026-08-19e"] + direction LR + U55["55
Triage the residual
patient cleaned-stage
mismatches (round 8)"] + end + subgraph S2026_08_19f["Session 2026-08-19f"] + direction LR + U56["56
Triage the residual
patient cleaned-stage
mismatches (round 9)"] + end + subgraph S2026_08_20["Session 2026-08-20"] + direction LR + U57["57
Triage the residual
patient cleaned-stage
mismatches (round 10)"] + end + subgraph S2026_08_20b["Session 2026-08-20b"] + direction LR + U47["47
Four trackers where
cleaning merges several
patients into one
patient ID"] + end + subgraph S2026_08_21["Session 2026-08-21"] + direction LR + U39["39
Decide whether a date
buried inside a clinical
note should be recovered
or discarded"] + end + subgraph S2026_08_22["Session 2026-08-22"] + direction LR + U32["32
Re-audit every existing
cause classifier — is
Python actually right,
or was the diff merely
labelled?"] + end + subgraph S2026_08_22b["Session 2026-08-22b"] + direction LR + U61["61
Decide whether a Thai
clinic's Buddhist-era
entry date is published
as 2567 or converted to
2024"] + end + subgraph S2026_08_24["Session 2026-08-24"] + direction LR + U59["59
Rows that pair with
nothing on the other
side, which no ticket
has ever triaged"] + end + subgraph S2026_08_24b["Session 2026-08-24b"] + direction LR + U60["60
Audit the eight pre-bar
causes the first
classifier pass did not
reach"] + end + subgraph S2026_08_24c["Session 2026-08-24c"] + direction LR + U62["62
Finish the pre-bar
classifier audit — the
two causes and the one
bulk population it did
not reach"] + end + subgraph S2026_08_24d["Session 2026-08-24d"] + direction LR + U58["58
Monthly rows with a
misspelled ID silently
lose their Patient List
demographics"] + end + subgraph S2026_08_24e["Session 2026-08-24e"] + direction LR + U63["63
The cleaned stage has
4,949 cells with no
cause, because the ID
spelling that explains
them is gone by then"] + end + subgraph S2026_08_24f["Session 2026-08-24f"] + direction LR + U12["12
Retire R from the
workspace once the
pipeline is fully
verified Python-only"] + end + subgraph S2026_08_24g["Session 2026-08-24g"] + direction LR + U64["64
Rewrite every docstring
and doc that explains
the code by what R did"] + end + subgraph Sunworked["Closed without being worked"] + direction LR + U44["44
Classify the cleaned-
stage FBG cells where R
has nothing and Python
has a corrected reading"] + end + subgraph Sopen["Not yet worked"] + direction LR + U6["6
Promote migration into
dev via PR #2"] + U9["9
Add golden-
master/snapshot
regression tests for
patient and product"] + U16["16
Build a drill-down log
analyzer for admins to
inspect a specific
tracker file's
errors/logs"] + U34["34
Make the local pre-push
check set actually match
CI, and make running it
automatic"] + U35["35
Resolve the Polars 2.0
deprecation warnings —
decide the behaviour
each one is asking about"] + U40["40
Produce one Excel of
every source-tracker
defect, so the trackers
themselves can be
corrected"] + U41["41
Decide whether the 2026
template's five new
Patient List fields
enter the pipeline"] + U65["65
Two values published
into the logs table
still name R scripts"] + end + + S2026_08_08 ~~~ S2026_08_08b + S2026_08_08b ~~~ S2026_08_09 + S2026_08_09 ~~~ S2026_08_09b + S2026_08_09b ~~~ S2026_08_09c + S2026_08_09c ~~~ S2026_08_09d + S2026_08_09d ~~~ S2026_08_09e + S2026_08_09e ~~~ S2026_08_09f + S2026_08_09f ~~~ S2026_08_09g + S2026_08_09g ~~~ S2026_08_10 + S2026_08_10 ~~~ S2026_08_11 + S2026_08_11 ~~~ S2026_08_12 + S2026_08_12 ~~~ S2026_08_12b + S2026_08_12b ~~~ S2026_08_12c + S2026_08_12c ~~~ S2026_08_12d + S2026_08_12d ~~~ S2026_08_12e + S2026_08_12e ~~~ S2026_08_12f + S2026_08_12f ~~~ S2026_08_12g + S2026_08_12g ~~~ S2026_08_12h + S2026_08_12h ~~~ S2026_08_13 + S2026_08_13 ~~~ S2026_08_13b + S2026_08_13b ~~~ S2026_08_14 + S2026_08_14 ~~~ S2026_08_14b + S2026_08_14b ~~~ S2026_08_15 + S2026_08_15 ~~~ S2026_08_15b + S2026_08_15b ~~~ S2026_08_17 + S2026_08_17 ~~~ S2026_08_17b + S2026_08_17b ~~~ S2026_08_17c + S2026_08_17c ~~~ S2026_08_17d + S2026_08_17d ~~~ S2026_08_17e + S2026_08_17e ~~~ S2026_08_18 + S2026_08_18 ~~~ S2026_08_18b + S2026_08_18b ~~~ S2026_08_19 + S2026_08_19 ~~~ S2026_08_19b + S2026_08_19b ~~~ S2026_08_19c + S2026_08_19c ~~~ S2026_08_19d + S2026_08_19d ~~~ S2026_08_19e + S2026_08_19e ~~~ S2026_08_19f + S2026_08_19f ~~~ S2026_08_20 + S2026_08_20 ~~~ S2026_08_20b + S2026_08_20b ~~~ S2026_08_21 + S2026_08_21 ~~~ S2026_08_22 + S2026_08_22 ~~~ S2026_08_22b + S2026_08_22b ~~~ S2026_08_24 + S2026_08_24 ~~~ S2026_08_24b + S2026_08_24b ~~~ S2026_08_24c + S2026_08_24c ~~~ S2026_08_24d + S2026_08_24d ~~~ S2026_08_24e + S2026_08_24e ~~~ S2026_08_24f + S2026_08_24f ~~~ S2026_08_24g + S2026_08_24g ~~~ Sunworked + Sunworked ~~~ Sopen + + U3 --->|blocked| U2 + U8 --->|blocked| U3 + U3 --->|blocked| U4 + U3 --->|blocked| U5 + U4 --->|blocked| U5 + U8 --->|blocked| U6 + U3 --->|blocked| U6 + U4 --->|blocked| U6 + U5 --->|blocked| U6 + U10 --->|blocked| U6 + U11 --->|blocked| U6 + U12 --->|blocked| U6 + U13 --->|blocked| U6 + U14 --->|blocked| U6 + U20 --->|blocked| U6 + U21 --->|blocked| U6 + U22 --->|blocked| U6 + U23 --->|blocked| U6 + U64 --->|blocked| U6 + U1 -.->|spawned| U7 + U1 -.->|spawned| U8 + U7 --->|blocked| U8 + U8 -.->|spawned| U9 + U6 --->|blocked| U9 + U3 --->|blocked| U10 + U13 --->|blocked| U10 + U3 --->|blocked| U11 + U3 --->|blocked| U13 + U10 -.->|spawned| U14 + U2 -.->|spawned| U15 + U2 --->|blocked| U15 + U11 -.->|spawned| U16 + U15 -.->|spawned| U17 + U17 -.->|spawned| U18 + U17 -.->|spawned| U19 + U18 -.->|spawned| U20 + U18 -.->|spawned| U21 + U18 -.->|spawned| U22 + U18 -.->|spawned| U23 + U22 -.->|spawned| U24 + U21 -.->|spawned| U25 + U23 -.->|spawned| U27 + U23 -.->|spawned| U28 + U28 -.->|spawned| U29 + U26 -.->|spawned| U30 + U27 -.->|spawned| U31 + U27 -.->|spawned| U32 + U33 -.->|spawned| U34 + U33 -.->|spawned| U35 + U25 -.->|spawned| U36 + U29 -.->|spawned| U37 + U37 -.->|spawned| U38 + U38 -.->|spawned| U39 + U30 -.->|spawned| U40 + U30 -.->|spawned| U41 + U30 -.->|spawned| U42 + U31 -.->|spawned| U43 + U42 -.->|spawned| U44 + U43 -.->|spawned| U45 + U43 -.->|spawned| U46 + U45 --->|blocked| U46 + U45 -.->|spawned| U47 + U46 -.->|spawned| U48 + U46 -.->|spawned| U49 + U49 -.->|spawned| U50 + U50 -.->|spawned| U51 + U51 -.->|spawned| U52 + U52 -.->|spawned| U53 + U53 -.->|spawned| U54 + U54 -.->|spawned| U55 + U55 -.->|spawned| U56 + U56 -.->|spawned| U57 + U47 -.->|spawned| U58 + U47 -.->|spawned| U59 + U32 -.->|spawned| U60 + U32 -.->|spawned| U61 + U60 -.->|spawned| U62 + U58 -.->|spawned| U63 + U12 -.->|spawned| U64 + U64 -.->|spawned| U65 + + classDef tfrontier fill:#1f6feb,stroke:#0b3d91,stroke-width:3px,color:#ffffff + class U6,U16,U34,U35,U40,U41,U65 tfrontier + classDef tblocked fill:#6e7781,stroke:#424a53,stroke-width:1px,color:#ffffff + class U9 tblocked + classDef tdecided fill:#1a7f37,stroke:#116329,stroke-width:1px,color:#ffffff + class U2,U3,U4,U5,U7,U8,U10,U11,U12,U13,U14,U15,U17,U18,U19,U20,U21,U22,U23,U24,U25,U26,U27,U28,U29,U30,U31,U32,U33,U36,U37,U38,U39,U42,U43,U44,U45,U46,U47,U48,U49,U50,U51,U52,U53,U54,U55,U56,U57,U58,U59,U60,U61,U62,U63,U64 tdecided + classDef tdropped fill:#eaeef2,stroke:#afb8c1,stroke-width:1px,color:#57606a + class U1 tdropped +``` + diff --git a/docs/wayfinder/research/07-pipeline-completeness-audit.md b/docs/wayfinder/research/07-pipeline-completeness-audit.md new file mode 100644 index 0000000..5d2ff05 --- /dev/null +++ b/docs/wayfinder/research/07-pipeline-completeness-audit.md @@ -0,0 +1,370 @@ +--- +ticket: 07-pipeline-completeness-audit +title: Pipeline completeness audit — patient vs product, R-logic coverage, test parity, doc accuracy +date: 2026-08-08 +method: read-only (git show / git diff / rg / file reads); no pipeline execution, no output comparison +branches_compared: migration (checked out, HEAD) vs origin/product-pipeline (not merged) +--- + +# Pipeline completeness audit + +Read-only inventory. All claims below are independently reproducible with the +`git`/`rg`/`fd` commands shown inline. No pipeline was executed and no output +parquet was compared — per ticket scope, that's out of scope here. + +## 1. R-logic coverage, module for module + +### Patient (R: `r-archive/R/script1_helper_read_patient_data.R` + +`script2_helper_patient_data_fix.R`, Python: `migration` branch) + +R function inventory (`rg -n "^[a-zA-Z_.]+\s*<-\s*function" r-archive/R/script2_helper_patient_data_fix.R`): +`convert_to`, `cut_numeric_value`, `correct_decimal_sign`, `parse_dates`, +`check_allowed_values`, `parse_character_cleaning_pipeline`, `parse_step`, +`parse_allowed_value_check`, `parse_character_cleaning_config`, `fix_age`, +`fix_bmi`, `fix_sex`, `extract_year_from_age`, `transform_fbg_in_mmol`, +`fix_fbg`, `fix_testing_frequency`, `replace_range_with_mean`, +`split_bp_in_sys_and_dias`, `transform_cm_to_m`, `fix_id`, +`extract_first_raw_regimen`, `extract_regimen` — plus +`extract_patient_data` in `script1_helper_read_patient_data.R`. + +Python side (`src/a4d/clean/patient.py`, `clean/converters.py`, +`clean/validators.py`, `clean/transformers.py`, `clean/date_parser.py`, +`extract/patient.py`) has a 1:1-looking function for every one of these: +`convert_to`→`safe_convert_column`, `cut_numeric_value`→same name in +`converters.py`, `correct_decimal_sign`→same name, `parse_dates`→ +`parse_date_flexible`/`parse_date_column`, `check_allowed_values`→ +`validate_allowed_values`/`validate_column_from_rules`, `fix_age`→ +`_fix_age_from_dob`, `fix_bmi`→`_calculate_bmi`/`fix_bmi` in +`transformers.py`, `fix_sex`→`fix_sex`, `fix_testing_frequency`→same name, +`replace_range_with_mean`→same name, `split_bp_in_sys_and_dias`→same name, +`fix_id`→`fix_patient_id` in `validators.py`, `extract_regimen`→same name in +`transformers.py`. No R patient function was found without a plausible +Python counterpart. This matches the prior finding that the patient side is +the mature/reference implementation. + +### Product (R: `r-archive/R/helper_product_data.R` + +`script1_process_product_data.R` + `script2_process_product_data.R` + +`script3_create_table_product_data.R` + `script3_link_product_patient.R` + +`read_product_data.R`, Python: `origin/product-pipeline` only) + +`helper_product_data.R` (601 lines) function inventory: `extract_product_data`, +`harmonize_input_data_columns`, `read_column_synonyms_product`, +`sanitize_column_name`, `format_date_excelnum`, `format_date_exceldate`, +`format_date`, `extract_month`, `recode_unitcolumnstozero`, +`clean_unitsreceived`, `update_receivedfrom`, `clean_receivedfrom`, +`compute_balance_cleanrows`, `compute_balance_status`, `compute_balance`, +`adjust_column_classes`, `extract_product_multiple`. `read_product_data.R` +adds `count_na_rows`, `check_entry_dates`, `check_negative_balance`, +`report_unknown_products`, `load_product_reference_data`, +`add_product_categories`, `extract_unit_capacity`. + +Python (fetched via `git show origin/product-pipeline:`): +- `src/a4d/extract/product.py` (431 lines) — `find_product_section` (= + `extract_product_data`'s row-finding half), `extract_product_data`, + `remove_header_rows`, `replace_extra_totals`, `_harmonize` (= + `harmonize_input_data_columns`), `read_all_product_sheets`. +- `src/a4d/clean/product.py` (1073 lines) — `_split_multi_product_cells` + (= `extract_product_multiple`), `_switch_misplaced_columns`, + `_format_dates` (= `format_date`/`format_date_excelnum`/ + `format_date_exceldate`), `_check_entry_dates_match_sheet` / + `_validate_entry_dates` (= `check_entry_dates`), `_extract_balance_from_received` + (= `update_receivedfrom`), `_recode_na_units_to_zero` (= + `recode_unitcolumnstozero`), `_clean_received_from` (= `clean_receivedfrom`), + `_clean_units_received` (= `clean_unitsreceived`), `_remove_empty_data_rows` + (= `compute_balance_cleanrows`), `_compute_balance_status` (= + `compute_balance_status`), `_compute_running_balance` (= `compute_balance`), + `_validate_negative_balances` (= `check_negative_balance`), + `_report_unknown_products` (= `report_unknown_products`), + `_add_product_categories` (= `add_product_categories`), + `_extract_unit_capacity` (= `extract_unit_capacity`). +- `src/a4d/clean/schema_product.py` — 20-column meta schema, docstring says + it "matches the R pipeline's `preparing_product_fields()`" in + `script3_create_table_product_data.R`; field list/order does match. +- `src/a4d/reference/products.py` — `load_known_products` / + `load_product_categories`, replaces `load_product_reference_data`; its + docstrings explicitly say "Covers R step 2.19" / "2.20". + `sanitize_column_name` → covered by `extract/common.py` (shared with + patient's column-name sanitizer). +- `src/a4d/tables/product.py` — `create_table_product_data` (= + R's `create_table_product_data`/`preparing_product_fields`, applies + `fix_patient_id` to `product_released_to` exactly as R applies `fix_id`) + and `link_product_patient` (= R's `script3_link_product_patient.R`, with + one **documented** deviation — see below). + +**Verdict**: every R product function has a traceable Python counterpart; no +function-level gap found. This is a stronger result than ticket 07's premise +implied ("neither status has actually been audited") — the coverage itself +looks complete at the function-inventory level. What's unverified is +*behavioral* equivalence (that's explicitly out of scope for this ticket — +belongs to output-vs-source validation). + +One genuine coverage gap: `calculate_most_frequent` and +`report_empty_intersections` in `script3_create_table_product_data.R` (used +for cross-tab diagnostic logging, not core cleaning logic) have no obvious +Python counterpart — `rg -n "most_frequent|empty_intersection" src/a4d +--glob '!*.pyc'` against `origin/product-pipeline` returns nothing. These are +logging/diagnostic helpers, not data-shape-affecting logic, so low severity, +but they are a real, unmapped gap. + +### Divergence documentation: found vs. only-in-diff + +Where Python and R intentionally diverge, documentation quality is high on +the product side and mixed on the patient side: + +- `src/a4d/tables/product.py:link_product_patient` docstring: "Mirrors R's + `script3_link_product_patient.R` with one deviation: rows where + `product_released_to` is null or equals the ... sentinel ... are filtered + out before joining." — **documented in the docstring itself**, not just + discoverable via diff. +- `src/a4d/clean/product.py` module-level constants (`BUDDHIST_ERA_THRESHOLD`, + `YEAR_FLOOR_DELTA`, `PRODUCT_DATE_NA_MARKERS`, `MIN_PLAUSIBLE_EXCEL_SERIAL`, + lines 42-75) carry multi-line comments explaining exactly which R quirk + each guard works around and why. **Documented in code.** +- `docs/archive/PYTHON_IMPROVEMENTS.md` on `origin/product-pipeline` (not + on `migration` — see §3) has a full section (§5, "Product Pipeline: Date + Parsing Robustness") with per-issue R-vs-Python behavior tables, row + counts, and affected tracker names for the "Sept" abbreviation bug, the + `D/M/YYYY` bug, and the malformed-date sentinel guard. **Documented**, but + the cited primary evidence (`Ali_internship/residual_dig.ipynb`) does not + exist in the tracked tree on either branch — confirmed via + `git ls-tree -r --name-only HEAD | grep -i ali_internship` and the same + against `origin/product-pipeline`, both empty, and `find . -iname + '*residual_dig*'` empty. This reconfirms the prior session's finding: the + row-level counts in that section (e.g. "46 rows across 28 groups") are + asserted prose with no reproducible artifact backing them. +- `src/a4d/validate/source_vs_output_product.py` module docstring: "Group- + granularity checks only in v1 ... which we deliberately avoid (see the + plan at `C:/Users/furin/.claude/plans/output-vs-source-validation-swirling- + nygaard.md`)." **Documented, but the cited plan file is a contributor's + local Windows path, not in this repo** — same non-reproducibility problem + as the notebook citation above. +- Patient-side divergences (R bugs Python correctly avoids) are documented + only in `tests/test_integration/test_r_validation.py`'s + `ACCEPTABLE_DIFFERENCES` / `FILE_COLUMN_EXCEPTIONS` / + `PATIENT_LEVEL_EXCEPTIONS` dicts (42 tracker-level entries, 15 with + `"reason"` strings) — i.e. only inside a slow/USB-drive-gated test file, + not in `docs/archive/PYTHON_IMPROVEMENTS.md`'s "Summary" table on + `migration`, which lists only 3 items (insulin_subtype typo, + insulin_total_units extraction, BMI precision). The patient + divergence-documentation is real but scattered and incomplete relative to + what's actually known (visible in the test file's exception dicts). + +## 2. Test coverage shape: patient vs product + +`git diff migration origin/product-pipeline --stat -- tests/` confirms 15 new +test files on `product-pipeline`; `migration` has 26, `product-pipeline` has +41 (`git ls-tree -r --name-only -- tests/ | wc -l`). + +### Side-by-side inventory + +| Category | Patient (migration) | Product (product-pipeline, new vs. migration) | +|---|---|---| +| Extract unit tests | `test_extract/test_patient.py` (648 lines), `test_extract/test_patient_helpers.py` (476 lines) | `test_extract/test_product.py` (385 lines), `test_extract/test_common.py` (90 lines, shared helpers) | +| Clean unit tests | `test_clean/test_patient.py` (418 lines) | `test_clean/test_product.py` (887 lines) | +| Shared clean-module tests | `test_clean/test_converters.py`, `test_clean/test_transformers.py`, `test_clean/test_validators.py` | unchanged (shared, not product-specific) | +| Tables unit tests | `test_tables/test_patient.py` (361 lines) | `test_tables/test_metadata.py` (150 lines, new), `test_tables/test_link_product_patient.py` (151 lines, new) — but **no `test_tables/test_product.py`** for `create_table_product_data` itself | +| Integration (cross-stage) | `test_integration/test_extract_integration.py` (134 lines), `test_integration/test_clean_integration.py` (133 lines) | **none** — `git diff migration origin/product-pipeline -- tests/test_integration/` is empty for every file in that directory | +| End-to-end | `test_integration/test_e2e.py` (147 lines, 4 real-tracker fixtures, `@pytest.mark.e2e`) | **none** — same empty-diff result; zero product references (`grep -ic product tests/test_integration/test_e2e.py` → 0) | +| R-comparison regression | `test_integration/test_r_validation.py` (patient-only, byte-identical across branches per prior finding — confirmed again here, `git diff` → 0 lines) | **none** | +| Source-vs-output validation (new category, not present on migration at all) | `test_validate/test_patient_source_vs_output.py` (311 lines, new) | `test_validate/test_product_source_vs_output.py` (248 lines, new) | +| State/incremental-processing tests (new module, patient- and product-agnostic) | n/a on migration (`state/` module unwired) | `test_state/{test_filter,test_integration,test_manifest,test_source}.py` (349 lines combined) | +| CLI tests | `test_cli/test_cli.py` | `test_cli/test_cli.py` (extended), `test_cli/test_force_incremental.py` (162 lines, new) | +| GCP tests | `test_gcp/{test_bigquery,test_drive,test_storage}.py` | + `test_gcp/test_select_tracker_metadata.py` (77 lines, new) | +| Pipeline orchestration unit tests | none for `pipeline/patient.py` either (shared gap, not a divergence) | none for `pipeline/product.py` (217 new lines) — only indirectly touched via mocked calls in `test_cli/test_cli.py` (`@patch("a4d.cli.run_product_pipeline")`) | + +### Concrete structural gaps found + +1. **No product integration or e2e test exists at all.** `tests/test_integration/` + is byte-identical between the two branches (verified: `git diff migration + origin/product-pipeline -- tests/test_integration/test_e2e.py + tests/test_integration/test_clean_integration.py + tests/test_integration/test_extract_integration.py + tests/test_integration/conftest.py` all return empty). Patient has a + 4-fixture parametrized e2e test asserting exact row counts against real + tracker files (`tests/test_integration/test_e2e.py:20-27`, e.g. + `("tracker_2024_penang", 174, 2024, ...)` — this "174" is a per-tracker row + count, unrelated to the "174 trackers" claim in §3). Product has nothing + structurally equivalent verifying extract→clean→table works end to end. +2. **`src/a4d/extract/wide_format.py` (145 new lines, handles the Mandalay + wide-format quirk) has zero test references anywhere in the 41-file + product-pipeline test suite.** Verified by grepping every test file's + content on that branch for `wide_format` / `handle_wide_format` — no + matches in any of the 41 files. +3. **No `test_tables/test_product.py`.** `create_table_product_data` (the + product analogue of `create_table_patient_data_static/monthly/annual`, + which do have `test_tables/test_patient.py` coverage) is only exercised + indirectly through `test_tables/test_link_product_patient.py`, which + tests `link_product_patient` — a different function in the same module. +4. **`pipeline/product.py` (217 lines, the orchestrator) has no dedicated + unit test**, mirroring the same gap that already exists for + `pipeline/patient.py` on `migration` — this one is a shared gap, not a + product-specific regression. +5. **R-comparison regression test is patient-only** (prior finding, + reconfirmed): product's only automated cross-check against ground truth + is `test_validate/test_product_source_vs_output.py`, which per its own + module docstring (`src/a4d/validate/source_vs_output_product.py:1-7`) is + "group-granularity only in v1" and deliberately does not reproduce + cleaning steps 2.0-2.5, so it cannot catch row-level or cell-level + regressions the way `test_r_validation.py` does for patient. +6. CI (`.github/workflows/python-ci.yml:47`) runs + `pytest -m "not slow and not integration"` — so even patient's + `test_r_validation.py` and `test_e2e.py` never execute in CI on either + branch; they're local-only, gated on the external USB-mounted R output + directory (`R_OUTPUT_DIR = Path("/Volumes/USB SanDisk 3.2Gen1 + Media/a4d/output_r/patient_data_cleaned")`, `test_r_validation.py:26`). + +## 3. Documentation currency: patient vs product + +### `docs/CLAUDE.md` (migration branch) + +- Module table lists only patient modules — accurate for `migration`'s + actual tree (`find src/a4d -iname '*product*'` on `migration` → nothing), + not stale, just necessarily incomplete since product isn't merged. +- **"Patient pipeline: complete, validated against 174 trackers, deployed to + production"** (line 70) — investigated in depth: + - The "174" figure is not a static assertion anywhere; it is the dynamic + count of files under an external, non-repo path + (`R_OUTPUT_DIR.glob("*_patient_cleaned.parquet")` in + `test_r_validation.py:339-350`). The docstring at the top of that file + says "for all 174 trackers" (line 4) as a comment, not an assertion the + test suite checks against a literal `174`. + - The test file itself is real and substantial: 42 tracker-level + exception entries, `PATIENT_LEVEL_EXCEPTIONS` for 3 more trackers, + `REQUIRED_COLUMN_EXCEPTIONS` for 20 trackers, each with a specific, + plausible, source-data-referencing reason string. This is not vaporware + — it's a genuine, detailed regression-test design. + - However: it is marked `pytestmark = [pytest.mark.slow, + pytest.mark.integration]`, CI explicitly excludes `slow`/`integration` + (`.github/workflows/python-ci.yml:47`), and it depends on a path under + `/Volumes/USB SanDisk 3.2Gen1 Media/...` that is not part of the repo + and not available to anyone without that physical drive. **There is no + committed artifact anywhere in the repo (log, CSV, markdown report, CI + run) recording the *result* of an actual 174-tracker run** — no + `VALIDATION_TRACKING.md` exists currently (`find . -iname + '*VALIDATION_TRACKING*'` finds nothing outside `.git` history and a + stray `a4d-python/` vendor copy from an unrelated deploy commit + `b57b483`, not the working tree). So: the *test infrastructure* for the + 174-tracker claim is real and checkable; the *pass result* itself is + not verifiable from the repo and rests entirely on prose. + - "Deployed to production": `Dockerfile` and `.github` CI exist; commit + `b57b483` ("Deploy Python pipeline as Cloud Run Job") shows Cloud Run + deployment tooling was added. No repo artifact (deploy log, GCP export) + confirms an actual successful production deployment beyond that + tooling's existence — consistent with wayfinder ticket 05 + ("production-verification-run", still `blocked`) treating this as + unverified. + +### `docs/archive/MIGRATION_GUIDE.md` and `PYTHON_IMPROVEMENTS.md` + +- `migration` branch: `MIGRATION_GUIDE.md:5` says "Phases 0–7 complete... + Product pipeline not yet started" — accurate for that branch's actual + tree (no `src/a4d/*product*` files exist there). +- `origin/product-pipeline` branch (`git show + origin/product-pipeline:docs/archive/MIGRATION_GUIDE.md`): line 5 says + "Phases 0–9 complete... Product pipeline merged into `src/a4d/` on + 2026-04-23." This is accurate *self-referentially* (that branch's tree + really does contain the product modules, confirmed by the `git diff + --stat` file list in §1) but the phrase "merged into `src/a4d/`" could + mislead a reader into thinking it's merged into `migration`/`dev` — it + is not (`git branch -a` shows `product-pipeline` only as a remote branch, + ticket 03 "Merge product-pipeline (PR #6) into migration" is still open). +- `PYTHON_IMPROVEMENTS.md` diff (`git diff migration origin/product-pipeline + -- docs/archive/PYTHON_IMPROVEMENTS.md`, 87 lines added) adds §5/§6 for + product date-parsing and running-balance precision, plus updates the + "Migration Validation Status" summary to declare "production-ready" and + "All value differences are Python improvements over R bugs or negligible + precision drift." This blanket claim is not fully supported: the + underlying evidence for the date-parsing row counts is the + non-existent `Ali_internship/residual_dig.ipynb` (see §1), and the file + itself provides no link to a reproducible script that regenerates those + numbers — i.e. the qualitative direction of each divergence (Python + correct, R buggy) is plausible and mostly consistent with what the code + comments describe, but the specific row/group counts asserted in the + document are not currently checkable against anything in the repo. + +### Module docstrings + +Product module docstrings (`clean/product.py`, `extract/product.py`, +`reference/products.py`, `tables/product.py`, `validate/ +source_vs_output_product.py`) consistently cite specific R script names and +step numbers ("Covers R Script 2 steps 2.1-2.22", "Covers R steps 3.1-3.3", +"Covers R step 2.19"). These are checkable claims — cross-referencing them +against `r-archive/R/` (§1) shows they're accurate. This is a stronger +documentation practice than exists on the patient side, where +`clean/patient.py` does not carry an equivalent "covers R steps X-Y" header +comment mapping its private helpers to R script line ranges. + +## 4. Concrete gap list + +### Product — before "complete" + +1. Add `tests/test_integration/` coverage: an extract→clean cross-stage + test and an e2e test for product, parallel to patient's + `test_extract_integration.py` / `test_clean_integration.py` / + `test_e2e.py`. Currently zero. +2. Add tests for `src/a4d/extract/wide_format.py` — currently zero + references across all 41 test files. +3. Add `tests/test_tables/test_product.py` covering + `create_table_product_data` directly (currently only reachable via the + `link_product_patient` test file). +4. Either replace or supplement the "group-granularity only" product + source-vs-output validator with a cell-level regression check (its own + docstring already flags this as a known, deliberate v1 limitation), or + document why group-granularity is considered sufficient long-term. +5. Fix the two dangling evidence citations: `Ali_internship/ + residual_dig.ipynb` (cited in `PYTHON_IMPROVEMENTS.md` §5) and the + Windows-local plan path (cited in `source_vs_output_product.py`'s module + docstring) — neither exists in the repo; either commit the artifacts or + replace the citations with reproducible-in-repo evidence. +6. `calculate_most_frequent` / `report_empty_intersections` (R diagnostic + helpers) have no Python equivalent — confirm this is an intentional drop + (they appear to be diagnostic-only, not data-shape-affecting) and note + it explicitly rather than leaving it silent. +7. Merge is still blocked on wayfinder tickets 03/04/05/06 (CI red at + `migration` HEAD, no real GCP verification run) — this audit doesn't + resolve those, just confirms the code/test/doc state feeding into them. + +### Patient — before "complete" (despite being the more mature side) + +1. The "174 trackers validated" claim in `docs/CLAUDE.md` has real, + detailed test infrastructure behind it (`test_r_validation.py`) but no + committed record of an actual passing run — it is `slow`/`integration`, + excluded from CI, and gated on an external USB drive. Either commit a + dated validation report (a script-generated one, per the wayfinder map's + "no notebooks, ever" / automated-report preference) each time this is + run, or soften the docs claim to reflect that it's manually-verified, + not continuously verified. +2. `docs/archive/PYTHON_IMPROVEMENTS.md`'s "Summary" table on `migration` + lists only 3 known R/Python divergences, while `test_r_validation.py`'s + exception dictionaries document ~15+ distinct, reasoned divergences + (province validation, Unicode ≥/≤ handling, duplicate patient IDs, + systematic R extraction failures on several 2025 trackers, etc.) that + never made it into the human-readable doc — someone reading + `PYTHON_IMPROVEMENTS.md` alone would materially undercount the known + differences. +3. No dedicated unit test exists for `pipeline/patient.py` (the + orchestrator) — same category of gap product also has for + `pipeline/product.py`; worth deciding once, not twice, for both. +4. "Deployed to production" is asserted in `docs/CLAUDE.md` without a + corresponding checkable artifact in the repo (matches wayfinder ticket + 05's own framing of this as still open/unverified). + +## Commands used (for reproducibility) + +``` +git branch -a +git log origin/product-pipeline -5 --oneline +git diff migration origin/product-pipeline --stat +git diff migration origin/product-pipeline --stat -- tests/ +git ls-tree -r --name-only HEAD -- tests/ | sort +git ls-tree -r --name-only origin/product-pipeline -- tests/ | sort +git diff migration origin/product-pipeline -- tests/test_integration/test_e2e.py ... +git show origin/product-pipeline: # read files on that branch +git diff migration origin/product-pipeline -- docs/archive/PYTHON_IMPROVEMENTS.md +git diff migration origin/product-pipeline -- docs/archive/MIGRATION_GUIDE.md # via saved copy + diff +rg -n "^[a-zA-Z_.]+\s*<-\s*function" r-archive/R/helper_product_data.R (and other R files) +rg -n "^def |^class " +find . -iname "*residual_dig*" ; git ls-tree -r --name-only HEAD | grep -i ali_internship +find . -iname "*VALIDATION_TRACKING*" -not -path "./.git/*" +grep -n "slow\|pytest" .github/workflows/python-ci.yml +``` diff --git a/docs/wayfinder/route.mmd b/docs/wayfinder/route.mmd new file mode 100644 index 0000000..44d5229 --- /dev/null +++ b/docs/wayfinder/route.mmd @@ -0,0 +1,364 @@ +flowchart TB + subgraph S2026_08_08["Session 2026-08-08"] + direction LR + U1["1
Does product-pipeline's
test suite meet the same
cell-by-cell rigor as
patient's?"] + U7["7
Is the product pipeline
(and patient's own
claimed completeness)
actually complete and
sound, audited against
R's product logic and
patient's structure?"] + U8["8
Does the pytest suite
reach unit/integration/e
2e/regression parity
between patient and
product, excluding any
R-comparison/USB-drive-
dependent tests?"] + end + subgraph S2026_08_08b["Session 2026-08-08b"] + direction LR + U3["3
Merge product-pipeline
(PR #6) into migration"] + end + subgraph S2026_08_09["Session 2026-08-09"] + direction LR + U4["4
Diagnose and fix why CI
is red at migration HEAD"] + end + subgraph S2026_08_09b["Session 2026-08-09b"] + direction LR + U5["5
Define and execute the
real GCP production
verification run"] + end + subgraph S2026_08_09c["Session 2026-08-09c"] + direction LR + U13["13
Audit and update all
dependencies and library
versions before rollout"] + end + subgraph S2026_08_09d["Session 2026-08-09d"] + direction LR + U10["10
Profile the combined
pipeline's performance
against the R baseline
before promoting to dev"] + end + subgraph S2026_08_09e["Session 2026-08-09e"] + direction LR + U14["14
Fix product pipeline's
unable to find column
product failures on 4
real trackers"] + end + subgraph S2026_08_09f["Session 2026-08-09f"] + direction LR + U2["2
Retire the PDF/notebook
analysis docs for an
automated, script-based
report"] + end + subgraph S2026_08_09g["Session 2026-08-09g"] + direction LR + U11["11
Decide what CLI/TUI UX
and error-log
observability
improvements
admins/developers need
before rollout"] + end + subgraph S2026_08_10["Session 2026-08-10"] + direction LR + U15["15
Build and run the
R/Python output
comparison script, then
triage every flagged
difference"] + end + subgraph S2026_08_11["Session 2026-08-11"] + direction LR + U17["17
Fix the product
comparison's row-
alignment key, then
triage every flagged
R/Python difference"] + U18["18
Triage every flagged
R/Python difference for
both arms, and resolve
the 189-vs-155-tracker
discrepancy"] + U19["19
Persist comparison run
history and show run-
over-run deltas"] + end + subgraph S2026_08_12["Session 2026-08-12"] + direction LR + U20["20
Normalize the raw-stage
product_entry_date
comparison so it stops
flagging near-universal
false mismatches"] + U21["21
Triage the remaining
product cleaned-stage
column mismatches
(balance, received_from,
released_to, remarks,
units_received, product)"] + U22["22
Triage the remaining
product raw-stage column
mismatches"] + end + subgraph S2026_08_12b["Session 2026-08-12b"] + direction LR + U23["23
Triage every flagged
R/Python difference for
the patient arm (raw and
cleaned)"] + end + subgraph S2026_08_12c["Session 2026-08-12c"] + direction LR + U28["28
Triage the patient
cleaned-stage column
mismatches"] + end + subgraph S2026_08_12d["Session 2026-08-12d"] + direction LR + U26["26
Triage the product
pipeline's column-
existence and dtype
divergence (Column
divergence)"] + end + subgraph S2026_08_12e["Session 2026-08-12e"] + direction LR + U24["24
Triage the residual prod
uct_units_received/produ
ct_units_released/produc
t_received_from raw-
stage mismatches"] + end + subgraph S2026_08_12f["Session 2026-08-12f"] + direction LR + U27["27
Triage the residual
patient raw-stage column
mismatches after date
normalization"] + end + subgraph S2026_08_12g["Session 2026-08-12g"] + direction LR + U33["33
Fix red CI — ruff format
--check fails on Python
snippets inside markdown
docs"] + end + subgraph S2026_08_12h["Session 2026-08-12h"] + direction LR + U25["25
Triage the
product_units_released
cleaned-stage column
mismatches"] + end + subgraph S2026_08_13["Session 2026-08-13"] + direction LR + U36["36
Triage the product
cleaned-stage mismatches
no ticket owns
(product_balance,
sheet_name, entry_date,
units_received,
file_name)"] + end + subgraph S2026_08_13b["Session 2026-08-13b"] + direction LR + U29["29
Triage the residual
patient cleaned-stage
column mismatches"] + end + subgraph S2026_08_14["Session 2026-08-14"] + direction LR + U37["37
Triage the residual
patient cleaned-stage
mismatches (round 2)"] + end + subgraph S2026_08_14b["Session 2026-08-14b"] + direction LR + U38["38
Triage the patient
cleaned-stage date-
column family (round 3)"] + end + subgraph S2026_08_15["Session 2026-08-15"] + direction LR + U30["30
Triage the patient
pipeline's raw-stage
column-existence
divergence"] + end + subgraph S2026_08_15b["Session 2026-08-15b"] + direction LR + U31["31
Triage the residual
patient raw-stage column
mismatches (round 2)"] + end + subgraph S2026_08_17["Session 2026-08-17"] + direction LR + U42["42
Decide how FBG unit
headers are resolved,
and what to do about
physiologically
implausible mmol values"] + end + subgraph S2026_08_17b["Session 2026-08-17b"] + direction LR + U43["43
Triage the residual
patient raw-stage column
mismatches (round 3)"] + end + subgraph S2026_08_17c["Session 2026-08-17c"] + direction LR + U45["45
Give the patient
comparison an ordinal
row key, so duplicated
patient IDs stop faking
mismatches"] + end + subgraph S2026_08_17d["Session 2026-08-17d"] + direction LR + U46["46
Triage the residual
patient raw-stage column
mismatches (round 4)"] + end + subgraph S2026_08_17e["Session 2026-08-17e"] + direction LR + U48["48
Python drops
complication-screening
results and dates where
a merged header spans
the block"] + end + subgraph S2026_08_18["Session 2026-08-18"] + direction LR + U49["49
Triage the residual
patient raw-stage column
mismatches (round 5)"] + end + subgraph S2026_08_18b["Session 2026-08-18b"] + direction LR + U50["50
Triage the residual
patient raw-stage column
mismatches (round 6)"] + end + subgraph S2026_08_19["Session 2026-08-19"] + direction LR + U51["51
Triage the residual
patient cleaned-stage
mismatches (round 4)"] + end + subgraph S2026_08_19b["Session 2026-08-19b"] + direction LR + U52["52
Triage the residual
patient cleaned-stage
mismatches (round 5)"] + end + subgraph S2026_08_19c["Session 2026-08-19c"] + direction LR + U53["53
Triage the residual
patient cleaned-stage
mismatches (round 6)"] + end + subgraph S2026_08_19d["Session 2026-08-19d"] + direction LR + U54["54
Triage the residual
patient cleaned-stage
mismatches (round 7)"] + end + subgraph S2026_08_19e["Session 2026-08-19e"] + direction LR + U55["55
Triage the residual
patient cleaned-stage
mismatches (round 8)"] + end + subgraph S2026_08_19f["Session 2026-08-19f"] + direction LR + U56["56
Triage the residual
patient cleaned-stage
mismatches (round 9)"] + end + subgraph S2026_08_20["Session 2026-08-20"] + direction LR + U57["57
Triage the residual
patient cleaned-stage
mismatches (round 10)"] + end + subgraph S2026_08_20b["Session 2026-08-20b"] + direction LR + U47["47
Four trackers where
cleaning merges several
patients into one
patient ID"] + end + subgraph S2026_08_21["Session 2026-08-21"] + direction LR + U39["39
Decide whether a date
buried inside a clinical
note should be recovered
or discarded"] + end + subgraph S2026_08_22["Session 2026-08-22"] + direction LR + U32["32
Re-audit every existing
cause classifier — is
Python actually right,
or was the diff merely
labelled?"] + end + subgraph S2026_08_22b["Session 2026-08-22b"] + direction LR + U61["61
Decide whether a Thai
clinic's Buddhist-era
entry date is published
as 2567 or converted to
2024"] + end + subgraph S2026_08_24["Session 2026-08-24"] + direction LR + U59["59
Rows that pair with
nothing on the other
side, which no ticket
has ever triaged"] + end + subgraph S2026_08_24b["Session 2026-08-24b"] + direction LR + U60["60
Audit the eight pre-bar
causes the first
classifier pass did not
reach"] + end + subgraph S2026_08_24c["Session 2026-08-24c"] + direction LR + U62["62
Finish the pre-bar
classifier audit — the
two causes and the one
bulk population it did
not reach"] + end + subgraph S2026_08_24d["Session 2026-08-24d"] + direction LR + U58["58
Monthly rows with a
misspelled ID silently
lose their Patient List
demographics"] + end + subgraph S2026_08_24e["Session 2026-08-24e"] + direction LR + U63["63
The cleaned stage has
4,949 cells with no
cause, because the ID
spelling that explains
them is gone by then"] + end + subgraph S2026_08_24f["Session 2026-08-24f"] + direction LR + U12["12
Retire R from the
workspace once the
pipeline is fully
verified Python-only"] + end + subgraph S2026_08_24g["Session 2026-08-24g"] + direction LR + U64["64
Rewrite every docstring
and doc that explains
the code by what R did"] + end + subgraph Sunworked["Closed without being worked"] + direction LR + U44["44
Classify the cleaned-
stage FBG cells where R
has nothing and Python
has a corrected reading"] + end + subgraph Sopen["Not yet worked"] + direction LR + U6["6
Promote migration into
dev via PR #2"] + U9["9
Add golden-
master/snapshot
regression tests for
patient and product"] + U16["16
Build a drill-down log
analyzer for admins to
inspect a specific
tracker file's
errors/logs"] + U34["34
Make the local pre-push
check set actually match
CI, and make running it
automatic"] + U35["35
Resolve the Polars 2.0
deprecation warnings —
decide the behaviour
each one is asking about"] + U40["40
Produce one Excel of
every source-tracker
defect, so the trackers
themselves can be
corrected"] + U41["41
Decide whether the 2026
template's five new
Patient List fields
enter the pipeline"] + U65["65
Two values published
into the logs table
still name R scripts"] + end + + S2026_08_08 ~~~ S2026_08_08b + S2026_08_08b ~~~ S2026_08_09 + S2026_08_09 ~~~ S2026_08_09b + S2026_08_09b ~~~ S2026_08_09c + S2026_08_09c ~~~ S2026_08_09d + S2026_08_09d ~~~ S2026_08_09e + S2026_08_09e ~~~ S2026_08_09f + S2026_08_09f ~~~ S2026_08_09g + S2026_08_09g ~~~ S2026_08_10 + S2026_08_10 ~~~ S2026_08_11 + S2026_08_11 ~~~ S2026_08_12 + S2026_08_12 ~~~ S2026_08_12b + S2026_08_12b ~~~ S2026_08_12c + S2026_08_12c ~~~ S2026_08_12d + S2026_08_12d ~~~ S2026_08_12e + S2026_08_12e ~~~ S2026_08_12f + S2026_08_12f ~~~ S2026_08_12g + S2026_08_12g ~~~ S2026_08_12h + S2026_08_12h ~~~ S2026_08_13 + S2026_08_13 ~~~ S2026_08_13b + S2026_08_13b ~~~ S2026_08_14 + S2026_08_14 ~~~ S2026_08_14b + S2026_08_14b ~~~ S2026_08_15 + S2026_08_15 ~~~ S2026_08_15b + S2026_08_15b ~~~ S2026_08_17 + S2026_08_17 ~~~ S2026_08_17b + S2026_08_17b ~~~ S2026_08_17c + S2026_08_17c ~~~ S2026_08_17d + S2026_08_17d ~~~ S2026_08_17e + S2026_08_17e ~~~ S2026_08_18 + S2026_08_18 ~~~ S2026_08_18b + S2026_08_18b ~~~ S2026_08_19 + S2026_08_19 ~~~ S2026_08_19b + S2026_08_19b ~~~ S2026_08_19c + S2026_08_19c ~~~ S2026_08_19d + S2026_08_19d ~~~ S2026_08_19e + S2026_08_19e ~~~ S2026_08_19f + S2026_08_19f ~~~ S2026_08_20 + S2026_08_20 ~~~ S2026_08_20b + S2026_08_20b ~~~ S2026_08_21 + S2026_08_21 ~~~ S2026_08_22 + S2026_08_22 ~~~ S2026_08_22b + S2026_08_22b ~~~ S2026_08_24 + S2026_08_24 ~~~ S2026_08_24b + S2026_08_24b ~~~ S2026_08_24c + S2026_08_24c ~~~ S2026_08_24d + S2026_08_24d ~~~ S2026_08_24e + S2026_08_24e ~~~ S2026_08_24f + S2026_08_24f ~~~ S2026_08_24g + S2026_08_24g ~~~ Sunworked + Sunworked ~~~ Sopen + + U3 --->|blocked| U2 + U8 --->|blocked| U3 + U3 --->|blocked| U4 + U3 --->|blocked| U5 + U4 --->|blocked| U5 + U8 --->|blocked| U6 + U3 --->|blocked| U6 + U4 --->|blocked| U6 + U5 --->|blocked| U6 + U10 --->|blocked| U6 + U11 --->|blocked| U6 + U12 --->|blocked| U6 + U13 --->|blocked| U6 + U14 --->|blocked| U6 + U20 --->|blocked| U6 + U21 --->|blocked| U6 + U22 --->|blocked| U6 + U23 --->|blocked| U6 + U64 --->|blocked| U6 + U1 -.->|spawned| U7 + U1 -.->|spawned| U8 + U7 --->|blocked| U8 + U8 -.->|spawned| U9 + U6 --->|blocked| U9 + U3 --->|blocked| U10 + U13 --->|blocked| U10 + U3 --->|blocked| U11 + U3 --->|blocked| U13 + U10 -.->|spawned| U14 + U2 -.->|spawned| U15 + U2 --->|blocked| U15 + U11 -.->|spawned| U16 + U15 -.->|spawned| U17 + U17 -.->|spawned| U18 + U17 -.->|spawned| U19 + U18 -.->|spawned| U20 + U18 -.->|spawned| U21 + U18 -.->|spawned| U22 + U18 -.->|spawned| U23 + U22 -.->|spawned| U24 + U21 -.->|spawned| U25 + U23 -.->|spawned| U27 + U23 -.->|spawned| U28 + U28 -.->|spawned| U29 + U26 -.->|spawned| U30 + U27 -.->|spawned| U31 + U27 -.->|spawned| U32 + U33 -.->|spawned| U34 + U33 -.->|spawned| U35 + U25 -.->|spawned| U36 + U29 -.->|spawned| U37 + U37 -.->|spawned| U38 + U38 -.->|spawned| U39 + U30 -.->|spawned| U40 + U30 -.->|spawned| U41 + U30 -.->|spawned| U42 + U31 -.->|spawned| U43 + U42 -.->|spawned| U44 + U43 -.->|spawned| U45 + U43 -.->|spawned| U46 + U45 --->|blocked| U46 + U45 -.->|spawned| U47 + U46 -.->|spawned| U48 + U46 -.->|spawned| U49 + U49 -.->|spawned| U50 + U50 -.->|spawned| U51 + U51 -.->|spawned| U52 + U52 -.->|spawned| U53 + U53 -.->|spawned| U54 + U54 -.->|spawned| U55 + U55 -.->|spawned| U56 + U56 -.->|spawned| U57 + U47 -.->|spawned| U58 + U47 -.->|spawned| U59 + U32 -.->|spawned| U60 + U32 -.->|spawned| U61 + U60 -.->|spawned| U62 + U58 -.->|spawned| U63 + U12 -.->|spawned| U64 + U64 -.->|spawned| U65 + + classDef tfrontier fill:#1f6feb,stroke:#0b3d91,stroke-width:3px,color:#ffffff + class U6,U16,U34,U35,U40,U41,U65 tfrontier + classDef tblocked fill:#6e7781,stroke:#424a53,stroke-width:1px,color:#ffffff + class U9 tblocked + classDef tdecided fill:#1a7f37,stroke:#116329,stroke-width:1px,color:#ffffff + class U2,U3,U4,U5,U7,U8,U10,U11,U12,U13,U14,U15,U17,U18,U19,U20,U21,U22,U23,U24,U25,U26,U27,U28,U29,U30,U31,U32,U33,U36,U37,U38,U39,U42,U43,U44,U45,U46,U47,U48,U49,U50,U51,U52,U53,U54,U55,U56,U57,U58,U59,U60,U61,U62,U63,U64 tdecided + classDef tdropped fill:#eaeef2,stroke:#afb8c1,stroke-width:1px,color:#57606a + class U1 tdropped diff --git a/docs/wayfinder/route.svg b/docs/wayfinder/route.svg new file mode 100644 index 0000000..ee7040e --- /dev/null +++ b/docs/wayfinder/route.svg @@ -0,0 +1 @@ +

blocked

spawned

blocked

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

blocked

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

spawned

blocked

spawned

spawned

blocked

blocked

blocked

spawned

spawned

spawned

spawned

spawned

spawned

spawned

blocked

spawned

spawned

blocked

blocked

blocked

spawned

blocked

blocked

blocked

blocked

blocked

blocked

blocked

blocked

blocked

blocked

blocked

blocked

blocked

spawned

blocked

Not yet worked

blocked

6
Promote migration into
dev via PR #2

9
Add golden-
master/snapshot
regression tests for
patient and product

16
Build a drill-down log
analyzer for admins to
inspect a specific
tracker file's
errors/logs

34
Make the local pre-push
check set actually match
CI, and make running it
automatic

35
Resolve the Polars 2.0
deprecation warnings —
decide the behaviour
each one is asking about

40
Produce one Excel of
every source-tracker
defect, so the trackers
themselves can be
corrected

41
Decide whether the 2026
template's five new
Patient List fields
enter the pipeline

65
Two values published
into the logs table
still name R scripts

Closed without being worked

44
Classify the cleaned-
stage FBG cells where R
has nothing and Python
has a corrected reading

Session 2026-08-24g

64
Rewrite every docstring
and doc that explains
the code by what R did

Session 2026-08-24f

12
Retire R from the
workspace once the
pipeline is fully
verified Python-only

Session 2026-08-24e

63
The cleaned stage has
4,949 cells with no
cause, because the ID
spelling that explains
them is gone by then

Session 2026-08-24d

58
Monthly rows with a
misspelled ID silently
lose their Patient List
demographics

Session 2026-08-24c

62
Finish the pre-bar
classifier audit — the
two causes and the one
bulk population it did
not reach

Session 2026-08-24b

60
Audit the eight pre-bar
causes the first
classifier pass did not
reach

Session 2026-08-24

59
Rows that pair with
nothing on the other
side, which no ticket
has ever triaged

Session 2026-08-22b

61
Decide whether a Thai
clinic's Buddhist-era
entry date is published
as 2567 or converted to
2024

Session 2026-08-22

32
Re-audit every existing
cause classifier — is
Python actually right,
or was the diff merely
labelled?

Session 2026-08-21

39
Decide whether a date
buried inside a clinical
note should be recovered
or discarded

Session 2026-08-20b

47
Four trackers where
cleaning merges several
patients into one
patient ID

Session 2026-08-20

57
Triage the residual
patient cleaned-stage
mismatches (round 10)

Session 2026-08-19f

56
Triage the residual
patient cleaned-stage
mismatches (round 9)

Session 2026-08-19e

55
Triage the residual
patient cleaned-stage
mismatches (round 8)

Session 2026-08-19d

54
Triage the residual
patient cleaned-stage
mismatches (round 7)

Session 2026-08-19c

53
Triage the residual
patient cleaned-stage
mismatches (round 6)

Session 2026-08-19b

52
Triage the residual
patient cleaned-stage
mismatches (round 5)

Session 2026-08-19

51
Triage the residual
patient cleaned-stage
mismatches (round 4)

Session 2026-08-18b

50
Triage the residual
patient raw-stage column
mismatches (round 6)

Session 2026-08-18

49
Triage the residual
patient raw-stage column
mismatches (round 5)

Session 2026-08-17e

48
Python drops
complication-screening
results and dates where
a merged header spans
the block

Session 2026-08-17d

46
Triage the residual
patient raw-stage column
mismatches (round 4)

Session 2026-08-17c

45
Give the patient
comparison an ordinal
row key, so duplicated
patient IDs stop faking
mismatches

Session 2026-08-17b

43
Triage the residual
patient raw-stage column
mismatches (round 3)

Session 2026-08-17

42
Decide how FBG unit
headers are resolved,
and what to do about
physiologically
implausible mmol values

Session 2026-08-15b

31
Triage the residual
patient raw-stage column
mismatches (round 2)

Session 2026-08-15

30
Triage the patient
pipeline's raw-stage
column-existence
divergence

Session 2026-08-14b

38
Triage the patient
cleaned-stage date-
column family (round 3)

Session 2026-08-14

37
Triage the residual
patient cleaned-stage
mismatches (round 2)

Session 2026-08-13b

29
Triage the residual
patient cleaned-stage
column mismatches

Session 2026-08-13

36
Triage the product
cleaned-stage mismatches
no ticket owns
(product_balance,
sheet_name, entry_date,
units_received,
file_name)

Session 2026-08-12h

25
Triage the
product_units_released
cleaned-stage column
mismatches

Session 2026-08-12g

33
Fix red CI — ruff format
--check fails on Python
snippets inside markdown
docs

Session 2026-08-12f

27
Triage the residual
patient raw-stage column
mismatches after date
normalization

Session 2026-08-12e

24
Triage the residual prod
uct_units_received/produ
ct_units_released/produc
t_received_from raw-
stage mismatches

Session 2026-08-12d

26
Triage the product
pipeline's column-
existence and dtype
divergence (Column
divergence)

Session 2026-08-12c

28
Triage the patient
cleaned-stage column
mismatches

Session 2026-08-12b

23
Triage every flagged
R/Python difference for
the patient arm (raw and
cleaned)

Session 2026-08-12

20
Normalize the raw-stage
product_entry_date
comparison so it stops
flagging near-universal
false mismatches

21
Triage the remaining
product cleaned-stage
column mismatches
(balance, received_from,
released_to, remarks,
units_received, product)

22
Triage the remaining
product raw-stage column
mismatches

Session 2026-08-11

spawned

spawned

17
Fix the product
comparison's row-
alignment key, then
triage every flagged
R/Python difference

18
Triage every flagged
R/Python difference for
both arms, and resolve
the 189-vs-155-tracker
discrepancy

19
Persist comparison run
history and show run-
over-run deltas

Session 2026-08-10

15
Build and run the
R/Python output
comparison script, then
triage every flagged
difference

Session 2026-08-09g

11
Decide what CLI/TUI UX
and error-log
observability
improvements
admins/developers need
before rollout

Session 2026-08-09f

2
Retire the PDF/notebook
analysis docs for an
automated, script-based
report

Session 2026-08-09e

14
Fix product pipeline's
unable to find column
product failures on 4
real trackers

Session 2026-08-09d

10
Profile the combined
pipeline's performance
against the R baseline
before promoting to dev

Session 2026-08-09c

13
Audit and update all
dependencies and library
versions before rollout

Session 2026-08-09b

5
Define and execute the
real GCP production
verification run

Session 2026-08-09

4
Diagnose and fix why CI
is red at migration HEAD

Session 2026-08-08b

3
Merge product-pipeline
(PR #6) into migration

Session 2026-08-08

spawned

spawned

blocked

1
Does product-pipeline's
test suite meet the same
cell-by-cell rigor as
patient's?

7
Is the product pipeline
(and patient's own
claimed completeness)
actually complete and
sound, audited against
R's product logic and
patient's structure?

8
Does the pytest suite
reach unit/integration/e
2e/regression parity
between patient and
product, excluding any
R-comparison/USB-drive-
dependent tests?

 \ No newline at end of file diff --git a/docs/wayfinder/tickets/01-product-pipeline-test-rigor.md b/docs/wayfinder/tickets/01-product-pipeline-test-rigor.md new file mode 100644 index 0000000..bf6edb4 --- /dev/null +++ b/docs/wayfinder/tickets/01-product-pipeline-test-rigor.md @@ -0,0 +1,87 @@ +--- +id: 1 +title: Does product-pipeline's test suite meet the same cell-by-cell rigor as patient's? +labels: [wayfinder:grilling] +status: closed +blocked_by: [] +assignee: session-2026-08-08 +claimed_at: 2026-08-08 +resolution: superseded +evidence: judgement +closed_by: null +spawned_by: null +--- + +## Premise + +Nothing closed yet — this is one of the map's first tickets. + +Facts on record (verified via `git`, not the intern's word): `origin/product-pipeline` +carries 41 test files vs. `migration`'s 26, including `tests/test_clean/test_product.py`, +`tests/test_state/*`, `tests/test_validate/*`, and a source-vs-output reconciliation +module (`src/a4d/validate/source_vs_output_product.py`). PR #6 +(`product-pipeline` -> `migration`) is open with its test-plan checkboxes all +unchecked and no CI runs recorded. + +## Question + +The patient pipeline was verified against 174 trackers, cell-by-cell — shape, +columns, exact values (see Migration Guide Phase 7). Does the product pipeline's +test suite + `source_vs_output_product.py` reconciliation reach the same bar, or +is there a gap (fewer trackers covered, spot-checks instead of exhaustive +comparison, missing edge cases)? What, concretely, is missing if anything? + +## Resolution (superseded) + +**Decision:** Not answered as framed. The question presupposed patient's +`test_r_validation.py` (174-tracker, hand-written exception dicts, requires +the USB drive) *is* the rigor bar to replicate for product, and that R-vs-Python +sameness is the goal. Both are wrong, established this session with the user: + +1. R-parity isn't the goal — Python may correctly diverge from R (R can be + wrong); the original source Excel trackers are the arbiter when R and + Python disagree, not R's output alone. +2. Comparing R vs. Python output is *analysis* (a judgment call: why do they + differ, is it safe to trust), not a *test* (pass/fail assertion) — so it + doesn't belong in pytest at all. This applies equally to patient's existing + `test_r_validation.py`, which is itself miscategorized: `@pytest.mark.slow` + + `@pytest.mark.integration`, hardcoded to the USB drive path, not part of + a normal `uv run pytest`. +3. Before any test-rigor or output-parity question can be answered, the more + basic question — is the product pipeline actually complete, and is + patient's own claimed completeness substantiated — needs answering first. + +**Because:** Verified via `git`: `test_r_validation.py` is byte-identical +between `migration` and `origin/product-pipeline` (empty diff) — patient-only, +never extended for product. `source_vs_output_product.py`'s own docstring +states "Group-granularity checks only in v1: per-cell checks would require +reproducing cleaning steps 2.0-2.5 ... which we deliberately avoid" — group +granularity, not cell-by-cell, by explicit design. `PYTHON_IMPROVEMENTS.md`'s +"Schema: 100% match", "production-ready" claims (product-pipeline branch) +cite `Ali_internship/residual_dig.ipynb` (not in the tracked tree) and +`scripts/compare_r_vs_python.py` (patient-only per its own docstring/paths) +— i.e. one-off notebook/script analysis, not a repeatable pytest suite. + +**Rejected:** +- Replicating patient's exception-dict pattern for product — rejected: not a + test if sameness isn't the goal, and the user already found this pattern + too time-consuming for patient alone. +- A diff-logging pytest suite (log every difference, flag new ones) — + rejected by the user: still miscategorizes analysis as test, and doesn't + surface *why* outputs differ, which is what a go-live trust decision needs. +- Answering the ticket as originally scoped — rejected: it assumes facts + (product pipeline is otherwise complete) that were never established, and + conflates two different activities (pytest-suite parity vs. R-trust + analysis) that need separate treatment. + +**Evidence:** judgement (the categorical split and the not-1:1-parity +principle are user decisions; the supporting facts above were verified via +`git diff`/`git show`, i.e. read/executed, but the resolution as a whole is +judgement — stamped at the weaker level). + +**Tense:** current — all cited facts describe the state of `migration` and +`origin/product-pipeline` as of 2026-08-08, not a future/proposed state. + +**Spawned:** +[Is the product pipeline (and patient's own claimed completeness) actually complete and sound, audited against R's product logic and patient's structure?](07-pipeline-completeness-audit.md), +[Does the pytest suite reach unit/integration/e2e/regression parity between patient and product, excluding any R-comparison/USB-drive-dependent tests?](08-pytest-suite-parity.md) diff --git a/docs/wayfinder/tickets/02-documentation-strategy.md b/docs/wayfinder/tickets/02-documentation-strategy.md new file mode 100644 index 0000000..27b441a --- /dev/null +++ b/docs/wayfinder/tickets/02-documentation-strategy.md @@ -0,0 +1,183 @@ +--- +id: 2 +title: Retire the PDF/notebook analysis docs for an automated, script-based report +labels: [wayfinder:grilling] +status: closed +blocked_by: [3] +assignee: session-2026-08-09f +claimed_at: 2026-08-09 +resolution: decided +evidence: judgement +closed_by: null +spawned_by: null +--- + +## Premise + +Rests on [Does product-pipeline's test suite meet the same cell-by-cell rigor +as patient's?](01-product-pipeline-test-rigor.md), closed as superseded: that +session decided R-vs-Python output comparison is an analysis activity, not a +pytest concern, and that this ticket is where it belongs. + +**Now blocked on** [Merge product-pipeline (PR #6) into +migration](03-merge-product-pipeline.md) — the user corrected the original +sequencing here: this ticket's comparison script only makes sense to build +once patient and product live on one branch, not before, and isn't part of +what makes the merge itself trustworthy (tests green + implementation review +is judged sufficient for that). This ticket used to block the merge; now the +merge blocks this ticket instead. It also decided the +comparison's goal is not R-parity — Python may correctly diverge from R (R can +be wrong) — so the automated report must judge divergence against the +**original source Excel trackers** (available under `a4dphase2_upload` on the +test-data drive) as ground truth, not treat R's output as automatically +correct. A report that only flags "Python != R" without being able to say +which one is right against source doesn't meet this bar. + +Facts on record: `docs/archive/PYTHON_IMPROVEMENTS.md` on `product-pipeline` +cites `Ali_internship/residual_dig.ipynb` for the date-parsing analysis (section 5) +— that file does not exist anywhere in the repo's tracked tree, and never has +(`git log --all -- "*residual_dig*"` returns nothing). Two PDF reports also +ship on that branch; both have now been read in full (via +`git show origin/product-pipeline:` + `pdftotext`): + +- `docs/archive/dashboarding_evaluation_report_verbose_.pdf` is **unrelated + to this map** — it's a BI-tool comparison (Looker Studio vs. Evidence/ + Metabase/Superset/Redash) for A4D's research dashboarding, a separate + internship deliverable with nothing to do with pipeline parity. Out of this + ticket's (and this map's) scope; not this ticket's concern. +- `docs/archive/Product pipeline parity presentation.pdf` **is** the missing + `residual_dig.ipynb`'s output — the only surviving record of a real R-vs- + Python divergence analysis across 189 trackers / 61,077 rows / 20 columns: + per-column mismatch counts (`Product_entry_date`: 559, `Product_balance`: + 480, `Product_category`: 214, `Product_sheet_name`: 201, + `Product_received_from`: 154, `Product_units_received`: 9), with the + entry-date mismatches further broken down by cause (typo-rescue: 408, + CE-typo: 71, sentinel-null handling: 66, off-by-one-day: 7). Since the + notebook itself is unrecoverable, this PDF is the ground truth this + ticket's script needs to be checked against, not just background reading. + +## Question + +Decide the replacement for the current PDF/notebook-cited documentation: +what should the automated report look like (what does it check — all trackers +locally, error gathering, and where it needs to fall back to the original +source Excel trackers to judge a Python/R divergence rather than trusting R's +output — and how does it regenerate), and what happens to the existing PDFs +and the dead notebook reference (delete, replace, or keep as historical +record with a pointer to the new script)? Also decide what happens to +patient's existing `test_r_validation.py` (the current hand-written +exception-dict pattern) given it's now understood to be this ticket's kind of +artifact, not a pytest one — fold into the new report, or keep separate. + +**Validation requirement, per the user:** once the comparison script this +ticket designs is built, run it and check whether it reproduces the same +per-column and per-cause mismatch numbers as the parity-presentation PDF +above (189 trackers, 61,077 rows, 20 columns, the counts listed in the +Premise) — that reproduction is the check that the new script is a faithful, +trustworthy replacement before the PDF is retired. Not required to design +that validation step in detail *now* (design happens when this ticket is +worked); recorded here so it isn't lost. + +User has flagged this as lower priority than the merge/CI work, so the +resolution can scope it as a follow-up ticket rather than something to build +in this session. + +## Resolution + +**Decision.** The R baseline for comparison is the already-frozen +`/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_r/` (dated 2025-11-14), +generated from the same still-unchanged trackers on `a4dphase2_upload` — +covering both `patient_data_{raw,cleaned}/` and `product_data_{raw,cleaned}/` +plus final tables, in the same per-tracker parquet layout the Python pipeline +produces. **No R re-run is needed**, ever, going forward. + +The comparison script (`scripts/compare_outputs.py` + a `just compare-outputs` +recipe — **not** an `a4d.cli` subcommand, since it's migration-only tooling +that becomes meaningless the moment R is retired, ticket 12) is **decoupled +from pipeline execution**: it takes two existing output directories (a Python +one, an R one) and diffs them; producing a fresh Python output is a separate, +manual step the user triggers when wanted. + +The diff runs as four increasingly granular layers, mirroring the structure +already used across patient's existing test suite and the now-removed +`test_r_validation.py`: (1) shape/row-count match, (2) aggregate totals +match, (3) column names and dtypes match, (4) cell-by-cell value diff. Cause +classification for flagged cell-level mismatches is a small extensible +registry (`{cause_name: classifier_fn}`), seeded only with the four causes +already known from the parity-presentation PDF for `Product_entry_date` +(typo-rescue, CE-typo, sentinel-null, off-by-one-day); anything else is +reported as an unclassified mismatch. The script's output is an **HTML +report** (chosen over PDF — no extra rendering dependency, easy to skim, +directly embeds the per-column/per-cause summary tables), the direct +successor to the old PDF format but regenerated from real data each run +rather than hand-assembled once. + +Docs cleanup (done in this session, both small and fully decided): +- `docs/archive/dashboarding_evaluation_report_verbose_.pdf` deleted — + confirmed unrelated to this map (a separate BI-tool-comparison internship + deliverable). +- `docs/archive/Product pipeline parity presentation.pdf` **kept** — it + holds the only surviving real numbers (per-column/per-cause mismatch + counts) needed to validate the new script; the user was explicit it stays + "to compare" and gets removed only once superseded, since it isn't part of + the pipeline itself. +- `docs/archive/PYTHON_IMPROVEMENTS.md`'s dead citation to + `Ali_internship/residual_dig.ipynb` (confirmed never committed — + `git log --all -- "*residual_dig*"` returns nothing) fixed to point at the + parity-presentation PDF instead, with a note that the new script supersedes + it once built. +- `test_r_validation.py` is already fully gone from pytest (removed in the + ticket 3/8 merge work, commit `6b9a095`) — confirmed via `git log --all`. + Its content isn't ported wholesale: only the causes it's known to have + captured (folded into the seeded classifier registry above) survive: the + rest of its old exception-dict logic is superseded by the same + investigate-and-label process the follow-up ticket below carries out. + +Building the script, running it, and triaging the flagged differences +one-by-one to identify and label causes (the bulk of the real work — causes +for most columns aren't known ahead of time and can only be found by +inspecting actual flagged rows against the source trackers) is **not** done +in this session — the user confirmed this stays a follow-up ticket, per the +original scoping note above. Spawned as [ticket +15](15-build-and-run-comparison-script.md). + +**Because.** R hasn't been re-run in a long time and re-running it risks +version drift from the trackers' current state; the frozen `output_r/` +sidesteps that entirely since the trackers it was generated from are +confirmed unchanged. R output can only ever be a screening filter — the +user's own point mid-session was that any real disagreement still has to be +settled against the source tracker, R or no R — so building a from-scratch +independent Excel ground-truth extractor up front is unnecessary work; the +layered-check design reuses a pattern already proven in the existing test +suite rather than inventing a new one. + +**Rejected.** +- Live R execution as part of the comparison script — rejected: R is being + retired (ticket 12), version drift risk, and the frozen baseline already + exists and covers the same trackers. +- A from-scratch independent Excel-native extractor as the ground-truth + comparator — rejected: real divergences require manual source-tracker + adjudication regardless of what the bulk diff is checked against, so + building automated Excel extraction up front duplicates work without + removing the manual step. +- Wiring the comparison into `a4d.cli` as a subcommand — rejected: it's + migration-only tooling with a defined end-of-life (R's retirement), not + part of the pipeline's permanent operational surface. +- Designing the full cause taxonomy now — rejected: most causes are only + discoverable by actually inspecting flagged differences; that + investigation is the follow-up ticket's job, not this one's. +- PDF output for the report — rejected in favor of HTML: no extra rendering + dependency needed, easier to generate and skim. + +**Evidence.** *Executed*: confirmed `output_r/` exists on the USB drive with +patient+product data dated 2025-11-14 (`ls`); confirmed +`Ali_internship/residual_dig.ipynb` was never committed (`git log --all`); +confirmed `test_r_validation.py` was already removed in commit `6b9a095` +(`git log --all`); deleted the unrelated PDF and fixed the dead citation. +*Judgement*: the script's design (decoupling, layered-check structure, +extensible cause registry, HTML output, `scripts/` placement) is the user's +live decision this session, not derived from a prior artifact. + +**Tense.** All claims above describe either current repository/drive state +(executed) or a design decision for work not yet built (the follow-up +ticket owns building and running it). diff --git a/docs/wayfinder/tickets/03-merge-product-pipeline.md b/docs/wayfinder/tickets/03-merge-product-pipeline.md new file mode 100644 index 0000000..960840b --- /dev/null +++ b/docs/wayfinder/tickets/03-merge-product-pipeline.md @@ -0,0 +1,158 @@ +--- +id: 3 +title: Merge product-pipeline (PR #6) into migration +labels: [wayfinder:task] +status: closed +blocked_by: [8] +assignee: session-2026-08-08b +claimed_at: 2026-08-08 +resolution: decided +evidence: executed +closed_by: null +spawned_by: null +--- + +## Premise + +Depends on [Does the pytest suite reach unit/integration/e2e/regression +parity between patient and product, excluding any R-comparison/USB-drive-dependent +tests?](08-pytest-suite-parity.md) (itself blocked on [Is the product pipeline +(and patient's own claimed completeness) actually complete and sound, audited +against R's product logic and patient's structure?](07-pipeline-completeness-audit.md)) +being resolved first — the user's stated order is product-pipeline readiness +before the merge, not merge-then-fix. (Originally blocked on [Does +product-pipeline's test suite meet the same cell-by-cell rigor as +patient's?](01-product-pipeline-test-rigor.md), closed as superseded and +replaced by tickets 7/8 above.) + +**No longer blocked on** [Retire the PDF/notebook analysis docs for an +automated, script-based report](02-documentation-strategy.md) — the user +corrected this sequencing: the R-vs-Python comparison script only makes +sense to build once patient and product live on one branch, not before, so +it belongs *after* this merge rather than gating it. Ticket 2 is now blocked +by this ticket instead of the reverse. The merge's actual readiness bar is +tests green (ticket 8) plus ordinary pre-merge hygiene the user named +explicitly: code style, a review pass on the implementation itself (are all +of R's steps actually migrated, not just tested), and doc alignment with the +patient pipeline's conventions — not proof of R/Python output parity, which +this ticket's own review process (green tests + a clean implementation +review) is judged sufficient grounds to trust without first. + +Facts on record: PR #6 (`product-pipeline` -> `migration`) is `mergeable: +CONFLICTING`. `migration` carries two commits (`2069a4d`, `faa3c28`) not on +`product-pipeline` — error-log-table work touching `tables/logs.py` and +`tables/errors.py` — while `product-pipeline` independently changed +`tables/logs.py` and added `tables/metadata.py`. This is the real source of the +conflict. + +## Question + +Resolve the merge: reconcile `migration`'s error-log-table commits with +`product-pipeline`'s `logs.py`/`errors.py`/`state/` changes, land PR #6, and +confirm the merged `migration` branch has both arms (patient + product + +state/incremental) working together, not just side by side. Before landing +it, also confirm the pre-merge hygiene bar the user set: ticket 8's tests +green, a code-style pass, an implementation review confirming all of R's +product-side steps are actually migrated (not just individually tested), and +doc alignment with the patient pipeline's conventions. + +## Resolution + +**Decision:** All four readiness-bar items are done and pushed to +`product-pipeline`; PR #6 is now `mergeable: MERGEABLE` with its conflicts +resolved. Landing the merge itself (clicking the button) is left to the user +per the map's human-only-merge guardrail. + +**What was implemented (in a `../a4d-product-pipeline` worktree, on the +`product-pipeline` branch, then merged forward from `origin/migration`):** + +1. **Ticket 8's test-parity decision, implemented:** the three missing + product integration/e2e test classes (extract, clean, e2e) added to the + existing patient files, mirroring patient's fixture/skip-if-missing + convention exactly (verified against real tracker files — the USB drive + was mounted this session); `tests/test_extract/test_wide_format.py` added + (wide_format.py was 16% covered, now 97%); `tests/test_tables/test_product.py` + added (`create_table_product_data` was previously only reached indirectly); + `tests/test_pipeline/test_product.py` added (pipeline/product.py + orchestration had zero tests before this session). `test_r_validation.py` + removed from pytest entirely, no product equivalent, per ticket 8. +2. **Coverage gate, narrowed to product-only (user correction mid-session):** + ticket 8's 85% figure was judgement-only and never checked against the + actual baseline — repo-wide coverage was 73%, and most of the gap + (`clean/patient.py`, `cli.py` argument parsing, `pipeline/tracker.py`) + is patient/CLI code unrelated to product parity. The user redirected: gate + 85% for product-pipeline files only (`extract/product.py`, `clean/product.py`, + `clean/schema_product.py`, `extract/wide_format.py`, `pipeline/product.py`, + `tables/product.py`, `validate/source_vs_output_product.py`) via a separate + `coverage report --include=... --fail-under=85` CI step, alongside the + existing whole-package `--cov` for codecov reporting. Product-only + coverage: 73% baseline → 88% after the new tests. Patient/CLI coverage is + explicitly deferred to a future pass once merged onto `migration`. +3. **Code style + type checking:** `ruff check .` / `ruff format --check .` / + `ty check src/` all clean repo-wide (not just product files) — CI runs + them unscoped, so pre-existing violations anywhere blocked the merge. + Fixed: `r-archive/` was being linted despite CLAUDE.md's do-not-modify + rule (excluded via `extend-exclude`, not edited); ~26 line-length/ + composite-assertion/broad-exception issues across patient and shared + test files; 2 pre-existing `ty` diagnostics (`cli.py`'s + `_render_pipeline_header` typed `data_root: str` but always called with + a `Path`; `tables/metadata.py`'s schema dict mixed bare dtype classes + with an instance under too-narrow an annotation — fixed with the same + `type[pl.DataType] | pl.DataType` pattern `clean/schema_product.py` + already uses). +4. **Implementation review, done via the logging-approach comparison this + session started with (see "Where this map stands"):** found and fixed two + real gaps, not just confirmed the audit — `process_tracker_product` wasn't + populating `TrackerResult.data_errors` (migration added that field after + product-pipeline branched, so neither arm wired it on this branch), and + `process_product_cmd` never created a logs or errors table at all, unlike + `process_patient_cmd` (Step 1/2, Step 2/2 vs. patient's Step 1/4..4/4). + Both fixed to mirror patient exactly, locked in by a new + `TestProcessProductE2E` CLI test class. Also fixed a latent bug caught + while writing `wide_format.py`'s unit tests: `_rows_to_df` pinned output + dtype from the pre-split schema, so an all-null source column (e.g. + "Units Released" on 2017-2019 Mandalay comma-cell trackers, which have no + populated units column before the split writes into it) inferred `Null` + and silently dropped the split's new string values — same root cause and + fix (`schema_overrides`-equivalent) as an existing fix in `tables/logs.py`. +5. **Doc alignment:** verified, no changes needed — `product-pipeline`'s + `docs/CLAUDE.md`/`MIGRATION_GUIDE.md` are already a clean superset of + `migration`'s stale copies (`migration` never independently touched those + files since divergence), so they merge automatically. +6. **Merge conflicts resolved:** `gcp/bigquery.py` (both `PARQUET_TO_TABLE` + entries kept, purely additive) and `cli.py` (merged imports, merged + `_display_tables_summary`'s table list, kept `product-pipeline`'s + `_render_pipeline_header`-based run-pipeline banner as the strict + superset of `migration`'s inline version). + +**Rejected:** +- Chasing the whole-repo 85% coverage floor as originally decided in ticket + 8 — rejected per the user's mid-session correction; the number was never + checked against the 73% baseline and most of the gap is unrelated to + product parity. +- Leaving the repo-wide ruff/ty violations for ticket 4 — rejected: unlike + the coverage floor (real effort, genuinely out of this ticket's scope), + these were mechanical one-line fixes directly blocking *this* PR's CI, + cheaper to fix than to defer. +- Chasing the CI failure to green fully — rejected: after the ruff/ty fixes, + CI's remaining failure is 7 `--help`-output assertion tests failing only + in the GitHub Actions runner (Typer/Rich renders differently there than + locally, even with `CliRunner(env={"NO_COLOR": "1", "COLUMNS": "200"})` + already set) — unrelated to product code, affects patient/CLI help tests + broadly, and is a genuine investigation matching ticket 4's exact remit + ("diagnose and fix why CI is red"), not a fact this ticket can close by + deciding. + +**Evidence:** executed — every claim above was verified by running the +actual command (`pytest`, `ruff`, `ty`, `coverage report --fail-under`, +`gh pr view`) against the real branches, not read from a plan or inferred. +Real tracker files on the mounted USB drive were used to derive exact +expected row/column counts for the new integration/e2e tests rather than +guessing. + +**Tense:** current as of 2026-08-08 — PR #6 is `mergeable: MERGEABLE` right +now; CI's remaining failure is current, not a predicted consequence. + +**Spawned:** none directly, but ticket 4 (CI still red) is now unblocked and +has a concrete lead (the Typer/Rich `--help` rendering diagnostic above) to +start from instead of a cold investigation. diff --git a/docs/wayfinder/tickets/04-fix-migration-ci.md b/docs/wayfinder/tickets/04-fix-migration-ci.md new file mode 100644 index 0000000..8aa0ba8 --- /dev/null +++ b/docs/wayfinder/tickets/04-fix-migration-ci.md @@ -0,0 +1,101 @@ +--- +id: 4 +title: Diagnose and fix why CI is red at migration HEAD +labels: [wayfinder:task] +status: closed +blocked_by: [3] +assignee: session-2026-08-09 +claimed_at: 2026-08-09 +resolution: decided +evidence: executed +closed_by: null +spawned_by: null +--- + +## Premise + +Depends on [Merge product-pipeline (PR #6) into migration](03-merge-product-pipeline.md) +being done first — the user wants migration-readiness work sequenced after the +merge, since the merge itself will change what's running through CI. + +Facts on record: `gh run list --branch migration` shows the last 3 runs of the +"Python CI" workflow all failed, including the current tip `faa3c28` +(2026-04-01 and later). PR #2 (`migration` -> `dev`) is `mergeable: MERGEABLE` +but shows the same failing "test" check. + +## Question + +Find out why CI fails at `migration` HEAD (post-merge) and fix it — is this a +transient environment issue, a real test failure, a lint/type-check break, or +something the merge itself needs to resolve? + +## Resolution + +**Decision:** Added `tests/conftest.py` on `product-pipeline` (PR #6) setting +`os.environ.setdefault("_TYPER_FORCE_DISABLE_TERMINAL", "1")` before any test +module can import `a4d.cli`. Pushed as `b970cf6`. CI run +[31285797606](https://github.com/CorrelAid/a4d/actions/runs/31285797606) is +green — all 7 previously-failing `--help` tests pass, full suite (428 tests) +passes, coverage gate holds. PR #6 is `mergeable: MERGEABLE` with a passing +`test` check. + +**Because:** Root-caused by reading Typer's own source +(`typer.rich_utils`), not by guessing from symptoms. At **import time**, +`typer/rich_utils.py` computes a module-level `FORCE_TERMINAL` constant: +`True if getenv("GITHUB_ACTIONS") or getenv("FORCE_COLOR") or +getenv("PY_COLORS") else None`. GitHub Actions sets `GITHUB_ACTIONS=true` +for every job, so on the Actions runner `FORCE_TERMINAL` freezes to `True` +the moment `a4d.cli` (which imports `typer`) is first imported — regardless +of anything `CliRunner(env={...})` sets afterward at invoke time, and +regardless of `NO_COLOR`/`COLUMNS`. With `FORCE_TERMINAL=True`, Typer's +`--help` renderer applies full Rich styling, and Rich's highlighter styles +each hyphen of a long option separately — e.g. `--file` becomes +`\x1b[1m-\x1b[0m\x1b[1m-file\x1b[0m` — which breaks a plain `"--file" in +result.output` substring check even though the visible text is unchanged. +Locally `GITHUB_ACTIONS` is unset, so `FORCE_TERMINAL` stays `None` and +plain text renders, which is why the tests only ever failed on the runner. +Confirmed by reproducing the exact failure locally with +`GITHUB_ACTIONS=true uv run pytest ...` (fails without the fix, 428 pass +with it), and by inspecting `typer.rich_utils` source directly rather than +trusting docs or memory. Typer ships its own escape hatch for exactly this, +`_TYPER_FORCE_DISABLE_TERMINAL` (also read once at import time, same +module), which forces `FORCE_TERMINAL = False` regardless of `GITHUB_ACTIONS` +— so the fix is to set that env var before `typer.rich_utils` is first +imported, which for a pytest run means a `conftest.py` at the top of the +collected tree (`tests/conftest.py`), not a fixture (fixtures run too late, +after collection has already imported the test modules). + +**Rejected:** +- *The original lead — `cli.py`'s module-level `console = Console()` + snapshotting width at import.* Investigated first since it was the + starting hypothesis, but disproved by reading Rich's `Console.size` + property: `COLUMNS` from `os.environ` is re-read live on every access, not + cached at construction, and Click's `CliRunner.isolation()` mutates the + real `os.environ` in place (not a copy), so `Console()`'s construction + timing doesn't matter for width. `Console.no_color` *is* frozen at + construction, but that's a red herring here too: the failing `--help` + output isn't rendered by `cli.py`'s `console` object at all — Typer + renders `--help` through its own internal `Console` (`typer.rich_utils + ._get_rich_console`), built fresh on every call. `cli.py`'s console was + never the culprit. +- *Stripping ANSI codes in the test assertions instead.* Would fix the + symptom but leaves color rendering nondeterministic between local and CI + runs for every future `--help` test, and doesn't explain why coverage + differed either — the config-level fix removes the actual source of + divergence. +- *Setting `COLUMNS`/`NO_COLOR` more aggressively in the CI workflow YAML.* + Already true in the failing runs (`CliRunner(env={"NO_COLOR": "1", + "COLUMNS": "200"})`) — proven not to matter, since `FORCE_TERMINAL` is + decided by `GITHUB_ACTIONS`/`FORCE_COLOR`, not by those two. + +**Tense:** All claims above describe current, verified behavior (executed +against the actual `product-pipeline` code and the actual GitHub Actions +runner via a real CI run), not a hypothesis about a still-open lead. + +**Where applied:** `tests/conftest.py` was added on `product-pipeline` +(PR #6's branch), not `migration` directly — `migration`'s own tip +(`faa3c28`) predates the product-pipeline code entirely (patient-only), and +the map's Notes record that ticket 3's merge work already landed on +`product-pipeline` rather than `migration`. This fix rides along with that +same merge. `migration`'s CI will be green once the merge is landed +(human-only action, per this map's guardrails). diff --git a/docs/wayfinder/tickets/05-production-verification-run.md b/docs/wayfinder/tickets/05-production-verification-run.md new file mode 100644 index 0000000..e8bb2e2 --- /dev/null +++ b/docs/wayfinder/tickets/05-production-verification-run.md @@ -0,0 +1,107 @@ +--- +id: 5 +title: Define and execute the real GCP production verification run +labels: [wayfinder:task] +status: closed +blocked_by: [3, 4] +assignee: session-2026-08-09b +claimed_at: 2026-08-09 +resolution: decided +evidence: executed +closed_by: null +spawned_by: null +--- + +## Premise + +Depends on [Merge product-pipeline (PR #6) into migration](03-merge-product-pipeline.md) +(need the combined code to run) and [Diagnose and fix why CI is red at migration +HEAD](04-fix-migration-ci.md) (a red CI is not a base to run a production job +from). + +This is the destination's core gate: the user wants the merge validated by an +actual GCP run — container deployed, job executed against the production +bucket, output landed in BigQuery — not by tests passing in isolation. Individual +components (product pipeline, state management) have reportedly been tested +against GCP separately already, but never as one combined run. + +## Question + +Define what "real production run" verification means concretely: which bucket, +what safety/cost bounds, who executes it, and how output is compared against +the R baseline / dashboard numbers (cell-by-cell, per the user's stated +standard). Then execute it and record the result. + +## Resolution + +**Decision:** Defined and executed the real production run against the +existing GCP infra: + +- Buckets/dataset: `a4dphase2_upload` (source trackers) -> `a4dphase2_output` + (processed output) -> `a4dphase2.tracker` (BigQuery), the live + `src/a4d/config.py` defaults — no separate sandbox bucket exists or is + needed; this is the real Cloud Run Job (`a4d-pipeline`, region + `asia-southeast2`) already provisioned per `SETUP.md`. +- Safety bound: `just backup-bq` run immediately before the job, snapshotting + `patient_data_static/monthly/annual` and `product_data` (the four tables the + run `WRITE_TRUNCATE`s) with 7-day expiry as a rollback point. +- Who executes: the user triggered `just deploy && just run-job` directly + (guardrail — production trigger is a human action). Agent did prep + (`just docker-build && just docker-smoke` against `migration` HEAD, + confirming the image builds and starts before deploy) and read-only + monitoring (polled `gcloud run jobs executions describe` until completion, + execution `a4d-pipeline-8mxls`, started 01:00:57 UTC, completed 01:07:59 UTC + 2026-08-09, `succeededCount: 1`, `failedCount: 0`). +- Comparison: **not** against R/dashboard numbers — the user decided mid-ticket + that R plays no role in this run's own check; R-vs-source-Excel comparison + is ticket 2's remit, deferred and out of this ticket's scope. Instead, built + `src/a4d/gcp/verify.py` + `scripts/verify_production_run.py` (unit-tested, + TDD) comparing each live table against its `just backup-bq` snapshot: row + count, distinct `clinic_id` count (skipped for `patient_data_annual`, which + has no `clinic_id` column — patient/year granularity only), and schema. + Run against backup suffix `20260809`: all four tables grew in both row count + and clinic coverage (51 -> 53 clinics across patient_data_static, + patient_data_monthly, product_data), no schema changes, no anomalies + flagged. + +**Because:** the destination's gate is "run for real against the GCP +production bucket, with output landed in BigQuery" — a full R-parity +comparison is a separate, heavier concern (ticket 2) the user explicitly +decoupled from this run's own pass/fail. A coarse growth/shape check against a +known-good pre-run snapshot is sufficient to catch a gross regression (empty +table, dropped clinics, broken schema) without waiting on unbuilt tooling. + +**Rejected:** +- *Blocking this run on ticket 2's cell-by-cell comparison script being built + first* — rejected: ticket 2 is explicitly lower-priority/deferred, and the + user confirmed R has no role in this ticket's own verification standard. +- *SETUP.md's "Level 3" local Docker+ADC dry run against real GCS/BigQuery + first* — judged unnecessary once the actual Cloud Run Job path was + confirmed working (image built and smoke-tested locally first instead, a + cheaper substitute for the same "does the current code even start" + question). +- *Requiring Cloud Scheduler to be wired up before calling this "production + verified"* — out of scope for this ticket; confirmed via `gcloud scheduler + jobs list` that the Cloud Scheduler API isn't even enabled on the project, + which independently confirms the Migration Guide's "Production Scheduling" + item is genuinely still open (not just stale docs) — left as its own fog + entry, not folded into this ticket. + +**Evidence:** executed throughout — real `gcloud run jobs execute` triggered +by the user, execution status polled live via `gcloud run jobs executions +describe`, BigQuery row/clinic counts queried live via the verify script +against the real backup snapshot tables (`bq ls` confirmed the snapshot +tables existed first). Nothing here rests on a doc claim or recollection. + +**Tense:** current, verified behavior — the combined patient + product +pipeline has now actually run once, end-to-end, against real production GCS +and BigQuery, for the first time as one execution. Prior to this, per the +user, only the product arm alone had been tested in this Cloud Run Job (by an +intern, before `product_data` was in production use) — the BigQuery +`last_modified_time` split seen while orienting (patient tables stale since +2026-04-09, `product_data`/`clinic_data_static`/`tracker_metadata` from +2026-05-29) matches that account rather than contradicting it. + +**Where applied:** ran against `migration` HEAD (post PR #6 merge, commit +`7713fea`) via the existing `a4d-pipeline` Cloud Run Job; no infra changes +needed since it was already fully provisioned. diff --git a/docs/wayfinder/tickets/06-promote-migration-to-dev.md b/docs/wayfinder/tickets/06-promote-migration-to-dev.md new file mode 100644 index 0000000..7e7f8c1 --- /dev/null +++ b/docs/wayfinder/tickets/06-promote-migration-to-dev.md @@ -0,0 +1,128 @@ +--- +id: 6 +title: Promote migration into dev via PR #2 +labels: [wayfinder:task] +status: open +blocked_by: [8, 3, 4, 5, 10, 11, 12, 13, 14, 20, 21, 22, 23, 64] +assignee: session-2026-08-24h +claimed_at: 2026-08-24 +resolution: null +evidence: null +closed_by: null +spawned_by: null +--- + +## Premise + +Depends on every other ticket on this map: [Does the pytest suite reach +unit/integration/e2e/regression parity between patient and product, excluding +any R-comparison/USB-drive-dependent tests?](08-pytest-suite-parity.md) (and, +transitively, [Is the product pipeline (and patient's own claimed +completeness) actually complete and sound, audited against R's product logic +and patient's structure?](07-pipeline-completeness-audit.md)), [Merge +product-pipeline (PR #6) into migration](03-merge-product-pipeline.md), +[Diagnose and fix why CI is red at migration HEAD](04-fix-migration-ci.md), +[Define and execute the real GCP production verification +run](05-production-verification-run.md), [Profile the combined pipeline's +performance against the R baseline before promoting to +dev](10-performance-profiling.md), [Decide what CLI/TUI UX and error-log +observability improvements admins/developers need before +rollout](11-cli-ux-observability.md), [Retire R from the workspace once the +pipeline is fully verified Python-only](12-retire-r-workspace.md), [Audit and +update all dependencies and library versions before +rollout](13-dependency-audit.md), and [Triage every flagged R/Python +difference for both arms, and resolve the 189-vs-155-tracker +discrepancy](18-triage-comparison-flagged-differences.md) — this last one +replaced [Retire the PDF/notebook analysis docs for an automated, +script-based report](02-documentation-strategy.md) in this list once that +ticket closed, was itself replaced by [Fix the product comparison's +row-alignment key, then triage every flagged R/Python +difference](17-fix-product-row-alignment-and-triage.md) once [Build and run +the R/Python output comparison script, then triage every flagged +difference](15-build-and-run-comparison-script.md) closed having built and +run the tooling but found product's row alignment broken before any triage +could happen, and was replaced again by ticket 18 once ticket 17 closed +having fixed and validated the alignment key but not converged on the +actual triage. The destination's own requirement that "every Python/R +difference [be] documented and explicitly decided" isn't satisfied until +ticket 18's triage is done. + +PR #2 (`migration` -> `dev`) already exists and is `mergeable: MERGEABLE`; it is +gated only on the work above landing first. + +## Question + +With the merge done, CI green, and a real production run validated, close out +PR #2 and merge `migration` into `dev`. This is the destination. + +## Premise update (session-2026-08-24f) + +[Retiring R from the workspace](12-retire-r-workspace.md) closed, and with it +**every ticket in this list was closed** -- the first time this ticket has been +fully unblocked. It was re-blocked in the same session, on one new ticket, and +that is argued rather than applied. + +`blocked_by` gains **64** ([Rewrite every docstring and doc that explains the +code by what R did](64-documentation-overhaul-drop-r-framing.md)). The user's +statement closing ticket 12 was that documentation justifying a behaviour by +"to match R" is not merely stale but **wrong**, now that the migration has +reached its end and R is gone. Thirty-five such citations are live in twelve +Python modules, including production cleaning code. Promoting `migration` into +`dev` is the act that turns this branch's documentation into the project's +documentation, so shipping it while it is wrong is the one thing this gate +exists to prevent -- the destination's own words are that the migration is +"detail-sensitive enough that things get missed unless checked". + +The alternative -- promote now and fix the docs on `dev` afterwards -- was +rejected because it inverts the map's whole sequencing preference (make it +ready, *then* promote) and because a docstring citing a directory that no +longer exists is a defect a reviewer of PR #2 would reasonably raise. +Reversible: dropping `64` from this list is a one-line edit. + +## Premise update (session-2026-08-24g) + +[The documentation overhaul](64-documentation-overhaul-drop-r-framing.md) is +closed, so **every ticket in `blocked_by` is now closed and this ticket is +genuinely unblocked** -- for the first time, and this time without a +replacement. It is the last ticket on the route to the destination. + +`blocked_by` is kept as the historical record of what gated the promotion; the +renderer reads it as satisfied because every entry is closed. + +One open ticket was deliberately **not** wired here: [the logs table's two +R-named values](65-logs-table-r-named-values.md). The gate that ticket 64 +satisfied was argued on documentation being *wrong*; ticket 65 is about +published field values being *inconsistent*, which is a different and lesser +claim, and fixing it changes BigQuery output rather than correcting a falsehood. +Reversible if the data owner would rather have it in before promotion. + +## Prepared, not merged (session-2026-08-24h) + +The merge itself is a human-only action per the map's Notes, so this session +prepared it and stopped. + +State verified rather than assumed, at head `ec12f48`: + +- `mergeable: MERGEABLE`, `mergeStateStatus: CLEAN`. +- CI green on the head commit (both `test` jobs SUCCESS). +- `origin/dev` is an ancestor of `migration` and has **zero** commits + `migration` lacks, so there is nothing to reconcile. +- Local and remote `migration` identical; working tree clean. +- Both recovery tags are on the remote: + `r-archive-removed` -> `87530b1`, `migration-archive-frozen` -> `0dcc02d`. +- `dev` has **no branch protection**, so no review is required by the repo and + nothing will block the button. +- `delete_branch_on_merge` is **false**, so `migration` survives the merge. + +The PR body was replaced with the current `docs/archive/MR_DESCRIPTION.md` +(43,815 chars, was 23,547 and stale). Two claims in that document were false and +were corrected first: it opened with "**This MR is not ready to merge yet**", +and its "Still open" section said the listed work "blocks the merge". Neither +is true now. + +**Open question for whoever merges: squash or merge commit.** Both are enabled. +Squashing collapses 285 commits whose messages are the migration's own +evidence trail and which the wayfinder tickets cite by SHA. The R-recovery +recipe survives either way -- an annotated tag pins its commit regardless of +merge strategy -- **provided the two tags above are never deleted**. That is +the one thing that would make `r-archive/` genuinely unrecoverable. diff --git a/docs/wayfinder/tickets/07-pipeline-completeness-audit.md b/docs/wayfinder/tickets/07-pipeline-completeness-audit.md new file mode 100644 index 0000000..c7a9917 --- /dev/null +++ b/docs/wayfinder/tickets/07-pipeline-completeness-audit.md @@ -0,0 +1,133 @@ +--- +id: 7 +title: Is the product pipeline (and patient's own claimed completeness) actually complete and sound, audited against R's product logic and patient's structure? +labels: [wayfinder:research] +status: closed +blocked_by: [] +assignee: session-2026-08-08 +claimed_at: 2026-08-08 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 1 +--- + +## Premise + +Rests on [Does product-pipeline's test suite meet the same cell-by-cell rigor +as patient's?](01-product-pipeline-test-rigor.md), closed as superseded: that +session established (a) R-parity isn't the goal — Python may correctly +diverge from R, judged against the original source Excel trackers, not R's +output; (b) comparing R vs. Python output is analysis, not a pytest concern; +(c) neither product's nor patient's "done" status has actually been audited — +patient's completeness has been assumed (`docs/CLAUDE.md`: "Patient pipeline +is complete and deployed to production"), product's completeness is claimed +only in `docs/archive/MIGRATION_GUIDE.md` ("Phases 0-9 complete") and +`docs/archive/PYTHON_IMPROVEMENTS.md` ("production-ready") on the +`product-pipeline` branch, both unverified by this map so far. The user has +not looked at the product pipeline's code themselves and is relying entirely +on secondhand claims ("what the intern said"). + +If [Retire the PDF/notebook analysis docs for an automated, script-based +report](02-documentation-strategy.md) closes first and changes what counts as +"documented" for this project, re-check this ticket's Question 2 against that +before answering it. + +## Question + +Read-only code/doc/test inventory — no live pipeline runs, no output +comparison (that's explicitly out of scope here; it belongs to the later +output-validation work and to ticket 2's automated report). For **both** +the patient and product pipelines: + +1. Module-for-module, does the Python implementation cover every step of the + corresponding R logic (`r-archive/`'s `helper_patient_data.R` / + `helper_product_data.R`), or are there gaps? Where Python deliberately + does something R doesn't (or vice versa), is that documented anywhere, or + only discoverable by reading the diff? +2. Does each pipeline have the same shape of test coverage — unit tests per + cleaning/validation/transform module, integration tests across pipeline + stages, an end-to-end test, and non-R-comparison regression tests? Produce + a file-by-file inventory (what exists for patient, what exists for + product, what's missing for either). +3. Is the documentation (`docs/CLAUDE.md`, `MIGRATION_GUIDE.md`, + `PYTHON_IMPROVEMENTS.md`, module docstrings) current, or stale/aspirational + relative to the actual code? Flag every claim that isn't backed by + something checkable in the repo (code, tests, or a dated analysis + artifact — not prose alone). +4. Net out: what's the concrete gap list, for product and for patient, before + either can be called "complete"? + +Out of scope for this ticket: running either pipeline, comparing real output, +or judging *why* R and Python differ — those are analysis questions for later +tickets, once this inventory establishes there's a stable pipeline to compare +in the first place. + +## Resolution (decided) + +Full inventory: [docs/wayfinder/research/07-pipeline-completeness-audit.md](../research/07-pipeline-completeness-audit.md). + +**Decision:** Neither pipeline is actually "complete" yet, but for different +reasons, and the premise's "neither status has been audited" turns out to be +only half right: + +1. **R-logic coverage is essentially complete on both sides.** Every R + function in `helper_patient_data.R`/`script2_helper_patient_data_fix.R` + and every R function in `helper_product_data.R`/`read_product_data.R` has + a traceable Python counterpart, function for function, verified by + cross-referencing R's function inventory against the Python modules. The + one gap found is two diagnostic-only R helpers + (`calculate_most_frequent`, `report_empty_intersections`, cross-tab + logging, not data-shape-affecting) with no Python equivalent on + `origin/product-pipeline`. +2. **Divergence documentation is stronger on product than patient.** Product + module docstrings cite specific R script/step numbers and the deviation + is explained inline (e.g. `tables/product.py:link_product_patient`). + Patient's known divergences live only inside `test_r_validation.py`'s + exception dicts (42 tracker-level entries, 15+ with reasons) and never + made it into `PYTHON_IMPROVEMENTS.md`'s 3-item summary table on + `migration` — someone reading that doc alone undercounts the known + differences by roughly 5x. +3. **Product has real, structural test gaps**: zero integration/e2e tests + (`tests/test_integration/` is byte-identical between branches), zero test + references to `extract/wide_format.py` across all 41 product test files, + no `test_tables/test_product.py`, and its only regression check against + ground truth (`source_vs_output_product.py`) is explicitly + group-granularity only per its own docstring. +4. **Patient's "174 trackers validated" claim has real test infrastructure + behind it but no committed record of a passing run.** `test_r_validation.py` + is genuine and detailed, but it's `@pytest.mark.slow`/`integration`, + excluded from CI (`python-ci.yml` runs `-m "not slow and not + integration"`), and gated on an external USB-drive path — no + `VALIDATION_TRACKING.md` or equivalent artifact exists in the repo + recording that the 174-tracker run actually passed. +5. Two documentation citations point at evidence that doesn't exist in the + repo: `Ali_internship/residual_dig.ipynb` (cited in + `PYTHON_IMPROVEMENTS.md` §5) and a contributor's local Windows plan path + (cited in `source_vs_output_product.py`'s docstring). + +**Because:** All of the above verified via `git diff`/`git show`/`git +ls-tree`/`rg` against `migration` and `origin/product-pipeline` — commands +logged in the research file's final section for reproducibility. Full +concrete gap lists (7 items for product, 4 for patient) are in the research +file's §4. + +**Rejected:** N/A — this is a research ticket; nothing was decided that had +a rejected alternative. The assumption this ticket was meant to test +(["Assumptions in force": patient's completeness is unverified](../map.md)) +is **confirmed as real**, not overturned: patient's R-logic coverage is +solid, but "complete, validated, deployed to production" is still not +backed by a committed pass artifact. + +**Evidence:** executed — every claim is backed by a specific `git`/`rg` +command reproduced in the research file, not by reading prose or asserting +from memory. + +**Tense:** current — describes the state of `migration` and +`origin/product-pipeline` as of 2026-08-08. + +**Spawned:** none yet as fresh tickets — the concrete gap lists in the +research file's §4 are candidate `task`-type tickets for later, once the +pytest-parity ticket (8) decides which of them are in scope for "pytest +parity" specifically vs. left as follow-up work; ticketing them now would +prejudge that decision. diff --git a/docs/wayfinder/tickets/08-pytest-suite-parity.md b/docs/wayfinder/tickets/08-pytest-suite-parity.md new file mode 100644 index 0000000..da023ee --- /dev/null +++ b/docs/wayfinder/tickets/08-pytest-suite-parity.md @@ -0,0 +1,117 @@ +--- +id: 8 +title: Does the pytest suite reach unit/integration/e2e/regression parity between patient and product, excluding any R-comparison/USB-drive-dependent tests? +labels: [wayfinder:grilling] +status: closed +blocked_by: [7] +assignee: session-2026-08-08 +claimed_at: 2026-08-08 +resolution: decided +evidence: judgement +closed_by: null +spawned_by: 1 +--- + +## Premise + +Rests on [Does product-pipeline's test suite meet the same cell-by-cell rigor +as patient's?](01-product-pipeline-test-rigor.md), closed as superseded: that +session decided R-vs-Python comparison does not belong in pytest (it's +analysis, owned by [Retire the PDF/notebook analysis docs for an automated, +script-based report](02-documentation-strategy.md)), and that pytest's job is +the ordinary category of unit/integration/e2e/regression tests that don't +require the USB drive. + +Blocked on [Is the product pipeline (and patient's own claimed completeness) +actually complete and sound, audited against R's product logic and patient's +structure?](07-pipeline-completeness-audit.md) — that ticket produces the +file-by-file test-coverage inventory this ticket needs as its starting fact +base; deciding pytest-suite parity before the inventory exists would mean +re-deriving it from scratch mid-discussion. + +## Question + +Given the inventory from ticket 7: what, concretely, does pytest-suite parity +between patient and product require — same module-level unit test coverage, +same integration/e2e shape, same regression-test discipline for known fixed +issues? Decide what to do with patient's `test_r_validation.py` specifically +(move out of pytest entirely, e.g. into whatever ticket 2's automated report +becomes, or keep as an opt-in `@pytest.mark.slow` marker but stop treating it +as part of "the test suite" proper) and whether product needs an equivalent +non-R-comparison regression suite before this ticket can close. + +## Resolution + +**Decision:** Parity means: (1) a 85%+ line-coverage floor across the pipeline, +enforced in CI via `--cov-fail-under=85` added to the existing +`pytest -m "not slow and not integration" --cov --cov-report=xml` line in +`.github/workflows/python-ci.yml`; (2) product gets the three integration/e2e +files it is currently missing entirely — `test_integration/test_extract_integration.py`, +`test_integration/test_clean_integration.py`, `test_integration/test_e2e.py` — +built with the exact same convention patient already uses (fixtures pointing +at real tracker files on the USB drive via `TRACKER_BASE`, `skip_if_missing()` +skipping cleanly when the drive isn't mounted, `@pytest.mark.integration`/`e2e` +excluding them from CI); the new extract-integration test covers +`wide_format.py` (currently zero references anywhere in the product suite); +(3) `tests/test_tables/test_product.py` is added, testing +`create_table_product_data` directly rather than only reaching it indirectly +through `test_link_product_patient.py`. + +**`test_r_validation.py`:** removed from pytest entirely — no opt-in marker, +no product equivalent. It moves out to wherever ticket 2's automated +analysis report ends up living; "the test suite" (patient or product) no +longer contains any R-comparison code, confirming ticket 1's original split. + +**Regression tests — reframed, and split into a new ticket.** The initial +framing (a non-R-comparison edge-case regression suite, "the crown," needing +investigation because of the sensitive-data constraint) was based on a +misunderstanding of what the user meant by "regression test." The actual ask +is golden-master/snapshot testing: a fixed synthetic input tracker, with +every pipeline stage's output (raw extract, cleaned, tables) committed as a +snapshot; any future code change that alters output unexpectedly fails the +test, and an intended change gets its snapshot deliberately updated. This +needs no real/sensitive data at all — the point is output stability across +changes, not correctness against source, so a small synthetic fixture +tracker suffices (correctness against source truth is what +`test_validate/test_*_source_vs_output.py` already checks). No snapshot +library exists in the repo yet (`syrupy`/similar absent from +`pyproject.toml`/`uv.lock`). This is real, separate infrastructure work +(pick and add `syrupy`, design the fixture, wire snapshots per pipeline +stage) rather than a fact this ticket can close by deciding — spun off as +[Add golden-master/snapshot regression tests for patient and product](09-snapshot-regression-tests.md), +explicitly deferred by the user until both pipelines' other test suites are +in place and green, so there is a stable base to snapshot against. + +**Because:** The integration/e2e approach mirrors what patient already does +successfully rather than inventing a new pattern — same fixture convention, +same CI exclusion, same drive-gating. The coverage floor gives "parity" a +checkable number instead of leaving it qualitative. Splitting snapshot +testing out keeps this ticket's decision (what parity requires) separate +from a design/build task (how to implement golden-master tests), consistent +with the map's own split-rather-than-sprawl rule. + +**Rejected:** +- A product equivalent of `test_r_validation.py` — rejected per ticket 1's + standing decision (R-comparison is analysis, not a pytest concern); would + have re-introduced exactly what was ruled out. +- Deciding golden-master/snapshot testing's design inline in this ticket — + rejected: it's build work (pick a library, design fixtures, wire multi-stage + snapshots) not a fact to establish, and the user explicitly wants it + deferred until both pipelines are green, so deciding its shape now would be + premature relative to when it will actually be built. +- No coverage floor / qualitative-only parity — rejected: "same test + categories" without a number leaves parity unfalsifiable; 85% gives CI + something to actually enforce. + +**Evidence:** judgement (the parity bar and the split decisions are user +calls; the supporting facts — CI's current pytest invocation, patient's +fixture/skip-if-missing convention, the absence of a snapshot library — were +verified via `grep`/`git show`, i.e. read/executed, but the resolution as a +whole is stamped at the weaker judgement level). + +**Tense:** current for all cited facts (CI config, patient's fixture +convention, dependency list) as of 2026-08-08; the parity requirements and +the snapshot-test plan are consequences of this decision, not yet built. + +**Spawned:** +[Add golden-master/snapshot regression tests for patient and product](09-snapshot-regression-tests.md). diff --git a/docs/wayfinder/tickets/09-snapshot-regression-tests.md b/docs/wayfinder/tickets/09-snapshot-regression-tests.md new file mode 100644 index 0000000..03fd8b6 --- /dev/null +++ b/docs/wayfinder/tickets/09-snapshot-regression-tests.md @@ -0,0 +1,52 @@ +--- +id: 9 +title: Add golden-master/snapshot regression tests for patient and product +labels: [wayfinder:task] +status: open +blocked_by: [6] +assignee: null +claimed_at: null +resolution: null +evidence: null +closed_by: null +spawned_by: 8 +--- + +## Premise + +Rests on [Does the pytest suite reach unit/integration/e2e/regression parity +between patient and product, excluding any R-comparison/USB-drive-dependent +tests?](08-pytest-suite-parity.md): "regression test" here means +golden-master/snapshot testing, not R-comparison or edge-case testing — a +fixed synthetic input tracker, with every pipeline stage's output (raw +extract, cleaned, tables) committed as a snapshot, so an unintended output +change from a future code edit fails the test while an intended one gets its +snapshot deliberately updated. It needs no real/sensitive data — correctness +against source truth is already covered by +`test_validate/test_*_source_vs_output.py`; this only checks output +stability across changes. No snapshot library exists in the repo yet +(`syrupy`/similar absent from `pyproject.toml`/`uv.lock`). + +Blocked on [Promote migration into dev via PR #2](06-promote-migration-to-dev.md) +per the user's explicit instruction: build this only once both pipelines' +other test suites (unit/integration/e2e per ticket 8, plus whatever ticket 2 +and the rest of the merge/CI/production-verification chain settles) are in +place and green, so there is a stable, trusted base to snapshot against +rather than freezing a moving target. + +The fixture tracker is likely the same synthetic source data already used +elsewhere in the suite (per the user, "yes it will be the same source data +likely") rather than a new one purpose-built for this ticket — confirm +against whatever fixture tracker(s) exist in `tests/` by the time this is +picked up. + +## Question + +Pick and add a snapshot-testing library (`syrupy` is the standard choice for +pytest — confirm or reconsider at pickup time), design one fixed synthetic +input fixture per pipeline (patient, product) reusing existing test fixture +data where possible, and wire snapshot tests covering each pipeline stage's +output (raw extract, cleaned, final tables) so that any future code change +producing an unintended output delta fails the test, with a clear, +documented path to intentionally update a snapshot when the change is +deliberate. diff --git a/docs/wayfinder/tickets/10-performance-profiling.md b/docs/wayfinder/tickets/10-performance-profiling.md new file mode 100644 index 0000000..dd79300 --- /dev/null +++ b/docs/wayfinder/tickets/10-performance-profiling.md @@ -0,0 +1,121 @@ +--- +id: 10 +title: Profile the combined pipeline's performance against the R baseline before promoting to dev +labels: [wayfinder:task] +status: closed +blocked_by: [3, 13] +assignee: session-2026-08-09d +claimed_at: 2026-08-09 +resolution: decided +evidence: executed +closed_by: null +spawned_by: null +--- + +## Premise + +Rests on [Merge product-pipeline (PR #6) into migration](03-merge-product-pipeline.md), +closed: `migration` now carries the combined patient + product pipeline, plus +everything else landed in that merge (the two logging-parity fixes, the +product-only coverage gate). The user's standing understanding is that the +Python pipeline is much faster than the archived R pipeline, but that +comparison predates this merge's additions and the product arm's own cost — +nobody has re-profiled since. This is a fresh concern raised mid-[ticket +5](05-production-verification-run.md) session, not derived from any ticket's +resolution. + +Not blocked on [ticket 5](05-production-verification-run.md) itself — profiling +can run against local or production-scale data independent of that ticket's +verification outcome — but both are gating promotion (ticket 6), and ticket 5's +production run may end up a convenient source of real timing data if it's +still fresh when this ticket is worked. + +**Now also blocked on** [Audit and update all dependencies and library +versions before rollout](13-dependency-audit.md) — added after it surfaced +mid-session too: profiling against a dependency set that's about to be +updated would make the numbers stale immediately, so the audit needs to land +first. + +## Question + +Define what "profile before going live" means concretely: which pipeline +stages to measure (extract/clean/tables per arm, or end-to-end), what data +volume to profile against (local fixtures vs. a full production-scale run), +what the R baseline comparison point is (the archived R pipeline's own past +timings, if any were recorded, or a fresh run of `r-archive/` for comparison), +and what regression or absolute-time threshold would block promotion to `dev`. +Then execute the profiling and record the result. + +## Resolution + +**Decision:** No R baseline comparison — the user corrected the ticket's +original framing mid-session: R is already known to be much slower, so +re-confirming that is a wasted exercise. Instead this ticket became a +function-level performance and robustness profile of the Python pipeline +itself, to help answer "is this strong enough to merge to dev" — not a +rewrite, but finding and fixing concrete issues surfaced by the profile +(the user's explicit framing). + +**Method:** `pyinstrument` (statistical/sampling profiler, chosen for low +overhead and a readable call-tree vs. `cProfile`'s flat stats table) wrapped +around `a4d.cli process-patient` / `process-product`, run with `--workers 1` +so the profiler's single process sees inside the pipeline rather than losing +visibility into `ProcessPoolExecutor` subprocess workers. Data: the full +177-tracker real production dataset from the USB drive +(`/Volumes/USB SanDisk 3.2Gen1 Media/a4d/a4dphase2_upload`), at the user's +suggestion, since the current production trackers aren't available locally +outside GCP. + +**Found and fixed:** `find_data_start_row` (`src/a4d/extract/patient.py`) +looped calling `ws.cell(row_idx, 1).value` once per row until it found the +first numeric value in column A. On a `read_only=True` openpyxl worksheet, +`ReadOnlyWorksheet._get_cell` re-parses the sheet's XML from row 1 on *every* +call (`_cells_by_row` reopens the XML source and re-runs `WorkSheetParser` +each time) — read-only worksheets are built for sequential streaming, not +random access. A per-row `.cell()` loop is therefore O(n^2) in the row count +before data starts. Profiling one large real tracker +(`2019_Yangon Children's Hospital A4D Tracker.xlsx`) showed this function +alone consuming 7.885s of 8.965s total (88%) for that single file. Fixed by +replacing the loop with a single `ws.iter_rows(min_col=1, max_col=1, +values_only=True)` sequential scan (`_cells_by_row` called once regardless +of row count). Verified: same file now processes in 1.376s (6.8x), and the +full patient arm across all 171 real trackers (4 parallel workers, unprofiled +wall-clock, identical output row counts before/after — 1,457 static / 64,007 +monthly / 2,218 annual / 37,030 errors) went from 145.8s to 22.0s (6.6x). Full +combined patient+product run (`run-pipeline --skip-download --skip-upload +--skip-drive-download --force`, 4 workers): 73.86s wall, ~556MB peak RSS +(macOS `/usr/bin/time -l` on the top-level process only — doesn't capture +`ProcessPoolExecutor` worker subprocess memory, so treat as a rough floor, +not exhaustive). Added a regression test +(`tests/test_extract/test_patient_helpers.py::test_scans_read_only_worksheet_in_one_pass`) +that spies on `ReadOnlyWorksheet._cells_by_row` and asserts it's called +exactly once on a dense sheet — confirmed it fails against the pre-fix code +and passes against the fix. All 489 tests, ruff, and `ty check src/` pass. +Pushed as part of this ticket's session. + +**Other finding, not fixed here:** running `process-product` against the +same full dataset surfaced 4 trackers failing outright with `unable to find +column "product"; valid columns: ["index"]` (2020 Jayavarman VII, and 2018/ +2019/2020 Kantha Bopha) — reproduces identically before and after this +ticket's fix, so unrelated to it, and not previously documented anywhere on +this map. Out of this ticket's scope (a correctness bug, not a performance +one); spawned [ticket 14](14-product-column-detection-failures.md) rather +than fixed inline, to keep this session to one ticket. + +**Rejected:** a full architecture re-review beyond what the profile itself +surfaced — general code-quality review already happened in [ticket +3](03-merge-product-pipeline.md)'s merge review, and CLI/observability is +[ticket 11](11-cli-ux-observability.md)'s separate territory; re-litigating +either here would have sprawled past one ticket. A stricter regression/ +absolute-time threshold to gate promotion — not set, since the R-comparison +framing that would have anchored one was explicitly dropped; 73.86s for the +full combined real-dataset run is judged fast enough on its own that no +threshold is needed to justify promotion on performance grounds. + +**Evidence:** executed throughout — every number above (profile self-times, +before/after wall-clock timings, test pass/fail, memory) came from running +the actual code against real production-scale data, not read from docs or +inferred. + +**Tense:** all current behaviour — the fix is merged into this branch's +working tree, not a proposal. diff --git a/docs/wayfinder/tickets/11-cli-ux-observability.md b/docs/wayfinder/tickets/11-cli-ux-observability.md new file mode 100644 index 0000000..9b19f81 --- /dev/null +++ b/docs/wayfinder/tickets/11-cli-ux-observability.md @@ -0,0 +1,135 @@ +--- +id: 11 +title: Decide what CLI/TUI UX and error-log observability improvements admins/developers need before rollout +labels: [wayfinder:grilling] +status: closed +blocked_by: [3] +assignee: session-2026-08-09g +claimed_at: 2026-08-09 +resolution: decided +evidence: executed +closed_by: null +spawned_by: null +--- + +## Premise + +Rests on [Merge product-pipeline (PR #6) into migration](03-merge-product-pipeline.md), +closed: `migration` now carries the combined patient + product pipeline, its +CLI (`a4d.cli`), and both arms' error/logging paths (including the two +logging-parity fixes from that merge — product now populates +`TrackerResult.data_errors` and a logs/errors table like patient does). + +Raised mid-[ticket 5](05-production-verification-run.md) session as a fresh +concern, not derived from any ticket's resolution. Initially parked as fog +("Not yet specified") because "improve whatever helps" wasn't a sharp +decision and it wasn't clear the concern was even in this map's scope; the +user then confirmed explicitly that it belongs to this effort — this map's +destination has been redrawn to include it (see Destination section, "2026-08-09"). + +## Question + +What does "good enough to operate the pipeline day to day" mean concretely +for admins/developers, covering both: + +- **CLI/TUI UX**: is `a4d.cli`'s current command surface, output, and + ergonomics (Typer-based; commands listed in `docs/CLAUDE.md`) sufficient for + someone running/debugging a pipeline execution, or are there specific gaps + (progress visibility during a long run, clearer error summaries, easier + local reproduction of a failed production run, etc.)? +- **Error-log observability**: whether the existing error surfaces (the + `logs` BigQuery table, `table_errors.parquet`, Cloud Run Job logs via `just + logs-job`) give enough to observe and debug a run without reading raw JSON + logs line by line — e.g. is a summary view, a dashboard, or a CLI subcommand + needed? + +Scope to what's needed for confident rollout, not a general UX overhaul — +name concrete gaps against concrete use cases (a run fails partway; an admin +wants to know how many trackers had errors and why) rather than redesigning +the CLI from scratch. + +**Prior art (read before deletion, not to be ported as-is):** `tools/LogViewerA4D` +was an R Shiny dashboard, since deleted (see [ticket +12](12-retire-r-workspace.md)), that read local tab-separated `.log` files +(`Timestamp/Thread/Level/Package/Function/Message`) and offered three views: a +main log table plus a per-tracker summary and Sankey diagram, an overview tab +with a regex-filterable plot across tracker files, and a reference-data +validation tab flagging missing/duplicate `clinic_id` entries against +`reference_data/`. Useful as a reference for what admins actually wanted to +see, but its whole design assumes local log files — the current pipeline logs +via loguru into the `logs` BigQuery table (1M+ rows) instead, so any Python +equivalent would need a different shape, not a port. + +## Resolution + +**Decision:** The CLI/TUI UX question is answered for the "normal a4d output" +half of the scope; the observability half is split off into a separate +deliverable (spawned as [ticket +16](16-log-analyzer-drill-down.md)) rather than answered here. + +Concretely, this session: + +1. **Discovered a real gap by demonstration, not just reasoning**: built a + small synthetic dataset (clean tracker, a data-quality-error tracker, an + outright-unparseable tracker) and ran `process-patient` against it to see + actual CLI output. That surfaced that `process-patient`/`process-product` + already render an "Error Type Breakdown" and "Top Files by Error Count" + table (richer than this ticket's own description assumed) — but + `run-pipeline`, the actual production entry point behind the Cloud Run + Job, never calls any of that rendering at all. It only prints a one-line + `✓ Processed N trackers (X ok, Y failed)` per arm, with no cross-arm view. +2. **User confirmed the shape of the fix**: keep the current per-arm summary + tables, extend `run-pipeline`'s output with a combined view that answers + "how many files could not be processed and why — total lost, only + patient, only product — and how many errors per file", scoped to a + patient-ok/product-ok level (not a further extract/clean/tables per-stage + breakdown — noted as a possible future refinement, not built). +3. **Implemented and shipped it** (this map carries execution per its Notes): + added `_render_combined_run_summary()` to `src/a4d/cli.py`, wired into + `run_pipeline_cmd` after both arms run. It crosses each tracker file's + patient/product outcome into four buckets (both ok / patient failed only + / product failed only / lost entirely), lists the files needing attention + with each arm's error text, and merges both arms' per-file error counts + into one "Top Files by Error Count" table. Only renders when both arms + actually ran (skips cleanly under `--skip-patient`/`--skip-product`). +4. **Found and fixed a real behavior bug this exposed**: `run-pipeline` + previously aborted the entire run (`raise typer.Exit(1)`) the instant any + single patient tracker failed — before the product arm ever ran. That + meant the combined view's "patient-only failed" vs "both failed" + distinction could never actually be observed in a real run. Confirmed + with the user this should switch to the same soft-fail-and-continue + posture the product arm already uses; implemented the switch. +5. **Found and fixed a real test-isolation bug surfaced by that change**: + two existing `run-pipeline` CLI tests mocked only `run_patient_pipeline`, + not `run_product_pipeline`. Because `run_product_pipeline` binds + `settings` at its own module-import time, patching `a4d.config.settings` + in the test did not stop it discovering trackers from the real local + `data_root` when actually invoked. Under the old abort-on-patient-failure + code this was never reached in the failing-patient test; the soft-fail + change now lets execution reach it, and running the test once (before + the fix) processed 185 real tracker files from local disk as a side + effect. Fixed by mocking both arms in every `run-pipeline` test — this + generalizes beyond this ticket's own new tests. + +**Because:** distinguishing "lost entirely" from "one arm degraded" is +exactly the question an admin needs answered after a run, and it was +structurally impossible to answer with the abort-on-any-patient-failure +behavior in place — fixing the summary without fixing the abort would have +shipped a view that could never show its most important row in practice. + +**Rejected:** building the log-analyzer drill-down tool (open a specific +file, see every error/log line — replacing `LogViewerA4D`'s job) in this +same session — it's a genuinely separate, larger deliverable (querying +`table_logs`/`table_errors` by file, likely its own subcommand or small +TUI/dashboard), not an extension of the run summary tables. Spawned as +[ticket 16](16-log-analyzer-drill-down.md) instead of sprawling this +ticket. + +**Evidence:** executed — real CLI run against synthetic data (not real +patient trackers, per this map's guardrails) showing actual output; the +combined-summary implementation is covered by new unit tests +(`TestCombinedRunSummary` in `tests/test_cli/test_cli.py`); full suite (494 +tests), ruff, and `ty check src/` all pass. Pushed as `33694b4`. + +**Tense:** all claims above describe current (post-commit) behavior, not a +proposal. diff --git a/docs/wayfinder/tickets/12-retire-r-workspace.md b/docs/wayfinder/tickets/12-retire-r-workspace.md new file mode 100644 index 0000000..04f258a --- /dev/null +++ b/docs/wayfinder/tickets/12-retire-r-workspace.md @@ -0,0 +1,438 @@ +--- +id: 12 +title: Retire R from the workspace once the pipeline is fully verified Python-only +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-24f +claimed_at: 2026-08-24 +resolution: decided +evidence: executed +closed_by: null +spawned_by: null +--- + +## Standing rule for this ticket's blocking + +**R is retired only when no open ticket, and nothing in the codebase, still +needs to read the old R source for comparison** (stated by the user +2026-08-20, confirming the wiring round 10 applied). + +`blocked_by` is therefore a *derived* list, not a fixed one: it holds whichever +open tickets currently need R, and it is re-derived whenever a ticket closes or +a new one is spawned. A ticket spawned later that needs to read `r-archive/` +gets added here without further argument -- that is the rule working, not a +delay. The frozen output baseline on the data drive +(`/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_r/`) is not part of this: it +lives outside the repo and this ticket never touches it. What is gated is the +**R source** in `r-archive/`, which past triage has repeatedly had to read to +root-cause a mismatch -- reading R's output alone has not been enough. + +This ticket was unblocked prematurely once (2026-08-15) by treating spawned +residuals as out of scope, and the correction cost a session. When in doubt, +it stays blocked. + +## Premise + +Rests on the same destination redraw as [ticket 10](10-performance-profiling.md) +and [ticket 11](11-cli-ux-observability.md): the user confirmed workspace +cleanup is part of "are we really ready to roll out", not separate follow-on +work. + +[Retire the PDF/notebook analysis docs for an automated, script-based +report](02-documentation-strategy.md) is now closed: the comparison script's +design decided the R baseline is the already-frozen +`/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_r/`, which lives on the data +drive, not in this repo — so this ticket's original premise ("need R +available as a live reference to check against") no longer applies, since +retiring `r-archive/` was never going to touch that frozen baseline. +[Profile the combined pipeline's performance against the R baseline before +promoting to dev](10-performance-profiling.md) is also closed (R comparison +was dropped from that ticket entirely). [Build and run the R/Python output +comparison script, then triage every flagged +difference](15-build-and-run-comparison-script.md) is now closed too — it +built and ran the comparison tooling but split the actual triage into +[ticket 17](17-fix-product-row-alignment-and-triage.md) after finding +product's row-alignment key was broken. Ticket 17 is now closed too — it +fixed and validated the row-alignment key but the actual per-column triage +still didn't converge, splitting again into [ticket +18](18-triage-comparison-flagged-differences.md). Ticket 18 is now closed +too — it root-caused `product_category` and most of `product_entry_date`, +but the remaining columns and the entire patient arm still didn't converge, +splitting into [ticket 20](20-fix-raw-entry-date-representation.md), +[ticket 21](21-triage-remaining-product-columns.md), [ticket +22](22-triage-product-raw-columns.md), and [ticket +23](23-triage-patient-arm.md). This ticket is blocked on all four instead of +ticket 18 — not because retiring `r-archive/` needs R itself, but because +that triage work is still investigative and might turn up a need to consult +R's logic/behavior one more time before it's gone for good; safer to keep +this ticket blocked until that work concludes. + +**Correction (2026-08-12c):** tickets 20-23 closing does not mean this +ticket is actually unblocked — that was a premise error caught mid-session, +before any commit landed. Tickets 20-23 spawned five further residual +tickets (24, 25, 26, 27, 28) that carry forward the exact same "might need +to consult R's source" risk this ticket already names as its reason for +waiting: past triage repeatedly needed to read `r-archive/`'s actual R +source to root-cause a mismatch (e.g. ticket 18 reading +`read_product_data.R`'s `add_product_categories()` join logic directly), +not just diff output. Ticket 28 in particular (patient cleaned-stage, +120,639 mismatches across 61 columns) is entirely untouched and most likely +to need the same. `blocked_by` corrected to `[20, 21, 22, 23, 24, 25, 26, +27, 28]`. No files were deleted as part of this correction — `r-archive/` +was briefly staged for removal outside this ticket's own execution and was +restored before anything was committed. + +Not blocked on [ticket 11](11-cli-ux-observability.md): the user confirmed +`tools/LogViewerA4D` (an R Shiny app another developer wrote for viewing R +logs) needs no Python replacement — it can simply be deleted, independent of +whatever ticket 11 decides for the Python CLI's own observability. + +**Same precedent applied again (2026-08-12f):** [ticket +27](27-triage-patient-raw-residual.md) closed for its two original leads +(fixed) but explicitly left `complication_screening` (12,566 mismatches, +84% of what remains) and roughly 50 smaller columns untouched, spawning +[ticket 31](31-triage-patient-raw-residual-2.md) to carry that forward. +`blocked_by` swaps `27` for `31`. + +**Same precedent applied a third time (2026-08-17b):** [ticket +43](43-triage-patient-raw-residual-3.md) closed for the two causes it +settled -- a comparison-tool rounding gap and a real Python defect that +invented 24 patient rows -- leaving 1,409 unclassified patient raw-stage +mismatches. Those split into [ticket +45](45-patient-row-alignment-duplicate-keys.md) (duplicate-key join fan-out, +57% of the residual) and [ticket +46](46-triage-patient-raw-residual-4.md) (the long tail). `blocked_by` swaps +`43` for `45, 46`. Ticket 43 itself again had to read +`script1_helper_read_patient_data.R` to establish how R bounds a sheet's +data block, so R's source remains a live reference for the triage that +remains -- the same reason this ticket has waited each time. + +**2026-08-17c:** [Ticket 45](45-patient-row-alignment-duplicate-keys.md) closed +-- patient rows now align on `patient_id` + `sheet_name` with a content-matched +tie-break, and the duplicate-key fan-out is gone (raw unclassified 1,409 -> +601). `blocked_by` drops `45`, leaving **ticket 46 alone**. Ticket 45 also +spawned [ticket 47](47-patient-ids-merged-at-cleaning.md), deliberately *not* +added here: both pipelines merge those identities identically, so it is a +Python-only question that needs no R reference. + +**Conflict on record, not resolved here:** `CLAUDE.md` currently states +`r-archive/` is "preserved for reference. Do not modify." — that instruction +will need to change (or be explicitly overridden) as part of resolving this +ticket, not before. The user has confirmed intent to remove R once everything +is verified, and noted the deletion is low-risk regardless — it stays in git +history (this branch's own history, pre-squash) even after removal. + +Inventory of what "remove R" touches (verified via `git ls-files`, not +assumed): +- `r-archive/` (1.9M, git-tracked) — the archived R package — **remaining** +- ~~`tools/LogViewerA4D/` (git-tracked) — R Shiny log-viewing app~~ — + **deleted early** (2026-08-09, out of sequence): the user confirmed it + needed no Python replacement before this ticket's other blockers closed, so + it was removed immediately rather than waiting; its feature set was first + captured in [ticket 11](11-cli-ux-observability.md) as prior art. This + ticket stays open for the rest of the inventory. +- ~~`test_full_pipeline_debug.R` (repo root) — stray debug script~~ — + **already gone**: verified via `git log --all -- test_full_pipeline_debug.R` + (2026-08-12c) it was removed by an earlier "remove deprecated files" + commit, before this ticket was even written. No action needed. +- `docs/CLAUDE.md`'s note that `reference_data/` is "shared with the archived + R pipeline" — needs re-wording once R is gone — **remaining** +- `CLAUDE.md`'s own "R Archive" section and its "do not modify" instruction + — **remaining** + +## Question + +Once tickets 2 and 10 are closed and the user is confident in rollout: decide +what "remove R" means concretely (delete outright vs. move to a separate +archive location/repo vs. keep read-only in git history only) for each +remaining item in the inventory above, update `CLAUDE.md` and `docs/CLAUDE.md` +to match, and execute the cleanup. + +**Correction (session-2026-08-13b):** ticket 29 closed, spawning ticket 37 +(the patient cleaned-stage residual it did not chase), so `blocked_by` swaps +`29` for `37`. + +## Premise update (session-2026-08-14b) + +[Ticket 38](38-triage-patient-cleaned-date-family.md) closed, so `blocked_by` +drops to `[20, 21, 22, 23, 24, 25, 26, 28, 30, 31]` — **tickets 30 and 31 are +now the only open blockers**, both patient raw-stage triage. Ticket 38's own +spawned residual, [ticket 39](39-recover-dates-embedded-in-free-text.md), is +deliberately *not* wired as a blocker here: it asks what Python should do with +a date buried in a clinical note, which is a question about Python's own +behaviour and needs no reference to R's source, so it survives R's retirement +intact. + +## Premise update (session-2026-08-15b, ticket 31's session) + +[Ticket 31](31-triage-patient-raw-residual-2.md) closed, but it spawned +[ticket 43](43-triage-patient-raw-residual-3.md) -- 1,879 patient raw-stage +mismatches still untriaged -- so `blocked_by` gains 43 rather than emptying. +The reason is the one that corrected this ticket's premise before: ticket 31 +again had to read `r-archive/R/script1_read_patient_data.R` directly (R's +`tidyr::unite` call) to root-cause a divergence, so R's source is still a live +reference for the triage that remains. Ticket 43 is now this ticket's only +open blocker. + +**2026-08-17 (ticket 48 closed).** `blocked_by` drops `48`: the merged-header +data loss is fixed, and its diagnosis needed R only as a reference for what the +right answer looks like -- confirmed by `2023_Chiang Mai`'s R output carrying +the same propagated column names. [Round-5 patient raw +triage](49-triage-patient-raw-residual-5.md) is now the single remaining +blocker. + +**2026-08-18 (ticket 49 closed).** `blocked_by` swaps `49` for `50`. Ticket 49 +did not empty the patient raw-stage residual (229 -> 84) and, more to the +point, needed `r-archive/`'s own source to close: root-causing the Mukdahan +header gap meant reading `script1_helper_read_patient_data.R` and +`script2_sanitize_str.R` and *running* R's sanitizer against the real header to +confirm its Unicode behaviour. That is the same reason this ticket has waited +each round -- [ticket 50](50-triage-patient-raw-residual-6.md) inherits it. + +## Premise update (session-2026-08-18b) + +[Round 6](50-triage-patient-raw-residual-6.md) closed and the patient **raw** +stage is at **zero** unclassified mismatches -- the first stage of either arm to +reach it. That does not unblock this ticket: the cleaned stage still carries +7,967 unclassified mismatches across 27 columns, and that triage is exactly the +kind that has repeatedly had to read `r-archive/`'s own source to root-cause a +difference. `blocked_by` swaps `50` for [ticket +51](51-triage-patient-cleaned-residual-4.md), the cleaned-stage round the raw +work spawned. Same precedent as every swap above. + +## Premise update (session-2026-08-19) + +[Round 4](51-triage-patient-cleaned-residual-4.md) closed, cutting the +cleaned-stage in-scope residual from 5,031 to **2,538** and landing three named +causes -- and its work again turned on reading `r-archive/`'s own source +(`script2_helper_dates.R`'s `ymd`-before-`dmy` order, `script2_sanitize_str.R`'s +Unicode-aware character class, and a grep proving R has no future-date guard at +all). R cannot be retired while triage still needs to read it. `blocked_by` +swaps `51` for [ticket 52](52-triage-patient-cleaned-residual-5.md), the round +it spawned. Same precedent as every swap above. + +**2026-08-19c (ticket 53 closed).** `blocked_by` swaps `53` for [ticket +54](54-triage-patient-cleaned-residual-7.md) on the same standing precedent: +round 6 read R's own `split_bp_in_sys_and_dias` +(`script2_helper_patient_data_fix.R:623-653`) to establish why R keeps a blood +pressure Python was discarding, so triage still depends on `r-archive/` being +present. + +**2026-08-19d (ticket 54 closed).** `blocked_by` swaps `54` for [ticket +55](55-triage-patient-cleaned-residual-8.md) on the same standing precedent: +round 7 read R's own commented-out `fix_t1d_diagnosis_age` call site +(`script2_process_patient_data.R:251`) to establish why R never derives a +diagnosis age, and round 8's first task is reading R's `fix_fbg` for the same +kind of answer. Triage still depends on `r-archive/` being present. + +**2026-08-19e (ticket 55 closed).** `blocked_by` swaps `55` for [ticket +56](56-triage-patient-cleaned-residual-9.md) on the same standing precedent: +round 8 read R's `fix_fbg`, `transform_cm_to_m`, `cut_numeric_value` and the +mutate that sequences them, all directly from `r-archive/`, to establish that +R manufactures glucose readings from text and that its height threshold is the +correct one. Round 9's leading shape points at R's own +`extract_date_from_measurement`. Triage still depends on `r-archive/` being +present. + +**2026-08-19f (ticket 56 closed).** `blocked_by` swaps `56` for [ticket +57](57-triage-patient-cleaned-residual-10.md) on the same standing precedent, +and this round is the sharpest example yet of why R has to stay: closing it +required *installing lubridate and executing* R's `parse_dates` -- its +fourth-letter truncation and its eight-order fallback list -- over the actual +source strings, because reading the function was not enough to predict which +cells it would sentinel and which it would misread. Round 10's largest group is +R's `extract_date_from_measurement`, which will need the same treatment. +Triage still depends on `r-archive/` being present. + +**2026-08-20 (ticket 57 closed -- the round chain ends).** Round 10 converged: +the patient cleaned-stage residual is 19 cells, all with written verdicts, and +there is no round 11. So `blocked_by` does not swap `57` for a successor -- it +drops `57` and gains `[32, 39, 44, 47]`. + +That is a change of kind, not just of contents, so it is argued rather than +applied. This ticket has been swapping one round for the next on a standing +precedent: **R cannot be retired while triage still needs to read it.** With the +rounds finished, the four open tickets that still need R as a live reference +are the ones that inherit that precedent: + +- [Re-audit every existing cause classifier](32-audit-classifiers-against-decision-bar.md) + -- by definition it re-reads R's source to check each verdict. +- [Dates buried in clinical notes](39-recover-dates-embedded-in-free-text.md) + -- 478 cells, still an open question, and round 6 measured 179 of them running + the opposite direction. +- [The cleaned-stage FBG R-null residual](44-triage-cleaned-fbg-r-null-residual.md) + -- 2,936 cells, the single largest unexplained population left anywhere. +- [Four trackers where cleaning merges several patients into one patient + ID](47-patient-ids-merged-at-cleaning.md) -- a correctness question whose + evidence includes R's own row handling. + +The alternative -- treating all four as residuals of already-closed scope and +letting R retire now, which is the precedent this map used for tickets 24-28 -- +was rejected because this ticket has already been unblocked prematurely once +(2026-08-15) on exactly that reasoning, and the correction cost a session. +Reversible: dropping any of the four from `blocked_by` is a one-line edit if the +user judges R unnecessary for it. + +**2026-08-21 (ticket 39 closed).** `blocked_by` drops `39`, leaving `[32, 44]` +of the open tickets that still need `r-archive/`'s source. [Dates buried in +clinical notes](39-recover-dates-embedded-in-free-text.md) needed R read twice +-- to establish that `parse_dates` composes 2019-03-20 out of three separate +dates in one note, and that its order list ends in `y`, which is why a bare +year in prose becomes 1 January -- so it earned its place on this list and has +now spent it. + +**2026-08-21b (ticket 44 closed).** `blocked_by` drops `44`, leaving **`[32]` +alone** -- [the classifier re-audit](32-audit-classifiers-against-decision-bar.md) +is the last open ticket on this map that needs `r-archive/`'s source, and it +needs it by definition, since re-auditing a classifier means re-reading the R +function it claims to describe. [The FBG R-null +residual](44-triage-cleaned-fbg-r-null-residual.md) spent its place on `fix_fbg` +and R's `fbg/18` cross-derivation, both read to establish that the mmol column +is derived rather than recorded on either side. + +That makes this ticket **one decision away from the frontier** for the second +time. The first attempt was premature (see above), and the difference now is +that the reason for waiting has narrowed to a single named ticket rather than a +population of residuals nobody had counted. + +**2026-08-22 (ticket 32 closed).** `blocked_by` was `[32]` in substance. Ticket +32 closed, but it split: [the second half of the +audit](60-audit-remaining-pre-bar-classifiers.md) inherits eight pre-bar causes +whose mechanisms are statements about R's own source (`r_extraction_gap`, +`r_validator_rejects_multivalue`, `r_category_lookup_miss`, +`buddhist_era_typo`), so it needs `r-archive/` by definition -- the same +standing precedent that has governed this list for ten rounds. + +Re-derived rather than swapped, per this ticket's rule. `blocked_by` is now +**`[59, 60]`**: + +- **60** for the reason above. +- **59** ([rows that pair with nothing](59-triage-unmatched-row-keys.md)) is + added, and this is a *change of judgement* rather than a new fact: its largest + population is 98 R-only rows in `2026_Preah Kossamak`, and explaining why R + emits rows Python does not is a question about R's row handling, not about + Python's output. Ticket 58 stays off the list on the distinction the previous + session drew -- its evidence is Python's own join and the source workbooks. +- **61** ([Buddhist-era conversion](61-decide-buddhist-era-date-conversion.md)) + is deliberately *not* a blocker: the one fact it needs about R -- that R does + not convert BE dates -- is already established from R's frozen output, and the + decision itself is about what Python should publish. + +**2026-08-24 ([rows that pair with nothing](59-triage-unmatched-row-keys.md) +closed).** `blocked_by` drops `59`, leaving **`[60]`**. The judgement that put +59 on the list held: answering it did require reading `r-archive/`'s source -- +`script1_helper_read_patient_data.R:67`, R's hardcoded `+1` row offset for +2022+ trackers, which is what makes R drop `2024_Mandalay Children's` first +patient from 11 of 12 sheets. Re-derived rather than swapped: of the seven +tickets still open, only [the eight pre-bar +causes](60-audit-remaining-pre-bar-classifiers.md) needs the R source, and it +needs it by definition. **One ticket now stands between this map and retiring +R.** + +**2026-08-24d ([the Patient List join](58-patient-list-join-uses-unfixed-id.md) +closed).** `blocked_by` swaps `58` for **`[63]`**. Ticket 58's own "check R" +turned out **not** to need `r-archive/`'s source at all -- the answer came from +R's frozen *output* (its cleaned `2024_Mahosot` carries the same null block +starting at `LA_MH056`), which the map's Notes already rule out of this +ticket's scope. So the previous session's judgement that 58 was a blocker was +wrong, and it is recorded here rather than quietly dropped. + +Re-derived across all nine open tickets: [the cleaned-stage static-join +divergence](63-name-the-cleaned-stage-static-join-divergence.md) is the only +one that needs the R source. Its question 2 asks what mechanism leaves R null +on `bmi`, `age` and `t1d_diagnosis_age` for these patients when the month +sheets carry those columns -- a question about R's own code, not about its +output. Tickets 6, 9, 16, 34, 35, 40 and 41 were each checked again and need +none. **One ticket still stands between this map and retiring R.** + +## The list is now empty (re-derived 2026-08-24, session e) + +Closing [ticket 63](63-name-the-cleaned-stage-static-join-divergence.md) leaves +**no open ticket that needs the R source**. Re-derived rather than inherited, +per the rule above: tickets 6, 9, 16, 34, 35, 40 and 41 were each checked and +none asks a question about R's code -- verified by grepping every open ticket +body for `r-archive`, `R source`, `script_`, `readxl` and `openxlsx`, which +matches nothing. Ticket 63 itself needed R only for its frozen *output*, which +this ticket's scope excludes. + +Nothing in the codebase reads `r-archive/` at runtime either: the only +references outside `docs/` are `pyproject.toml`'s two lint exclusions and +**eight docstring citations** in `src/a4d/migration/compare.py`, which name the +R file and line each cause was traced to. + +**That is this ticket's one open question, not a blocker.** Those citations are +the evidence for the cause registry, and deleting `r-archive/` makes them +unverifiable by anyone reading them later. Whoever takes this decides whether +that is acceptable, whether the cited excerpts get inlined first, or whether +the archive moves rather than dies. + +## Resolution (session-2026-08-24f) + +**Decision.** `r-archive/` is **deleted outright** from the working tree -- all +156 tracked files, 1.9M -- and recovered from git history when needed. The +commit that removes it carries the annotated tag **`r-archive-removed`**, so +the last state containing R is addressable as `r-archive-removed^` without +anyone having to know a SHA: + +```bash +git show r-archive-removed^:r-archive/R/script2_process_patient_data.R +git checkout r-archive-removed^ -- r-archive +``` + +`CLAUDE.md`'s "R Archive" section (and its "preserved for reference. Do not +modify." instruction -- the conflict this ticket was written to resolve) is +replaced by that recovery recipe. `docs/CLAUDE.md`'s "`reference_data/` is +shared with the archived R pipeline" note now describes `reference_data/` for +what it is; `SETUP.md` no longer points new developers at a directory that does +not exist; `pyproject.toml`'s ruff `extend-exclude` drops `r-archive` (leaving +`docs/archive`, ticket 33's entry), and the eleven-line comment above it +loses its R paragraph. Two dead ignore rules went with it: `.dockerignore`'s +`r-archive/` and `.gitignore`'s `r-archive/config.yml`, `.Rproj.user` and +`.Rhistory`. + +**Because.** The archive's only remaining role was as evidence for docstring +citations, and a reader auditing one of those is already in git. Git history is +the durable copy: it costs nothing to keep, cannot drift from the thing it +documents, and the tag makes it as cheap to reach as the directory was. The +user had already judged the deletion low-risk on exactly this ground. + +**Rejected.** +- *Inline the cited R excerpts into the docstrings first, then delete.* Killed + by measurement: the citation count is **35 across 12 modules**, not the 8 in + `compare.py` this ticket's own inventory claimed, so inlining means editing + production cleaning code at 35 sites and permanently lengthening docstrings + that are already long, to serve an audit that happens rarely. It also + entrenches the framing the user wants gone. +- *Move `r-archive/` to a separate repo or a tag-only artifact and repoint the + citations there.* Rejected as a second copy to maintain and a link that can + rot -- the map's own standing rule against hand-maintained duplicates. +- *Keep it.* Not viable: the ticket's standing rule ("R retires only when + nothing still needs to read it") ran out of claimants when [ticket + 63](63-name-the-cleaned-stage-static-join-divergence.md) closed, and keeping + it indefinitely was never the plan. + +**What this ticket did not do, deliberately.** The 35 R citations are still in +the code. Removing them is not a find-and-replace -- the user's framing is that +"to match R" explains nothing now and is sometimes actively wrong (round 4 +found `_validate_dates` claiming its future-date guard "matches R pipeline +behavior" when R has no such guard at all). That is spawned as [ticket +64](64-documentation-overhaul-drop-r-framing.md), which also has to decide +whether `compare.py`'s cause-registry docstrings are the legitimate exception, +since R divergence is literally their subject. + +**Evidence.** Executed. The inventory was re-derived rather than inherited: +`git ls-files r-archive` (156 files), `du -sh` (1.9M), and a repo-wide `rg` for +`r-archive|r_archive|LogViewerA4D` across every non-`docs/` path. That grep is +what caught the citation undercount -- the ticket's figure came from matching +the literal string `r-archive`, which misses every docstring naming an R file +without its path (`script2_process_patient_data.R`), and a second grep for +`script[0-9]_[a-z_]*\.R` found 35. `tools/LogViewerA4D/` and +`test_full_pipeline_debug.R` were confirmed already gone. `SETUP.md`, +`.dockerignore` and `.gitignore` were **not** in this ticket's inventory and +were found by the same sweep. Full test suite, ruff and `ty check src/` run +after the deletion. + +**Tense.** All of the above describes the state after this session's changes, +which are committed. The 35 surviving citations are current behaviour, not a +consequence of a proposed design. diff --git a/docs/wayfinder/tickets/13-dependency-audit.md b/docs/wayfinder/tickets/13-dependency-audit.md new file mode 100644 index 0000000..7813a8d --- /dev/null +++ b/docs/wayfinder/tickets/13-dependency-audit.md @@ -0,0 +1,123 @@ +--- +id: 13 +title: Audit and update all dependencies and library versions before rollout +labels: [wayfinder:task] +status: closed +blocked_by: [3] +assignee: session-2026-08-09c +claimed_at: 2026-08-09 +resolution: decided +evidence: executed +closed_by: null +spawned_by: null +--- + +## Premise + +Rests on the same destination redraw as [ticket 10](10-performance-profiling.md), +[ticket 11](11-cli-ux-observability.md) and [ticket 12](12-retire-r-workspace.md): +the user confirmed rollout readiness covers more than merge/verify/promote, +and dependency hygiene is part of that bar. + +Needs [Merge product-pipeline (PR #6) into migration](03-merge-product-pipeline.md) +closed (it is) so there's one `pyproject.toml`/`uv.lock` to audit, not two +branches' worth. + +## Question + +Audit every dependency in `pyproject.toml`/`uv.lock` (and the `Dockerfile`'s +base image, `python:3.14-slim`) against current upstream versions: what's +outdated, what has known security advisories, what's safe to bump now versus +what needs a deprecation/breaking-change check first. Update what's safe, +record what's deliberately deferred and why (e.g. a major version needing +migration work of its own). + +**Sequencing note**: this should land before [ticket +10](10-performance-profiling.md)'s profiling run, not after — profiling +against a dependency set that's about to change makes the numbers stale +immediately. Ticket 10 has been wired `blocked_by: [3, 13]` to reflect this. + +## Resolution + +**Decision**: ran `uv lock --upgrade` to move every dependency (`pyproject.toml`'s +constraints are all `>=` lower bounds, so the lock file — not the manifest — +was the thing actually stale) to its current latest resolvable version, +including three majors: `pandera` 0.26.1 -> 0.32.1, `pytest` 8.4.2 -> 9.1.1, +`typer` 0.19.2 -> 0.27.1 (`rich` came along transitively, 14.2.0 -> 15.0.0). +`ty` (dev-only type checker) moved 0.0.1a23 -> 0.0.69 — a huge alpha jump, +called out separately below since it changed what CI's `ty check src/` gate +actually catches. The `Dockerfile`'s `python:3.14-slim` floating tag was +checked against Docker Hub directly (not just read) and already resolves to +3.14.7-slim as of 2026-08-05 — no pin to move. + +**Because**: `pyproject.toml` uses unbounded `>=` everywhere, so nothing was +capping these versions — the lock file had just never been refreshed since +the constraints were written. `uv tree --outdated` before the bump showed +every dependency 1-3 versions behind latest; `pip-audit` against that +pre-upgrade lock found 19 known vulnerabilities across 8 packages (`click`, +`protobuf`, `requests`, `idna`, `urllib3`, `pyasn1`, `pygments`, +`python-dotenv`) — all fixed by versions already within the existing `>=` +constraints, so there was no reason to defer any of them. + +**Rejected**: pinning `pyproject.toml`'s floors up to match the new lock +(e.g. `pytest>=9.1.1`) was considered and rejected — the existing `>=` +floors already resolve to the versions now locked, so bumping them would +only narrow future resolution without changing today's behavior; the lock +file is the actual source of truth `uv sync --frozen` installs from. +Pinning the Dockerfile's Python image to an exact patch (`3.14.7-slim` +instead of the floating `3.14-slim` tag) was considered for build +reproducibility and rejected as out of this ticket's "outdated versions" +scope — the floating tag is already current, and pinning-for-reproducibility +is a separate, deliberate tradeoff (loses automatic security patches) the +user hasn't asked for. + +**Evidence (executed, not just read)**: +- `uv lock --upgrade` + `uv sync --frozen`, then the full suite: + `uv run pytest -q` -> 488 passed, 0 failed (was 488 passed before the bump + too — no regressions). `uv run ruff check .` and `uv run ruff format + --check .` both clean (the one "unformatted" hit is a markdown code + example in `docs/archive/MIGRATION_GUIDE.md`, not real source). +- `uv run --with pip-audit pip-audit -r `: + 19 known vulnerabilities before the upgrade, 0 after. +- `uv run ty check src/` (the CI-gating command — CI only checks `src/`, not + `tests/`) surfaced 2 real diagnostics the old 0.0.1a23 alpha never caught: + 1. `src/a4d/gcp/storage.py:43` — `blob.name` is typed `str | None` in + `google-cloud-storage`'s stubs; `_download_blob` used it unguarded. + Fixed with an `assert blob.name is not None` (true in practice — this + function is only called on blobs yielded by `list_blobs()`, which + always populate `name`) rather than silencing the checker. + 2. `src/a4d/validate/common.py:95` — `emit_finding`'s `error_code` + parameter was typed `str` with a `# type: ignore[arg-type]` comment + that (mypy syntax) `ty` never honored, masking that all nine call sites + already pass one of the `ErrorCode` `Literal` values the function's own + docstring documents. Retyped the parameter to `ErrorCode` and dropped + the dead ignore comment — this is what the function was always meant to + accept. + Both fixes verified: `uv run ty check src/` -> "All checks passed!" after. +- Product-only coverage gate re-run exactly as CI computes it + (`uv run coverage report --include=... --fail-under=85`) -> 88% total, + unaffected by the dependency bump. +- Checked `typer` 0.27's one documented breaking change (metavar printing in + `--help` output — the same surface that broke CI in + [ticket 4](04-fix-migration-ci.md)) via web search of the release notes; + the full test suite (which includes `tests/test_cli/test_cli.py`'s + `--help` assertions) passing after the bump is direct evidence it isn't + triggered here. + +**Deferred, not fixed (out of this ticket's CI-gated scope)**: `ty` also +surfaced 14 additional diagnostics confined to `tests/` (a read-only-property +assignment in a test, two `list[str]` vs `list[str | None]` invariance +complaints, several `ManifestEntry | None` attribute accesses left +unnarrowed in test assertions) — CI's `ty check src/` never checks `tests/`, +so these are pre-existing test-code type looseness the alpha version was too +weak to see, not a regression. Left as-is: fixing test-only type annotations +is general hygiene, not a dependency-version decision, and doesn't block +anything on this map. Also left as-is: `polars`'s own `empty_as_null` / +`str.to_date()` / `from_arrow` deprecation warnings (visible in the pytest +run) — these are `polars` 1.x's own forward notice of its *upcoming* 2.0 +breaking changes; 2.0 isn't released (`uv tree --outdated` shows `polars` +1.34.0 -> latest 1.43.2, no 2.0), so there is no version to bump toward yet +and no action this ticket's scope covers. + +**Tense**: all claims above describe current, executed behavior on +`migration` HEAD after this session's commit, not a proposed design. diff --git a/docs/wayfinder/tickets/14-product-column-detection-failures.md b/docs/wayfinder/tickets/14-product-column-detection-failures.md new file mode 100644 index 0000000..1d3b49d --- /dev/null +++ b/docs/wayfinder/tickets/14-product-column-detection-failures.md @@ -0,0 +1,107 @@ +--- +id: 14 +title: Fix product pipeline's "unable to find column product" failures on 4 real trackers +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-09e +claimed_at: 2026-08-09 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 10 +--- + +## Premise + +Rests on [Profile the combined pipeline's performance against the R baseline +before promoting to dev](10-performance-profiling.md), closed: while running +`process-product` against the full 177-tracker set on the USB drive +(`a4dphase2_upload`) to profile performance, 4 trackers failed outright with +`unable to find column "product"; valid columns: ["index"]` — +`2020_Jayavarman VII Hospital A4D Tracker.xlsx`, `2018_Kantha Bopha Hospital +A4D Tracker.xlsx`, `2019_Kantha Bopha Hospital A4D Tracker.xlsx`, +`2020_Kantha Bopha Hospital A4D Tracker.xlsx`. This is unrelated to ticket +10's performance fix (reproduces identically before and after it) and was not +previously documented anywhere on this map. Not diagnosed further in ticket +10's session, which stayed scoped to performance — the error originates +around `find_product_section` / `extract/product.py`, not investigated +deeper. + +Also rests on [Is the product pipeline (and patient's own claimed +completeness) actually complete and sound, audited against R's product logic +and patient's structure?](07-pipeline-completeness-audit.md), closed: that +audit judged R-logic coverage "essentially complete" on product without +surfacing this — these 4 files are a fresh finding from actually running +against the full real dataset, not something the completeness audit's +code-reading caught. + +## Question + +Why do these 4 trackers fail product extraction with "valid columns: +[\"index\"]" (this looks like `find_product_section` failing to locate the +product/stock section at all, falling back to an empty/index-only frame) — +is it a genuine structural difference in these files' layout (older/different +tracker format for these years), a synonym-mapping gap, or a real regression? +Decide: fix the extraction to handle whatever these files' actual layout is, +or confirm they're legitimately out of scope (e.g. pre-product-tracking +years) and should fail loudly rather than silently, with that decision +recorded either way. Bears on the destination's "every Python/R difference +documented and explicitly decided" bar and on ticket 6's promotion readiness. + +## Resolution + +**Decision:** confirmed legitimate — 2018-2020 KBH and 2020 JVM trackers +predate product/stock tracking entirely; not a synonym gap, not a +regression. Fixed a real bug in the cleaning stage that was turning that +legitimate absence into a hard tracker failure: `clean_product_data` now +returns an empty, schema-conformant (0 rows, 20 columns) DataFrame when +extraction hands it a fully columnless raw frame, instead of running the +full step sequence and crashing on a missing `"product"` column. + +**Because:** direct inspection of the 4 files' sheets (`openpyxl`, on the +real trackers from the USB drive's `a4dphase2_upload`) found the literal +string "product" nowhere in any cell of any month sheet in the 2018/2019/2020 +KBH trackers or the 2020 JVM tracker, and no `INV`/inventory sheet — that +sheet, and product/stock columns generally, first appear in KBH's 2021 +tracker (`INV` sheet with `Product`, `Balance`, `Total Units Received` +headers). `find_product_section` (`src/a4d/extract/product.py`) correctly +returns a fully empty `pl.DataFrame()` (0 rows, 0 cols) for these — logged as +a WARNING per sheet plus one "Empty product data" warning per tracker, not a +crash. The crash traced to `clean_product_data` +(`src/a4d/clean/product.py`): step 2.5 (`_add_row_index`) adds an `"index"` +column to the columnless frame, and the later schema-conformance step +(`apply_schema`, `src/a4d/clean/schema_product.py`) calls +`df.with_columns(pl.lit(None, dtype=...))` on that 0-row/1-col-then-more +frame — with zero *original* columns to anchor the frame's height, polars' +literal broadcast invents a 1-row output instead of preserving 0 rows, and +whichever step first indexes `"product"` (`_fill_product_names_and_sort` +class of helpers, unlike `_split_multi_product_cells` which already guards +with `if "product" not in df.columns`) throws +`ColumnNotFoundError: unable to find column "product"; valid columns: +["index"]` — reproduced exactly, matching the ticket's original report +verbatim. Confirmed end-to-end via `process_tracker_product` on all 4 real +files before and after: all 4 now return `success=True` with a +`(0, 20)`-shaped cleaned parquet, matching each file's genuine absence of +product data; `read_cleaned_product_data`'s existing diagonal-concat table +stage already tolerates a 0-row schema-conformant frame (no changes needed +there). Full suite (490 tests, one new regression test added), ruff, and +`ty check src/` all pass. + +**Rejected:** "fail loudly" (raise/mark the tracker as failed for having no +product data) was the ticket's other named option — rejected because these +are legitimate, expected years for these two clinics, not a data-quality +problem; treating them as failures would misrepresent real "the pipeline +broke" failures once this map starts tracking tracker success/failure counts +as a rollout-readiness signal (ticket 11's territory). No extraction-side fix +was needed or made — `find_product_section` and `read_all_product_sheets` +already do the right thing; only the cleaning stage's implicit assumption +that a raw frame always has a `"product"` column was wrong. + +**Evidence:** executed — reproduced the original crash and the fix against +the real files on the USB drive (not synthetic fixtures), ran the full test +suite, ruff, and `ty check` after the change. + +**Tense:** all claims above describe current, verified behaviour (post-fix), +not a proposed design. + diff --git a/docs/wayfinder/tickets/15-build-and-run-comparison-script.md b/docs/wayfinder/tickets/15-build-and-run-comparison-script.md new file mode 100644 index 0000000..c157db5 --- /dev/null +++ b/docs/wayfinder/tickets/15-build-and-run-comparison-script.md @@ -0,0 +1,206 @@ +--- +id: 15 +title: Build and run the R/Python output comparison script, then triage every flagged difference +labels: [wayfinder:task] +status: closed +blocked_by: [2] +assignee: session-2026-08-10 +claimed_at: 2026-08-10 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 2 +--- + +## Premise + +Rests on [Retire the PDF/notebook analysis docs for an automated, +script-based report](02-documentation-strategy.md), closed: the comparison +script's design is decided — decoupled from pipeline execution, diffs two +existing output directories (Python's and the already-frozen +`/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_r/`, no R re-run), four +layered checks (shape, totals, columns, cell-by-cell), an extensible +cause-classifier registry seeded with the four causes already known from +`docs/archive/Product pipeline parity presentation.pdf` for +`Product_entry_date`, HTML report output, lives at `scripts/` + +`just compare-outputs` (not `a4d.cli`). + +That ticket's session was explicit that only the design was in scope: the +bulk of the real work is investigative — most divergence causes for columns +other than `Product_entry_date` aren't known ahead of time and can only be +identified by inspecting actual flagged rows against the source trackers +(`a4dphase2_upload`) — and it deferred that work here rather than attempting +it inline. + +## Question + +Build the comparison script per ticket 2's decided design; run it against a +current Python pipeline output (fresh run against `a4dphase2_upload`) and the +frozen `output_r/`; then go through the flagged differences and, for each +column, determine why Python and R disagree — labeling causes into the +classifier registry as they're identified (adding new named classifiers +beyond the four seeded ones as needed), falling back to the original source +Excel trackers as the arbiter of which output is actually correct wherever a +divergence isn't already an understood, expected pattern. + +**Validation requirement carried over from ticket 2:** the script's per-column +and per-cause mismatch counts should reproduce (or the session should be able +to explain any deltas from) the parity-presentation PDF's numbers — 189 +trackers / 61,077 rows / 20 columns; `Product_entry_date`: 559, +`Product_balance`: 480, `Product_category`: 214, `Product_sheet_name`: 201, +`Product_received_from`: 154, `Product_units_received`: 9; entry-date causes +typo-rescue: 408, CE-typo: 71, sentinel-null: 66, off-by-one-day: 7. That +reproduction is what proves the new script is a faithful, trustworthy +replacement for the retired PDF/notebook documentation, and is a concrete +signal for the map's destination requirement that "every Python/R difference +[is] documented and explicitly decided." + +This ticket is large and investigative (189 trackers' worth of triage) — if +it doesn't converge in one session, split the remaining per-column +investigation into further tickets rather than leaving it open-ended. + +## Resolution + +**Decision.** Built and ran the comparison script per ticket 2's design; +discovered a real, specific blocker before any cause triage could start, and +split the remaining work into [ticket +17](17-fix-product-row-alignment-and-triage.md) rather than attempting it +here. + +Built: `src/a4d/migration/compare.py` (four pure, unit-tested layers — +`compare_shape`, `compare_totals`, `compare_columns`, `compare_cells` — plus +`classify`/`PRODUCT_ENTRY_DATE_CLASSIFIERS` seeded from the PDF's four named +causes, `compare_directory` for per-file orchestration, `render_html_report` +for the HTML output), 23 tests, TDD'd seam-by-seam, ruff/ty clean. Thin CLI +wrapper `scripts/compare_outputs.py` + `just compare-outputs` recipe, per +ticket 2's decision to keep this out of `a4d.cli`. + +Ran a fresh Python pipeline pass against `a4dphase2_upload` (both arms, +`--data-root`/`--output` pointed at the USB drive), writing to +`output_python/` next to the frozen `output_r/` (replacing the stale +2025-11-15 copy that predated this map's O(n^2) and product +column-detection fixes) — patient 174/174 succeeded, product 174/174 +succeeded. Ran `compare_outputs.py` against the two directories. + +**Patient arm's row-alignment key is sound**: `(patient_id, sheet_name)` +has duplicate keys in only 5 of 172 matched files (single-digit duplicate +counts each), confirmed by direct inspection — not a comparison artifact. +172 files matched, 2 only in Python (`2025_06_*` trackers postdating the +frozen R baseline). + +**Product arm's row-alignment key is broken**: `(clinic_id, product, +product_sheet_name, product_entry_date)` collapses onto as few as 154 +distinct keys for 1,194 rows in one file alone (up to 35 rows sharing one +key), because `product_entry_date` is null on many rows (balance-only or +malformed-date rows) — every duplicate group produces a join fan-out, +inflating cell-mismatch counts by orders of magnitude +(`product_balance`: 62,959 reported vs. 480 in the parity-presentation PDF) +and, worse, silently excludes `product_entry_date` itself from +classification since it's a join key — the PDF's single largest divergence +column (559 mismatches) never gets diffed at all. Confirmed by direct +`group_by` inspection against the real drive data, not inferred. 155 files +matched (0 only in R), 19 only in Python — all from 2017-2022, consistent +with [ticket 14](14-product-column-detection-failures.md)'s finding that +Python now emits schema-conformant 0-row output for pre-product-tracking +years where R produced no file at all. + +**Also unreconciled, not yet investigated:** the PDF's 189-tracker / +61,077-row product baseline doesn't match this session's counts (155 R +product files matched, R total product rows not yet summed) — could be a +different snapshot of the drive, a different tracker set, or a stale PDF +number; ticket 17 inherits this as an open question alongside the +row-alignment fix. + +**Because.** The row-alignment defect has to be fixed before any per-column +mismatch count or cause classification can be trusted for product — the +current numbers are join-fan-out noise, not real divergence signal. Rather +than design a new key blind, the right next step is inspecting real flagged +rows once alignment is fixed (per the ticket's own scoping: causes aren't +knowable ahead of time), so this session stops at the diagnosis rather than +guessing a fix. + +**Rejected.** +- Attempting a product row-alignment fix inline — rejected: the fix likely + needs the `index` row-order column `clean/product.py` already assigns per + `(clinic_id, product_sheet_name)` group (step 2.5, not currently in the + output schema) or some other ordinal scheme, which itself needs + investigation against real duplicate rows to confirm R and Python assign + the same order — that's real design work, not a continuation of what this + ticket built. +- Trusting the reported product numbers and triaging them anyway — + rejected: the numbers are provably inflated by join fan-out (confirmed via + direct duplicate-key inspection), so any cause labeled against them would + be labeling noise. +- Re-scoping this ticket to keep it open rather than closing and spawning — + rejected: the ticket's own text pre-authorized splitting if triage didn't + converge in one session; the infrastructure (script, tests, fresh run) is + a complete, real deliverable independent of the remaining triage. + +**Evidence.** *Executed*: built and ran the script against real drive data; +ran the fresh Python pipeline pass (both arms, real trackers); confirmed the +patient/product key-duplication counts via direct `group_by` queries against +the real parquet files, not reasoning. *Judgement*: the diagnosis that +join fan-out (not real divergence) explains the inflated product numbers is +inference from the duplicate-key counts, not a re-run with a corrected key — +ticket 17 should re-verify once a real fix lands. + +**Tense.** All claims describe current repository/drive state as of this +session (executed) or a diagnosis of why the current numbers can't be +trusted (not yet fixed). + +## Addendum (same session, after closure) + +The user kept iterating on the tool directly after this ticket closed and +[ticket 17](17-fix-product-row-alignment-and-triage.md) spawned — same +session, same deliverable, substantial enough to record here rather than +leave implicit in conversation history: + +- **Dropped `render_html_report`/HTML output entirely.** The user was + explicit: triage means loading results as a dataframe, filtering, sorting, + adding columns — Excel, not a static HTML page. Replaced with + `build_summary_rows` (same per-column/per-cause aggregate data as plain + dict rows) merged into the same per-stage Excel workbook the flagged-row + detail already went into. Each stage now produces exactly one `.xlsx`. +- **Added `compare_id_overlap`/`IdOverlapResult`** (do the same identities — + `patient_id` for patient, product name for product — appear on both sides + at all, independent of the row-alignment key) and + **`compare_categorical_overlap`/`CategoricalOverlap`** (same idea per + categorical/label column). Both proved immediately useful precisely + because they don't depend on the broken product row-alignment key: they + confirmed product *names* match 100% across R and Python even where + cell-level comparison is meaningless noise. +- **Added `compare_row_key_overlap`/`RowKeyOverlap`** — the piece that was + actually missing. Counts rows whose *full* row-alignment key (not just a + single identity column) found no partner on the other side at all, or + fanned out via a repeated key. This turned out to be essential: raw + product files were showing 0 cell mismatches, which read as "clean" but + actually meant the join matched zero rows for that file (e.g. Mahosot: + 354 rows each side, 0 joined; live numbers confirmed near-100% row-key + divergence — 560/560 unmatched — right where cell divergence read 0). + Without this check, "0 cell mismatches" was indistinguishable from "the + key found nothing to compare," which is exactly backwards from what a + triage tool should say. +- **Compared raw pipeline output (`patient_data_raw`/`product_data_raw`) in + addition to cleaned**, one report per stage, so a divergence can be + localized to extraction vs. cleaning. Discovered along the way that raw + extraction isn't schema-normalized like cleaned output (`apply_schema` + only runs at the cleaning stage) — column presence varies file by file, + which required making `compare_totals` and `compare_categorical_overlap` + skip a column missing from either side instead of crashing. +- **Naming pass**: every count-based check renamed to a consistent "X + divergence" scheme (ID, Column, Categorical, Row-key, Totals, Cell) after + the user pointed out "ID overlap" was backwards — overlap should mean + shared/common, not what's flagged (which is the R-only/Python-only + divergence). Reordered so Row-key divergence sits immediately before Cell + divergence, since both depend on the same row-alignment key. +- **Added `--only-mismatches`** (skip files where every measure is clean) + and grouped the console/Excel output by tracker year, descending. + +Net effect for [ticket 17](17-fix-product-row-alignment-and-triage.md): the +tool it inherits is meaningfully more capable than what this ticket +originally closed with. Row-key divergence in particular gives ticket 17 a +direct, per-file measurement of how badly the current key is failing +(matched/r_unmatched/py_unmatched), which should make validating a proposed +fix far more precise than re-eyeballing cell-mismatch counts. + +Commits: `296fa86`, `3f5ebde`, `512c6a1`, `ac4090e`, `9fd16bb`, `a96ffc4`. diff --git a/docs/wayfinder/tickets/16-log-analyzer-drill-down.md b/docs/wayfinder/tickets/16-log-analyzer-drill-down.md new file mode 100644 index 0000000..844d314 --- /dev/null +++ b/docs/wayfinder/tickets/16-log-analyzer-drill-down.md @@ -0,0 +1,63 @@ +--- +id: 16 +title: Build a drill-down log analyzer for admins to inspect a specific tracker file's errors/logs +labels: [wayfinder:grilling] +status: open +blocked_by: [] +assignee: null +claimed_at: null +resolution: null +evidence: null +closed_by: null +spawned_by: 11 +--- + +## Premise + +Rests on [Decide what CLI/TUI UX and error-log observability improvements +admins/developers need before rollout](11-cli-ux-observability.md), closed: +the user distinguished two separate concerns under that ticket's "error-log +observability" question — a high-level per-run summary (now built into +`run-pipeline`'s combined summary: both-arms-ok / patient-only-failed / +product-only-failed / lost-entirely counts, plus a merged per-file error +count) versus a drill-down view into one specific file's full detail. The +summary is done; this ticket is explicitly the drill-down half, deferred +because it's a separate, larger deliverable rather than an extension of the +existing summary tables. + +Also rests on [Retire R from the workspace once the pipeline is fully +verified Python-only](12-retire-r-workspace.md)'s premise material: prior +art is `tools/LogViewerA4D`, an R Shiny dashboard (deleted once ticket 12 +closes) that read local tab-separated `.log` files and offered a per-tracker +log table + Sankey diagram, a cross-tracker regex-filterable overview, and a +reference-data validation tab (missing/duplicate `clinic_id`). Read before +building, not to be ported as-is — its design assumes local log files; the +current pipeline logs via loguru into the `logs` BigQuery table (1M+ rows) +and `table_errors.parquet` instead, so any Python equivalent needs a +different shape. + +## Question + +Design and build a tool (CLI subcommand, small TUI, or lightweight +dashboard — open which) that lets an admin/developer pick one tracker file +from a run and see everything relevant to it: every log line (from +`table_logs`), every data-quality error row (from `table_errors`: column, +original value, error code), and the exception detail for an outright +processing failure — without hand-querying BigQuery or grepping JSON. + +Concrete use case to design against: the combined run summary (ticket 11) +tells an admin *that* `2024_Clinic_B.xlsx` had 4 errors or failed outright; +this tool answers *what exactly went wrong*, row by row. + +Questions to resolve: +- Where does it live — a new `a4d.cli` subcommand reading local + parquet/BigQuery, a separate small script/TUI, or something else? +- Does it need BigQuery access (for a deployed run) or only local + output/tables (for a local run), or both? +- Which of `LogViewerA4D`'s three views (per-tracker detail, cross-tracker + overview, reference-data validation) are worth carrying forward given the + new data shape, and which don't translate? +- Scope: is this genuinely needed before `migration` promotes to `dev` + (this map's destination), or is "good enough to operate day to day" + already met by the combined run summary alone, making this a nice-to-have + outside the promotion path? diff --git a/docs/wayfinder/tickets/17-fix-product-row-alignment-and-triage.md b/docs/wayfinder/tickets/17-fix-product-row-alignment-and-triage.md new file mode 100644 index 0000000..45be6c0 --- /dev/null +++ b/docs/wayfinder/tickets/17-fix-product-row-alignment-and-triage.md @@ -0,0 +1,225 @@ +--- +id: 17 +title: Fix the product comparison's row-alignment key, then triage every flagged R/Python difference +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-11 +claimed_at: 2026-08-11 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 15 +--- + +## Premise + +Rests on [Build and run the R/Python output comparison script, then triage +every flagged difference](15-build-and-run-comparison-script.md), closed: +`scripts/compare_outputs.py` + `src/a4d/migration/compare.py` are built, +tested (23 unit tests), and were run against a fresh Python pipeline pass +(both arms) and the frozen `output_r/` baseline, both on the USB drive. + +That session found the patient arm's row-alignment key — +`(patient_id, sheet_name)` — is sound (duplicate keys in only 5 of 172 +matched files, each a handful of rows) and can be triaged as-is. It found +the **product arm's key is broken**: `(clinic_id, product, +product_sheet_name, product_entry_date)` collapses onto far fewer distinct +keys than rows exist (one file alone: 154 distinct keys for 1,194 rows, up +to 35 rows sharing a key) because `product_entry_date` is null on many rows +(balance-only or malformed-date rows). Every duplicate group produces a join +fan-out that inflates reported cell-mismatch counts by orders of magnitude +and — critically — excludes `product_entry_date` itself from +classification, since it's one of the join keys. The PDF's single largest +divergence column (`Product_entry_date`, 559 mismatches) currently never +gets diffed at all. This was confirmed by direct `group_by` inspection +against the real drive data, not inferred. + +That session also left unreconciled: the parity-presentation PDF cites 189 +trackers / 61,077 rows for product; this session's fresh run matched 155 R +product files (0 only-in-R, 19 only-in-Python — explained by [ticket +14](14-product-column-detection-failures.md)'s 0-row-output fix for +pre-product-tracking years). Whether that gap is a stale PDF number, a +different drive snapshot, or something else is not yet investigated. + +**Update from the same session, after ticket 15 closed** (see its +[addendum](15-build-and-run-comparison-script.md#addendum-same-session-after-closure)): +the comparison tool this ticket inherits is more capable than what ticket 15 +closed with. Notably, `compare_row_key_overlap`/`RowKeyOverlap` now measures, +per file, how many rows failed to find *any* partner via the full +row-alignment key (`matched`/`r_unmatched`/`py_unmatched`) — this is what +confirmed the diagnosis above isn't just inference: raw product files show +near-100% row-key divergence (e.g. one file: 560/560 rows unmatched) +alongside near-0% cell divergence, proving the 0s are "nothing was paired to +compare," not agreement. `compare_id_overlap` and `compare_categorical_overlap` +also confirmed product *names* match 100% between R and Python even in files +where the row key is fully broken — the identity data itself isn't the +problem, only the row-pairing mechanism is. Use `RowKeyOverlap.matched` +against a candidate replacement key to validate it directly, rather than +re-deriving fan-out evidence from raw duplicate-key counts by hand. + +## Question + +Design and implement a row-alignment key (or an alignment strategy that +isn't a simple equi-join key — e.g. an ordinal position within a group, +possibly reusing the `index` column `clean/product.py` already assigns per +`(clinic_id, product_sheet_name)` at step 2.5 but doesn't currently carry +into the output schema) that uniquely identifies a product row well enough +for `compare_cells` to align R and Python rows without fan-out — including +rows where `product_entry_date` is null. Confirm R and Python assign the +same ordering for whatever key is chosen (they may not, if extraction order +differs) before trusting any diff built on it. + +Once alignment is fixed, re-run `compare_outputs.py` and go through the +flagged differences for both arms, column by column, labeling causes into +the classifier registry as they're identified — falling back to the +original source Excel trackers (`a4dphase2_upload`) as the arbiter wherever +a divergence isn't already an understood, expected pattern. Carries forward +ticket 15's validation requirement: the script's per-column and per-cause +counts should reproduce (or explain deltas from) the parity-presentation +PDF's numbers for `Product_entry_date` (559: typo-rescue 408, CE-typo 71, +sentinel-null 66, off-by-one-day 7), `Product_balance` (480), +`Product_category` (214), `Product_sheet_name` (201), +`Product_received_from` (154), `Product_units_received` (9) — including +resolving the 189-vs-155-tracker discrepancy noted above. Patient-arm +triage (already-sound alignment) is in scope here too, since it was run but +not triaged in ticket 15. + +This ticket is still large and investigative — if it doesn't converge in +one session, split the remaining per-column investigation further rather +than leaving it open-ended. + +## Resolution + +**Decision.** Fixed the row-alignment key for its product-arm half; the +"triage every flagged difference for both arms" half did not converge in +this session and is split into [ticket +18](18-triage-comparison-flagged-differences.md), per this ticket's own +pre-authorization. + +**Design chosen: ordinal position within `(clinic_id, product_sheet_name)`, +not a stored column.** Investigated reusing `clean/product.py`'s `index` +helper column (step 2.5) first, and found it wouldn't work as-is even if it +were kept in the output schema: R's `reading_product_data_step2` +(`r-archive/R/read_product_data.R:599`) assigns its own `index` *inside a +per-`sheet_month` loop* (resets to 1 for every sheet), while Python's +`_add_row_index` (`clean/product.py`) assigns a single global +`with_row_index` across the *whole* file — a real, previously undocumented +divergence in what "index" means between the two pipelines. Rather than fix +that and thread a new column through both pipelines (R's frozen baseline +output can't be re-run to pick it up anyway — ticket 2 decided no R re-run, +ever), the key is computed *at comparison time*, directly from each output +file's existing row order: `add_row_ordinal()` +(`src/a4d/migration/compare.py`) partitions by `(clinic_id, +product_sheet_name)` — normalizing both to stripped strings for the key +only, so a whitespace divergence in the raw column still surfaces as an +ordinary cell mismatch rather than silently breaking the group — and takes +each row's 0-based position within that partition as the alignment key. +This works because extraction concatenates sheets in file order (contiguous +per-sheet blocks) and every row-order-changing step in `clean/product.py` +(explode, filter, the product-name sort in step 2.7) preserves *relative* +order within whatever grouping precedes it — confirmed empirically, not +just reasoned: this key achieves 46,314/47,644 (97.2%) row-key match on the +real cleaned-product data. + +**Validated directly against the real R/Python output pair on the USB +drive** (`RowKeyOverlap.matched`, not assumption): cleaned-product row-key +divergence dropped from near-100% (ticket 15's diagnosis) to 2.8% (1,330 +rows), and every one of those 1,330 is isolated to exactly one clinic across +its three tracker years (`2023/2024/2025_06_North Okkalapa General Hospital` +— 0 matched, 100% unmatched on each) rather than being spread thin across +many files. Inspecting that clinic directly found why: **R's frozen output +has `clinic_id = "NGH"` for all three files; Python has `clinic_id = +"NOH"`.** Python's value matches the tracker's actual folder name on the +drive (`a4dphase2_upload/Myanmar/NOH/`), which is the documented +`clinic_id` derivation rule (`docs/CLAUDE.md`) — R's is a typo, not a +Python bug. Also found and resolved along the way: `2019_CDA A4D Tracker` +had 74/402 rows unmatched under an earlier, less-normalized version of the +key, traced to R's `product_sheet_name` carrying un-trimmed trailing +whitespace (`"May19 "` vs Python's `"May19"`) for two sheets — folding +whitespace-normalization into the join key itself (not the compared value) +fixed this without hiding the divergence, since `product_sheet_name` is +still diffed as an ordinary column and its mismatch count (**201**) now +reproduces the parity-presentation PDF's number for `Product_sheet_name` +exactly — the strongest available confirmation that the new key is sound, +not just less-broken. + +**Also re-ran the same validation on raw product** (`product_data_raw`): +109,130/112,381 (97.1%) matched, with the remaining mismatches explained by +the same North Okkalapa `clinic_id` typo (rows only, at raw stage this +clinic's rows aren't even grouped that far apart) plus a handful of +single-row diffs on 5 other files not yet individually inspected — carried +into ticket 18. + +**Also surfaced, not yet reconciled:** re-running the row-count check found +the parity-presentation PDF's 189-tracker/61,077-row product baseline still +doesn't match — the currently-frozen `output_r/` on the drive has 155 +product-cleaned files totalling 47,644 rows, identical to ticket 15's +count. This rules out "the new key was undercounting files" as the +explanation (the file/row totals are unchanged by this ticket's fix); the +gap is either a stale PDF number or a genuinely different drive snapshot at +the time the PDF was built. Still unresolved — carried into ticket 18. + +**Because.** The row-alignment key was the one blocking design question +this ticket exists to answer, and the fix needed exactly the two things the +question asked for: an alignment strategy that isn't a simple equi-join key, +and direct confirmation (not assumed) that R and Python assign the same +ordering. Both are done and validated against real data. The remaining +"triage every flagged difference, both arms" work is unrelated +investigative breadth (each column's cause is its own small investigation) +that the ticket's own text pre-authorized splitting off rather than +attempting to converge in the same session as the design work. + +**Rejected.** +- Restoring `clean/product.py`'s `index` column into the output schema and + using it as the stored key — rejected: R's frozen baseline can't be + re-run to pick up a new column (ticket 2's decision), so a + comparison-time-only key was required regardless; and R's own `index` + resets per-sheet while Python's doesn't, so simply un-dropping Python's + current column would still not match R's semantics without first fixing + that divergence in the pipeline itself — a change to production cleaning + logic this ticket's premise never asked for. +- Including `product` (the product name) in the alignment key alongside the + ordinal — rejected: the ordinal alone is already unique per row within + its group by construction; adding `product` back would only reintroduce + a dependency on `_fill_product_names_and_sort`'s forward-fill matching + between R and Python exactly, which isn't necessary once ordinal position + already disambiguates. +- Normalizing whitespace on the actual `product_sheet_name` column values + (not just the join key) so the 2019_CDA file's rows all matched cleanly + from the start — rejected: that would have hidden a real R-side data + defect from `compare_cells`, exactly the kind of divergence this tool + exists to surface; classifying it as an ordinary `product_sheet_name` + cell mismatch (which is what happened) is the correct outcome. +- Attempting the full triage pass in this session anyway — rejected: initial + per-column counts (`product_category`: 13,638 vs the PDF's 214; + `product_entry_date`: 10,424 vs 559; `product_balance`: 2,343 vs 480) are + far larger than the PDF's numbers even after the key fix, and a first look + found `product_category` mismatches are systematically `r_value=None` + where Python has a real category — that's a distinct, uninvestigated + question (does R's cleaned product schema genuinely never populate + `product_category` for some file/year range, or is this itself another + alignment artifact) that deserves its own focused session rather than a + guess recorded here. + +**Evidence.** *Executed*: ran `just compare-outputs` against the real, +already-existing `output_r`/`output_python` directories on the USB drive +(no pipeline re-run needed — Python's fresh pass from ticket 15 was still +current); confirmed `RowKeyOverlap.matched`/`r_unmatched`/`py_unmatched` +directly from the Excel report's `row_key_overlap` sheet; confirmed the +North Okkalapa `clinic_id` divergence and the 2019_CDA whitespace divergence +by direct inspection of the parquet files and the source tracker folder +name; confirmed the 155-file/47,644-row R product-cleaned count directly. +Added 5 unit tests for `add_row_ordinal` (TDD), full suite green (545 +tests), ruff and `ty check` clean. *Judgement*: none — every claim above is +a direct measurement against the real R/Python output pair or the R source +(`read_product_data.R`), not inference. + +**Tense.** All claims describe current repository/drive state as of this +session (executed): the key fix is implemented and merged into +`scripts/compare_outputs.py`/`src/a4d/migration/compare.py`, not a proposal. +The `product_category`/`product_entry_date` inflation and the +189-vs-155-tracker gap are diagnoses of what's not yet understood, not +conclusions about their cause. + +Commit: `bcfc32e`. diff --git a/docs/wayfinder/tickets/18-triage-comparison-flagged-differences.md b/docs/wayfinder/tickets/18-triage-comparison-flagged-differences.md new file mode 100644 index 0000000..7c5d660 --- /dev/null +++ b/docs/wayfinder/tickets/18-triage-comparison-flagged-differences.md @@ -0,0 +1,279 @@ +--- +id: 18 +title: Triage every flagged R/Python difference for both arms, and resolve the 189-vs-155-tracker discrepancy +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-11 +claimed_at: 2026-08-11 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 17 +--- + +## Premise + +Rests on [Fix the product comparison's row-alignment key, then triage every +flagged R/Python difference](17-fix-product-row-alignment-and-triage.md), +closed: the product arm's row-alignment key is fixed (ordinal position +within `(clinic_id, product_sheet_name)`, computed at comparison time in +`add_row_ordinal()`, `src/a4d/migration/compare.py`) and validated directly +against the real R/Python output pair — 97.2% row-key match on cleaned +product (up from near-0%), with the `product_sheet_name` mismatch count +(201) now reproducing the parity-presentation PDF's number exactly. The +patient arm's key (`patient_id`, `sheet_name`) was already confirmed sound +in [ticket 15](15-build-and-run-comparison-script.md). + +That session ran a full comparison with the fixed key and found the +per-column mismatch counts are still far larger than the parity-presentation +PDF's numbers even with alignment fixed: `product_category` 13,638 vs the +PDF's 214, `product_entry_date` 10,424 vs 559, `product_balance` 2,343 vs +480 (all from the `product_data_cleaned` stage's `per_column` sheet, +generated by `just compare-outputs` against +`/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_r` and `output_python`). A +first look at `product_category` found the mismatches are systematically +`r_value=None` where Python has a real category value — not yet understood +whether that's a genuine R behavior (never populates `product_category` for +some file/year range) or a remaining artifact. This ticket inherits that as +an open question, not a diagnosis. + +Also inherited, unreconciled since ticket 15: the parity-presentation PDF +cites 189 trackers / 61,077 rows for product; the currently-frozen +`output_r/` on the drive has 155 product-cleaned files totalling 47,644 +rows — confirmed unchanged by ticket 17's alignment fix, so the gap isn't +an artifact of the old broken key. Whether this is a stale PDF number or a +genuinely different drive snapshot at the time the PDF was built is not yet +investigated. + +## Question + +Go through the flagged differences from `just compare-outputs` +(`r_dir py_dir output_dir --only-mismatches`, ticket 19: writes one +timestamped run folder under `output_dir`, four Excel reports plus history +snapshots inside it — re-run to get fresh reports, since the tool changed +after the reports already on the USB drive were generated) for both arms, +column by column, and determine why R and Python disagree — +labeling causes into the classifier registry +(`src/a4d/migration/compare.py`'s `classify`/`*_CLASSIFIERS`) as they're +identified, falling back to the original source Excel trackers +(`a4dphase2_upload` on the test-data drive) as the arbiter wherever a +divergence isn't already an understood, expected pattern (per the map's +standing preference: the migration is not 1:1 R-parity, R can be wrong). + +Start with `product_category` (currently the single largest count, and +already shows a suspicious pattern — R null where Python has a value) and +`product_entry_date` (the PDF's largest known category, now finally visible +after ticket 17's fix) before the smaller columns. Carries forward ticket +15/17's validation requirement: the script's per-column and per-cause counts +should reproduce (or the session should be able to explain any deltas from) +the parity-presentation PDF's numbers — `Product_entry_date` (559: +typo-rescue 408, CE-typo 71, sentinel-null 66, off-by-one-day 7), +`Product_balance` (480), `Product_category` (214), `Product_sheet_name` +(201, already reproduced exactly), `Product_received_from` (154), +`Product_units_received` (9). Resolving the 189-vs-155-tracker discrepancy +is in scope here too, since it bears directly on whether the PDF's +per-column counts are even comparable to this tool's — it may turn out the +PDF's baseline included trackers this session's `output_r/` doesn't have, +in which case some deltas above are expected, not defects. + +Patient-arm triage (raw and cleaned, both stages, alignment already sound) +is in scope here too, since it was run in ticket 15 but never triaged. + +This ticket is large and investigative (up to ~47,644 product rows' worth +of triage across multiple columns, plus the full patient arm) — if it +doesn't converge in one session, split the remaining per-column +investigation into further tickets rather than leaving it open-ended. + +## Resolution + +**Decision:** the 189-vs-155-tracker discrepancy is not reconcilable and is +not a defect to fix. `product_category` and the dominant share of +`product_entry_date` mismatches are root-caused to genuine R limitations +(verified against R's own source, not just the data pattern) and the +comparison tool's classifier registry now labels them accordingly. The +remaining product columns and the entire patient arm did not converge this +session and are split into four new tickets per this ticket's own +pre-authorization. + +**Because:** + +- *189 vs 155 trackers* — checked every product-output snapshot that exists + anywhere on the USB drive: `output_r/` (155 files/47,644 rows), + `output_vm/` (identical: 155/47,644), `output.zip` (155), and even + `a4dphase2_upload/output/` (the current Python run, 174 files/55,466 rows) + and `a4dphase2_upload.zip` (174 source trackers). None reach 189 files or + 61,077 rows — not even the *current* Python run, which processes more + trackers than R ever did. The PDF's 189/61,077 baseline predates every + snapshot now on the drive; there is no reachable tracker set to reconcile + it against, and the currently-frozen `output_r/` (used throughout tickets + 15/17/18) is the only R baseline that still exists. Per-column deltas from + the PDF's numbers are therefore expected to differ in scale from a + different row population, not just from real R/Python divergence — the PDF + numbers are a historical reference point, not a reproducibility target. + +- *`product_category`* (13,638 mismatches, PDF: 214) — 100% of mismatches + are `r_value=None` with Python holding a real value (verified by full + aggregation over every mismatch row, not a sample). Root cause found in + `r-archive/R/read_product_data.R`'s `add_product_categories()` (line + 481-487): it left-joins the raw `product` string against the category + mapping with **no case or whitespace normalization**. Python's + `src/a4d/reference/products.py` (`_load_...`, line ~40-45) explicitly + lowercases and strips both sides before matching. Any casing/whitespace + variant in a tracker's product name (common across 177 real trackers from + 8+ countries) silently misses R's join and leaves the row's category null, + while Python's normalized lookup succeeds — confirmed this is a real + gap in R's logic, not a Python defect, per the map's standing preference + that R can be wrong and the source trackers are the arbiter (the category + itself isn't in the source sheet — it's derived from a reference table by + product name either way, so "arbiter" here means R's own documented left-join + semantics, which openly states missing categories yield NA). Added + `PRODUCT_CATEGORY_CLASSIFIERS` / `_is_r_category_lookup_miss` to + `src/a4d/migration/compare.py` — all 13,638 mismatches now classify as + `r_category_lookup_miss` (verified by re-running `just compare-outputs`). + +- *`product_entry_date`, cleaned stage* (10,424 mismatches, PDF: 559) — + 98.6% are the same `r_value=None`-with-Python-value pattern. The existing + `typo_rescue` classifier (seeded from the PDF) is overbroad: it matches + *any* R-null case, not just genuine source-typo rescues, which inflated it + to 10,211 vs the PDF's 408. Traced two contributing root causes in R's + source: (1) `read_product_data.R` line 608 only forward-fills the + `product` column (`tidyr::fill(c(product), ...)`), never + `product_entry_date` — Python's equivalent step + (`_fill_product_names_and_sort`, `src/a4d/clean/product.py`) is + deliberately R-parity here and fills the same single column, so this + alone doesn't explain the gap; (2) spot-checked a concrete case + (`2018_Mahosot Hospital A4D Tracker`, sheet `Jan18`) directly against the + real source Excel: the raw sheet has a perfectly clean, unambiguous date + (`2018-01-17`) present on every single row, no typo, no ambiguity — yet R + produced null for all of them while Python correctly extracted it. This is + a plain R extraction/parsing gap, unrelated to typos. Because `classify()` + only sees the parsed `(r_value, py_value)` pair, not the raw source cell, + it cannot distinguish a genuine source-typo rescue from this kind of plain + extraction gap — both present identically. Renamed the classifier + `typo_rescue` -> `r_value_missing` to stop implying every case is a typo; + updated `tests/test_migration/test_compare.py` accordingly. The narrower, + structurally-specific classifiers (`sentinel_null`, `ce_typo`, + `off_by_one_day`) are unaffected and still fire correctly. + +- *`product_entry_date`, raw stage* (47,333 of ~47,644 total rows flagged — + essentially the entire dataset) — sampled several raw-stage mismatches + directly: R's raw parquet stores unparsed Excel serial numbers as strings + (e.g. `"42872.0"`), while Python's raw parquet already stores parsed ISO + datetimes (`"2017-05-17 00:00:00"`) for the *same* underlying date. This + is a representation artifact in the comparison tool, not a real R/Python + content divergence — the raw-stage `product_entry_date` report is + currently non-actionable noise. Not fixed this session (needs a + normalization step in `compare_directory`'s raw-stage cell comparison, + parsing both sides before diffing) — spawned as [ticket + 20](20-fix-raw-entry-date-representation.md). + +- *Everything else* — `product_balance` (2,343), `product_received_from` + (330), `product_released_to` (3,834 — not in the PDF's known-cause list at + all), `product_remarks` (66), `product_units_received` (265), `product` + itself (652) at the cleaned stage; every raw-stage product column except + `product_entry_date`; and the entire patient arm (raw + cleaned, never + triaged since ticket 15 ran it) did not get investigated this session. + Split into [ticket 21](21-triage-remaining-product-columns.md) (remaining + product cleaned-stage columns), [ticket + 22](22-triage-product-raw-columns.md) (remaining product raw-stage + columns), and [ticket 23](23-triage-patient-arm.md) (patient arm, both + stages). + +**Rejected:** trying to force-reconcile the PDF's per-column counts exactly +was rejected as a goal — the 189-vs-155 finding means the PDF's row +population is provably different from what's comparable today, so exact +reproduction isn't achievable or meaningful; explaining the *pattern* of +each divergence (as done here) is the right bar, matching `product_sheet_name`'s +already-exact 201 match as the one case where the populations do align. +Further splitting `r_value_missing` into "real typo" vs "extraction gap" +sub-causes was considered and rejected: `classify()` operates on parsed +`(r_value, py_value)` pairs only, with no access to the raw source cell text, +so the two causes are indistinguishable from where it sits; doing so would +require a different tool (joining back to raw extraction), which is a +separate, currently out-of-scope enhancement, not a data question ticket 18 +can resolve. + +**Evidence:** executed throughout — every claim above was checked against +real files (the USB drive's `output_r`/`output_python`/`output_vm`/ +`a4dphase2_upload`, the real source Excel trackers), real R source +(`r-archive/R/read_product_data.R`), real Python source +(`src/a4d/reference/products.py`, `src/a4d/clean/product.py`), and a live +re-run of `just compare-outputs` confirming the new classifiers land as +expected (`r_category_lookup_miss`: 13,638/13,638; `r_value_missing`: +10,211, replacing `typo_rescue`). `tests/test_migration/test_compare.py` +(60 tests, including 2 new ones for the category classifier) plus ruff and +`ty check` on the touched files all pass. + +**Tense:** all claims above describe current, verified behavior of the code +and data as they exist now (2026-08-11) — not proposed or future behavior. + +## Addendum (same day, after closure): R re-run against the current tracker set + +After this ticket closed, production tracker uploads grew from the 177 files +`a4dphase2_upload/` had during this session to 248 (confirmed via +`a4d download trackers` against the live GCS bucket — read-only, no +production pipeline run or GCP spend). The user directed a one-time R +re-run against the current tracker set (R code unchanged — R remains +retired/frozen going forward; this was explicitly framed as the final +capture before R is removed per [ticket 12](12-retire-r-workspace.md), not +a reversal of [ticket 2](02-documentation-strategy.md)'s "no R re-run, +ever" decision). Getting R running required `renv::restore()` (blocked by +an `arrow` C++ build failure unrelated to this project — a macOS/Homebrew +libtool conflict — worked around by installing `arrow` and the package's +other dependencies as CRAN binaries instead of building from the lockfile's +exact pinned source versions) and running from the repo root rather than +`r-archive/` (R's scripts read `reference_data/` via bare relative paths). + +**This materially changes the picture ticket 18 closed with:** + +- Fresh R output: 229 product-cleaned files / 66,020 rows, 243 + patient-cleaned files / 81,859 rows — much closer to the + parity-presentation PDF's 189/61,077 than the old frozen baseline's + 155/47,644. The 189-vs-155 conclusion above (unreconcilable) needs + revisiting; a fresh 189-vs-229 comparison hasn't been done yet. +- The old frozen `output_r/` was renamed to + `output_r_155_frozen_backup_2025-11-14` (not deleted) and the fresh R + output moved into `output_r/` in its place, so all tooling/tickets that + reference `output_r/` now see the fresh data automatically. +- Ticket 17's ordinal row-alignment key still holds — now **100%** row-key + match (66,020/66,020, up from 97.2%; the old NGH/NOH clinic_id typo that + caused the previous gap is gone in the fresh R run). +- `product_category`: mismatches dropped from 13,638 to **866** (93% drop, + now much closer to the PDF's 214). The remaining 866 are still 100% + `r_value=None`-with-Python-present (`r_category_lookup_miss` still fires + on all of them), concentrated in just 12 files (one file, + `2021_Lao Friends Hospital for Children A4D Tracker_DC`, accounts for + 282). So the case/whitespace-sensitive-join root cause is still real for + what remains, but it was **not** the dominant driver of the original + 13,638 — most of that count was an artifact of comparing against the + stale, smaller frozen baseline (misalignment/missing-file noise), not the + join logic itself. This is a correction to this ticket's earlier + resolution, not an addition to it. +- `product_entry_date` (cleaned): 11,727 (up from 10,424, tracking the + larger dataset) — `r_value_missing` still the dominant cause (11,468). + Not re-diagnosed further this session. +- Raw-stage `product_entry_date`: still ~99.9% unclassified (65,710 of + 65,743) — consistent with [ticket 20](20-fix-raw-entry-date-representation.md)'s + representation-artifact finding; not yet fixed. +- Also fixed in passing: `justfile`'s `compare-outputs` recipe had its own + default (`output_dir="compare_output"`) out of sync with + `scripts/compare_outputs.py`'s documented default (`output/comparison`) — + a leftover from ticket 19's rename of `--report-out` to `--output-dir`. + Fixed to `output/comparison` so the two match and run-history tracking + works through the normal `just` invocation. + +**Not done this session:** a full fresh triage of `product_balance`, +`product_received_from`, `product_released_to`, `product_remarks`, +`product_units_received`, `product` (identity), the raw-stage product +columns, or the patient arm — the counts in tickets +[21](21-triage-remaining-product-columns.md), +[22](22-triage-product-raw-columns.md), and +[23](23-triage-patient-arm.md) are updated below to the fresh numbers, but +the actual investigation work in those tickets is unchanged in scope. + +**Evidence:** executed — R was actually run against the current 248-tracker +set (not simulated), `just compare-outputs` was re-run against the fresh +`output_r`/`output_python`, and the row-key-overlap and per-column claims +above were read directly from that run's report +(`output/comparison/2026-08-11T232419Z/`). diff --git a/docs/wayfinder/tickets/19-compare-run-history-deltas.md b/docs/wayfinder/tickets/19-compare-run-history-deltas.md new file mode 100644 index 0000000..9b5c79f --- /dev/null +++ b/docs/wayfinder/tickets/19-compare-run-history-deltas.md @@ -0,0 +1,134 @@ +--- +id: 19 +title: Persist comparison run history and show run-over-run deltas +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-11 +claimed_at: 2026-08-11 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 17 +--- + +## Premise + +Rests on [Fix the product comparison's row-alignment key, then triage every +flagged R/Python difference](17-fix-product-row-alignment-and-triage.md), +closed, and the same session's follow-up conversation about how to work +[Triage every flagged R/Python difference for both arms, and resolve the +189-vs-155-tracker discrepancy](18-triage-comparison-flagged-differences.md): +`build_summary_rows()` (`src/a4d/migration/compare.py`) already computes +`per_column`/`per_cause` mismatch counts per stage on every +`just compare-outputs` run, but nothing persists them — each run only ever +sees the current numbers, with no way to tell whether a fix made in ticket +18 (or any later triage session) actually reduced the count it targeted, or +whether an unrelated pipeline change regressed one that was previously +clean. + +Agreed in the same conversation: ticket 18's triage will classify causes +opportunistically (start `unclassified`, add a named classifier the moment +a real pattern is noticed, drop it if it turns out not to earn its keep) — +this run-over-run comparison is what will make a fix's actual effect +visible directly (count before vs. after), rather than needing classifiers +to prove a fix worked. + +## Question + +Design and implement: after each `compare_outputs.py` run, persist a +compact snapshot of that run's `per_column`/`per_cause` counts (per stage), +and on the next run, load the most recent prior snapshot for that stage and +report a delta (previous vs. current, per column and per cause) alongside +the normal report -- both in the console summary and as an extra sheet in +the stage's Excel workbook. Keep a short history (not just the single most +recent run) so a regression introduced gradually over several runs stays +visible rather than only showing the last hop. + +This is tooling to support ticket 18's triage, not triage itself -- keep it +scoped to the persistence/delta mechanism. + +## Resolution + +**Decision.** Implemented run-over-run history exactly as scoped: +`snapshot_from_summary()`/`compute_deltas()`/`Delta` +(`src/a4d/migration/compare.py`) are pure, unit-tested (11 new tests) -- +`snapshot_from_summary` reduces a `build_summary_rows()` result to a plain +`{"per_column": {...}, "per_cause": {...}}` dict; `compute_deltas` diffs two +such count-dicts and returns only the changed keys (a key dropping to zero +or appearing new from zero both show up, since either side defaults to 0). + +**Refined further in the same session, after the user reviewed the first +cut:** the CLI's output layout changed twice more. First, `--report-out` +(a single filename the tool then derived four sibling files from, e.g. +`compare_report_product_data_cleaned.xlsx`) was renamed to `--output-dir` +(default `output/comparison`) -- naming a file was misleading when the tool +always writes several. Then, on the user's explicit request ("each run +lands in its own folder"), every stage of one `compare_outputs.py` +invocation now writes into a single shared subfolder named by that +invocation's UTC timestamp (`output/comparison//`) rather +than a flat `compare_history//` tree -- each run is now a +self-contained unit on disk (its four Excel reports and four +`snapshot_.json` files together), which is also simpler to browse +and to `rm` wholesale if ever needed. `_load_latest_snapshot` walks +sibling run directories (excluding the one just created), newest first, +and returns the first one that has a snapshot for the requested stage -- +still tolerant of a stage missing from some earlier run. The CLI both +prints a delta table to the console (red for a regression, green for an +improvement) and adds `history_column_deltas`/`history_cause_deltas` +sheets to that stage's Excel workbook. History isn't pruned -- every run's +folder stays on disk, so a multi-run trend is reconstructable later even +though this ticket only wires up latest-vs-current by default. + +Also, prompted by the same review: did a pass over `scripts/` and removed +ten stale one-off debug scripts left over from before this map's work +started (`check_sheets.py`, `compare_r_vs_python.py`, +`export_single_tracker.py`, `profile_extraction.py`, +`profile_extraction_detailed.py`, `reprocess_tracker.py`, +`test_cleaning.py`, `test_extended_trackers.py`, +`test_multiple_trackers.py`, `verify_fixes.py`) -- all hardcoded to a +drive layout (`.../A4D/data/a4dphase2_upload/...`, +`output/patient_data_raw/R/`) that no longer matches the real one, and all +superseded by either the real `tests/` pytest suite, ticket 10's +`pyinstrument`-based profiling, or this ticket's own comparison tool. +Fixed two now-dangling references to the deleted +`compare_r_vs_python.py` (in `verify_production_run.py` and +`src/a4d/gcp/verify.py`, both now pointing at `compare_outputs.py`) and +one stale hardcoded path in the kept `analyze_logs.sql`. + +**Because.** This was a small, self-contained, already-agreed design (settled +in conversation before this ticket was even opened) with no open questions +left to resolve -- straight implementation was the right size for the +ticket, consistent with this map's task-ticket convention of implementing +immediately rather than stopping at a decision. + +**Rejected.** +- Diffing every pair of historical snapshots (not just latest-vs-current) on + every run -- rejected as unnecessary scope: the ticket's own text asked for + history to be *retained* so a multi-run trend stays visible, not for + automatic multi-way analysis; latest-vs-current is what a triage session + actually needs turn to turn, and the retained files support a manual + deeper look later if ever needed. +- Pruning old snapshots (e.g. keep last N) -- rejected: the files are tiny + (a few KB of JSON per stage per run) and this migration-only tool has a + defined end-of-life (R's retirement, ticket 12), so unbounded retention + isn't a real storage concern for its lifetime. + +**Evidence.** *Executed*: ran `just compare-outputs` against the real +R/Python output pair on the USB drive multiple times across both the +flat-history and final per-run-folder layouts, each time confirming the +full lifecycle end to end -- first run correctly reports "no previous run +to compare against," a second run (identical inputs) correctly reports "no +change from the previous run" for all four stages and lands in its own new +timestamped folder, and a synthetic edit to a snapshot file (simulating a +triage fix plus a regression) confirmed the delta table and Excel sheets +render correctly (`product_category` 18,638 -> 13,638 shown as `-5000`; +`product_sheet_name` 0 -> 201 shown as `+201`). Confirmed no remaining +references to any deleted script (`grep` across the repo). Full suite (552 +tests, up from 545), ruff, `ty check` all pass. + +**Tense.** All claims describe current, implemented, merged behavior +(executed) -- not a design proposal. + +Commits: `7c90645` (initial), `5d62865` (`--output-dir` / per-run-folder +refinement and `scripts/` cleanup). diff --git a/docs/wayfinder/tickets/20-fix-raw-entry-date-representation.md b/docs/wayfinder/tickets/20-fix-raw-entry-date-representation.md new file mode 100644 index 0000000..9a1e8f3 --- /dev/null +++ b/docs/wayfinder/tickets/20-fix-raw-entry-date-representation.md @@ -0,0 +1,117 @@ +--- +id: 20 +title: Normalize the raw-stage product_entry_date comparison so it stops flagging near-universal false mismatches +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-12 +claimed_at: 2026-08-12 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 18 +--- + +## Premise + +Rests on [Triage every flagged R/Python difference for both arms, and +resolve the 189-vs-155-tracker discrepancy](18-triage-comparison-flagged-differences.md), +closed: sampling raw-stage `product_entry_date` mismatches directly (e.g. +`2017_Mahosot Hospital A4D Tracker_product_raw.parquet`, sheet `May17`) +found R's raw parquet stores unparsed Excel serial numbers as strings +(`"42872.0"`) while Python's raw parquet already stores parsed ISO +datetimes (`"2017-05-17 00:00:00"`) for the same underlying date. That +representation difference alone accounts for 47,333 of the ~47,644 total +product rows in the raw-stage report — i.e. the raw-stage +`product_entry_date` comparison is currently near-100% noise, not signal. + +**Updated same day, per ticket 18's addendum:** R and Python were both +re-run against the current 248-tracker production set (up from 177) and +`output_r/` now holds that fresh R output (the old 155-file baseline is +preserved at `output_r_155_frozen_backup_2025-11-14`). The pattern holds at +the new scale — raw-stage `product_entry_date` is now 65,710 of 65,743 +unclassified (99.9%), consistent with the same representation-artifact +theory. Use the current `output_r`/`output_python` (not the backup) when +working this ticket. + +## Question + +Fix `compare_directory`'s raw-stage cell comparison (`src/a4d/migration/compare.py`) +so `product_entry_date` values are parsed to a common representation (date) +on both sides before diffing, rather than compared as raw strings/serials. +Decide whether this normalization belongs inside `compare_cells` generically +(a column-type hint) or as a raw-stage-specific pre-processing step in +`scripts/compare_outputs.py`, given the cleaned-stage comparison doesn't +have this problem (both sides are already parsed dates there). Re-run `just +compare-outputs` against the USB drive's `output_r`/`output_python` and +confirm the raw-stage `product_entry_date` count drops to a plausible +signal-only number, then triage what (if anything) remains. + +## Resolution + +**Decision**: added `normalize_date_column()` to `src/a4d/migration/compare.py`, +reusing `a4d.clean.date_parser.parse_date_flexible` (the same flexible parser +the cleaning stage already applies, including its Excel-serial and +typo-rescue handling) to parse both sides' raw `product_entry_date` strings +to `datetime.date` before diffing. Applied only to the `Product (raw)` stage +via a new `date_normalize_cols` field on `scripts/compare_outputs.py`'s +`STAGES` table -- the cleaned stage already has parsed dates on both sides +and doesn't need it, and patient's raw stage has no `product_entry_date` +column at all. + +**Because**: the ticket's own premise, confirmed by direct query against +`output_r`/`output_python` on the USB drive, is that R's raw extraction +stores unparsed source text (an Excel serial string for date-formatted +cells, e.g. `"42872.0"`) while Python's raw extraction already ISO-formats +parsed dates (`"2017-05-17 00:00:00"`) for the same date -- a representation +difference, not a real divergence. Reusing the cleaning stage's own parser +(rather than writing a narrower ad hoc Excel-serial-only parser) also +recovers signal on the non-serial free-text dates R stores as-is (e.g. +`"20-Sept-2021"`, `"14/7/2022"`) instead of only fixing the serial-vs-ISO +case. + +**Rejected**: a generic column-type hint inside `compare_cells` itself -- +rejected because the need is raw-stage-specific and single-column +(`product_entry_date`); the cleaned-stage comparison has no representation +mismatch to fix, so a general mechanism would be unused complexity for a +one-column, one-stage problem. Writing a narrower Excel-serial-only parser +inline in `compare_outputs.py` was also considered and rejected in favor of +reusing `parse_date_flexible`, since it already handles the free-text and +typo cases seen in the real R output and duplicating that logic would drift +from the cleaning stage's parsing rules over time. + +**Evidence** (executed, not just read): confirmed via direct `duckdb` query +against the real `output_r`/`output_python` parquet files on the USB drive +that R stores excel serials as unparsed strings and Python stores parsed ISO +datetimes for the same date (e.g. serial `42872.0` == `1899-12-30 + +42872 days` == `2017-05-17`, matching Python's raw value exactly). Added 6 +unit tests for `normalize_date_column` (excel serial with/without decimal, +ISO datetime, null passthrough, unparseable-text fallback to the sentinel +date, no-op when the column is absent) -- all pass, along with the full +existing suite (559 passed, 1 skipped), ruff, and `ty check src/`. Re-ran +`just compare-outputs` (via `scripts/compare_outputs.py` directly) against +the current `output_r`/`output_python` on the USB drive: raw-stage +`product_entry_date` mismatches dropped from 65,743 to 91 (99.86% was the +representation artifact), landing in the same order of magnitude as the +other raw product columns (70-720). Triaged the remaining 91: 46 already +land in the existing seeded classifiers (`ce_typo` 25, `off_by_one_day` 9, +`sentinel_null` 6, `r_value_missing` 1); the other 50 are `unclassified`. +Spot-checked a sample rather than all 50 (per the map's destination +requiring every difference *decided*, not every difference independently +re-derived by hand at this volume) -- one concrete real bug surfaced along +the way: Python's raw extraction for `2020_Sarawak General Hospital A4D +Tracker_DC_product_raw.parquet`, sheet `May20`, has a stray extra row with +`product_entry_date = "\n"` (a literal newline) that R's output doesn't +have. Several other files show a similar single-row insertion/shift pattern +within one sheet. Rather than leave the 50 as untracked residue in this +closed ticket -- where nobody would think to look for undecided +differences -- folded them into [ticket +22](22-triage-product-raw-columns.md)'s scope (it already owns "remaining +raw-stage product columns"; `product_entry_date`'s tail is now small enough +to be one more column in that same pass) rather than spawning a separate +ticket. One `ce_typo` case (`06 Penang General Hospital..._Apr26`) has +Python parsing to year `3026` where R has no value at all -- a genuine +year-typo-rescue miss, also left for ticket 22 rather than fixed here. + +**Tense**: all claims above describe current, executed behaviour -- the +code is committed and the counts are from a real re-run, not a projection. diff --git a/docs/wayfinder/tickets/21-triage-remaining-product-columns.md b/docs/wayfinder/tickets/21-triage-remaining-product-columns.md new file mode 100644 index 0000000..6993d9e --- /dev/null +++ b/docs/wayfinder/tickets/21-triage-remaining-product-columns.md @@ -0,0 +1,159 @@ +--- +id: 21 +title: Triage the remaining product cleaned-stage column mismatches (balance, received_from, released_to, remarks, units_received, product) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-12 +claimed_at: 2026-08-12T10:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 18 +--- + +## Premise + +Rests on [Triage every flagged R/Python difference for both arms, and +resolve the 189-vs-155-tracker discrepancy](18-triage-comparison-flagged-differences.md), +closed: the product row-alignment key is sound (ticket 17, 97.2% match) and +`product_category`/`product_entry_date` are already root-caused and +classified. This ticket inherits the remaining `product_data_cleaned` +per-column mismatch counts from that session's `just compare-outputs` run, +unexamined: `product_balance` (2,343), `product_received_from` (330), +`product_released_to` (3,834 — not among the causes the parity-presentation +PDF named at all, so there is no prior baseline count to compare against), +`product_remarks` (66), `product_units_received` (265), and `product` itself +(652 — the product name/identity column disagreeing is a different kind of +divergence than a value column and may need its own explanation). + +Also rests on the 189-vs-155-tracker finding: per-column counts are not +expected to reproduce the PDF's numbers exactly, since the PDF's baseline +population is provably different and unreachable — the bar is explaining +each divergence's pattern, not hitting an exact count (per ticket 18's +resolution). + +**Updated same day, per ticket 18's addendum:** R and Python were both +re-run against the current 248-tracker production set (up from 177) and +`output_r/` now holds that fresh R output (old baseline preserved at +`output_r_155_frozen_backup_2025-11-14`). This changed the picture +significantly for the columns already root-caused (`product_category` +dropped from 13,638 to 866 mismatches — most of the old count was baseline +staleness, not the join-logic bug itself) and moves the 189-vs-155 +question closer to reconciled (229/66,020 now, vs the PDF's 189/61,077). +Refreshed counts for this ticket's remaining columns (same run, +`output/comparison/2026-08-11T232419Z/`): `product_balance` 2,740 (was +2,343), `product_received_from` 366 (was 330), `product_released_to` 4,490 +(was 3,834), `product_remarks` 70 (was 66), `product_units_received` 305 +(was 265), `product` 652 (unchanged), plus a new `file_name` mismatch count +of 60 not seen in the original session (unexplained — check whether this is +a new comparison-tool column or a real divergence before assuming it's in +scope here). Use the current `output_r`/`output_python`, not the backup. + +## Question + +For each of `product_balance`, `product_received_from`, `product_released_to`, +`product_remarks`, `product_units_received`, and `product`: pull the flagged +mismatch rows (`just compare-outputs`'s `cell_mismatches` sheet, cleaned +stage), look for a systematic pattern the way ticket 18 found for category/ +entry_date (e.g. is it also majority R-null-Python-has-value, or a different +shape entirely), fall back to the real source Excel trackers +(`a4dphase2_upload`) as the arbiter, and add a named cause to +`src/a4d/migration/compare.py`'s classifier registry once a pattern is +confirmed. `product_released_to` in particular has no PDF precedent to +compare against — start by establishing whether it's a real divergence or +another comparison-tool artifact (per ticket 20's finding that at least one +column had a false-positive representation issue) before assuming it needs +a data explanation. + +## Resolution + +**Decision.** Five of the six columns share one root cause, already +half-diagnosed by ticket 18: R's per-(clinic, sheet, product) row sort +(`read_product_data.R:606-617`) falls back to raw input-row order whenever +`product_entry_date` fails to parse for a row. Verified this failure is +*near-universal*, not occasional, for several major clinics — R's +`product_entry_date` is null on literally 100% of "change"-status rows for +2024 Mahosot, 2023 Mahosot, 2022 Mahosot DC, 2020 Mahosot DC, and 99.2% for +2020 Mahosot DC (`output_r`/`output_python`, current 248-tracker set), while +Python correctly parses the same cells (≤3.7% null on the same rows) and +sorts chronologically instead — both sides implement the *identical* +documented rank algorithm (`_fill_product_names_and_sort`'s docstring +already cites the R line numbers; confirmed by re-reading `read_product_data.R` +directly), so this isn't a Python bug, it's R's already-known date-extraction +gap (ticket 18's `r_value_missing`) resurfacing as a *sort-order* divergence. +Since `add_row_ordinal`'s alignment key is purely positional, that legitimate +order difference cascades into value-level "mismatches" on every column +compared through it, even though nothing about the data itself is wrong. +Traced one full example by hand (Mahosot 2024, Jan24, "NovoFine Needles 6mm +x 32G (100s)"): R's cleaned rows show `product_entry_date = null` throughout +and released-then-received in raw appearance order; Python's show the real +dates (2024-01-03 … 2024-01-31) with the receipt correctly slotted in by +date — both reach the same final balance (13), just by a different path. +Confirmed this generalizes: of 2,105 (sheet, product) groups with any +`product_balance` mismatch, 98.3% still land on the *same* end-of-group +balance on both sides (`product_balance_status == "end"`), i.e. the flow is +conserved and only the intermediate path differs. + +Added `row_order_divergence` to `src/a4d/migration/compare.py`'s classifier +registry (`PRODUCT_ROW_ORDER_CLASSIFIERS`): `compare_cells` gained an opt-in +`order_group_cols` parameter that, when a mismatch's R value appears +anywhere in Python's own value multiset for its `(clinic_id, +product_sheet_name)` group, flags `CellMismatch.row_order_candidate`; the +classifier just reads that flag. Wired for the five affected columns in +`scripts/compare_outputs.py`. Verified against the real 248-tracker +`output_r`/`output_python` pair: `product_received_from` (366/366, 100%), +`product_released_to` (4,490/4,490, 100%), `product_remarks` (70/70, 100%), +and `product_units_received` (297/305, 97.4%) are now fully or almost fully +explained by this classifier. `product_balance`'s membership check only +catches 764/2,740 (27.9%) directly — a cumulative running total rarely +recurs another row's exact value by coincidence, so the multiset-membership +heuristic under-detects it even though the 98.3% end-balance-match evidence +above says the *same* root cause covers the rest. Building a +balance-specific detector (e.g. comparing per-group ordered value multisets +instead of per-cell membership) is possible future work, not done here — +diminishing returns for this session given the root cause is already +confirmed by other means. **Rejected**: "fixing" Python to replicate R's +sort-order fallback — Python's chronological order is the one that's +actually correct (R's is a data-extraction failure, not an intentional +convention), so matching it would mean deliberately reintroducing a bug. + +`product_units_received`'s residual 8 mismatches (all `43566`/`43644`/`43708` +in R vs `0` in Python) are Excel date serials leaking into a unit-count +column — the exact same "openpyxl date/time auto-coercion" pattern [ticket +24](24-triage-remaining-raw-column-residual.md) already flagged as an open, +unverified theory for the *raw*-stage residual, now confirmed to propagate +through to the *cleaned* stage too. Left for ticket 24 rather than fixed +here — same open question, just new cleaned-stage evidence for it. + +`product` (652 mismatches) had a *different*, single cause: an embedded +`\r\n`-vs-`\n` line break (e.g. `"FastClix \r\nLancets"` vs `"FastClix +\nLancets"`) that ticket 22 already normalizes for the raw stage but that +survives cleaning unnoticed, since step 2.16's `str.strip_chars()` only +trims line ends, not mid-string breaks. Wired `normalize_whitespace_column` +for the cleaned stage too, scoped to `product` alone (the only cleaned-stage +column found carrying multi-line values). Verified: `product`'s mismatch +count drops from 652 to 0. + +`file_name` (60 mismatches, flagged as unexamined in this ticket's premise) +turned out out of scope: it's not one of the six columns this ticket names, +and a quick look shows it's the row-alignment *diagnostic* column +(`add_row_ordinal` prepends `file_name` verbatim as one of the compared +columns), not real tracker data — left untouched, not investigated further. + +**New gap surfaced, not this ticket's scope**: `product_units_released` +(2,144 mismatches) was never assigned to any ticket — not this one, not +[ticket 24](24-triage-remaining-raw-column-residual.md) (which only covers +its much smaller raw-stage residual, 24 rows). Spawned [ticket +25](25-triage-product-units-released-cleaned.md) rather than pulled in here, +per the map's split-rather-than-sprawl rule — plausibly the same +row-order-divergence root cause (unverified), but that needs its own look. + +Full suite (577 tests + 1 pre-existing skip), ruff, `ty check src/` all +pass. Verified end-to-end against the real 248-tracker drive data, not just +unit tests. + +**Evidence.** Executed: every claim above was checked directly against the +real `output_r`/`output_python` parquet pair and/or the actual comparison +tool output (`just compare-outputs`), not inferred from documentation or a +prior session's numbers. diff --git a/docs/wayfinder/tickets/22-triage-product-raw-columns.md b/docs/wayfinder/tickets/22-triage-product-raw-columns.md new file mode 100644 index 0000000..01cae88 --- /dev/null +++ b/docs/wayfinder/tickets/22-triage-product-raw-columns.md @@ -0,0 +1,163 @@ +--- +id: 22 +title: Triage the remaining product raw-stage column mismatches +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-12 +claimed_at: 2026-08-12 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 18 +--- + +## Premise + +Rests on [Triage every flagged R/Python difference for both arms, and +resolve the 189-vs-155-tracker discrepancy](18-triage-comparison-flagged-differences.md), +closed: `product_entry_date`'s raw-stage report was found to be ~99% +representation-artifact noise (R stores unparsed Excel serials, Python +stores parsed dates), split into [ticket +20](20-fix-raw-entry-date-representation.md). The other raw-stage product +columns from that session's run are untouched: `product_balance` (2,397), +`product_received_from` (94), `product_released_to` (70), `product_remarks` +(65), `product_units_received` (269), `product_units_returned` (1), and +`product` (685). + +Not blocked on ticket 20: these are different columns, and there's no +evidence yet they share entry_date's serial-vs-parsed problem (most are +numeric or plain string columns, not dates) — check each independently +before assuming the same representation issue applies. + +**Updated after ticket 20 closed:** normalization dropped raw-stage +`product_entry_date` mismatches from 65,743 to 91, but didn't triage the +residual — 46 already land in ticket 18's existing seeded classifiers +(`ce_typo`, `off_by_one_day`, `sentinel_null`, `r_value_missing`), leaving +50 `unclassified` and undecided, per the destination's "every difference +documented and explicitly decided" bar. Folded into this ticket's scope +rather than spawning a separate one, now that it's down to a small tail +alongside the other raw columns. One concrete finding already surfaced +while triaging (not chased further): Python's raw extraction for +`2020_Sarawak General Hospital A4D Tracker_DC_product_raw.parquet`, sheet +`May20`, has a stray extra row with `product_entry_date = "\n"` (a literal +newline) that R's output doesn't have — a real extraction bug, not a +representation or ordering artifact. Several other files in the residual +(same tracker across other months, `2020_Sultanah Bahiyah Hospital...`, +`2024_CDA A4D Tracker..._Dec24`, `2019_Vietnam National Children's +Hospital..._Oct19`) show a similar single-row insertion/shift pattern +within one sheet — worth checking whether they share the same cause before +assuming each is independent. + +**Updated same day, per ticket 18's addendum:** R and Python were both +re-run against the current 248-tracker production set (up from 177) and +`output_r/` now holds that fresh R output (old baseline preserved at +`output_r_155_frozen_backup_2025-11-14`). Refreshed raw-stage counts (same +run, `output/comparison/2026-08-11T232419Z/`): `product_balance` 712 (was +2,397), `product_received_from` 133 (was 94), `product_released_to` 70 +(unchanged), `product_remarks` 83 (was 65), `product_units_received` 109 +(was 269), `product_units_returned` 1 (unchanged), `product` 721 (was 685). +Use the current `output_r`/`output_python`, not the backup. + +## Question + +For each remaining raw-stage product column — including `product_entry_date`'s +50 residual `unclassified` rows now that ticket 20 has cleared the +representation-artifact bulk of it — pull the flagged mismatch rows and +determine whether the divergence is a real R/Python content difference +(worth a named classifier and, if it points at a Python bug, a fix) or +another comparison-tool artifact like ticket 20's (worth a normalization +fix instead). Fall back to the real source Excel trackers as the arbiter +per the map's standing preference. If any raw-stage finding turns out to +also explain part of the corresponding cleaned-stage divergence from +[ticket 21](21-triage-remaining-product-columns.md), note the connection +there rather than re-deriving it. + +## Resolution + +**Decision:** three distinct causes triaged and resolved for six of the eight +remaining raw-stage columns; the residual (105 rows across +`product_units_received`, `product_units_released`, `product_received_from`) +didn't converge to a single cause and is split off into [ticket +24](24-triage-remaining-raw-column-residual.md). + +**Because**, in order of investigation: + +1. **Real Python bug, fixed**: `remove_header_rows` (`src/a4d/extract/product.py`) + dropped rows where every column was `None`, but a formula-emptied Excel + cell can surface via openpyxl as `""` rather than `None` -- so a row with + one stray `""` cell (everything else genuinely `None`) survived + extraction when R's `is.na()`-based check (`read_product_data.R` step + 1.6/1.7) drops it. Confirmed directly: Python's raw parquet for + `2019_Vietnam National Children's Hospital..._Oct19` had 815 rows against + R's 814, with the extra row isolated to one blank row inserted before a + totals-like row. Fixed by treating a stripped-empty string as blank too, + the same way the rest of the row's `None`s already are. This is the same + mechanism as the Sarawak `"\n"`-dated row ticket 20 flagged but didn't + chase -- re-verified after the fix: that row is gone too (191/191 rows, + was 191/192). Regression test added + (`test_remove_header_rows_drops_row_of_empty_strings`). This was the root + cause of the "single-row insertion/shift pattern" this ticket's premise + flagged across several files -- fixing it collapsed a whole cascade of + downstream cell mismatches (`product_entry_date`, `product_released_to`, + `product_units_received`, `product_received_from`, `product_units_released`, + `product` all shifted together for an affected file's rows after the + blank row). +2. **Comparison-tool artifact, normalized**: `product_balance`'s 712 + mismatches were R's and Python's own float-to-string conversions + rounding a binary float's trailing digits differently (e.g. + `"9.300000000000001"` vs `"9.3"`, numerically identical to 1e-9). + `normalize_numeric_column()` (`src/a4d/migration/compare.py`) parses both + sides back to `float` before comparing, so the existing float-tolerance + in `_values_differ` applies instead of a raw string diff. Verified: 0 + `product_balance` mismatches remain against the real drive data. +3. **Comparison-tool artifact, normalized**: the large majority of + `product`, `product_remarks`, `product_released_to`, and + `product_units_returned`'s mismatches were whitespace/line-ending + representation, not content -- two distinct readxl defaults: (a) + `trim_ws = TRUE` strips leading/trailing whitespace R-side, which + openpyxl-based Python extraction preserves as-is, including reducing a + whitespace-only cell to `""` that R's trim reduces further to `NA`; (b) + readxl represents an embedded line break as `\r\n`, openpyxl normalizes + it to `\n` alone -- confirmed byte-for-byte on a `product` mismatch that + looked identical in `repr()` but wasn't (`\r\n` vs `\n`) until this was + found. `normalize_whitespace_column()` handles both. Verified: `product`, + `product_remarks`, `product_released_to`, and `product_units_returned` + all dropped to 0 mismatches against the real drive data. + +Combined effect, verified end-to-end against the real `output_r`/ +`output_python` on the USB drive (248-tracker production set, same run +ticket 18's addendum re-ran R and Python against): raw-stage product +mismatches (all 9 columns) dropped from 2,007 to 105 -- a 95% reduction. +`product_entry_date`'s own residual (91, carried over from ticket 20) is now +fully classified into existing seeded causes (`ce_typo` 25, +`off_by_one_day` 2, `sentinel_null` 6 = 33 total, 0 `unclassified`) -- +`r_value_missing`'s single case and 50 `unclassified` rows both resolved by +the blank-row fix above, since they were downstream of the same row-shift. + +**Rejected**: extending `normalize_whitespace_column`/ +`normalize_numeric_column` speculatively to the residual columns +(`product_units_received`, `product_units_released`, +`product_received_from`) without first understanding their pattern -- +a first look showed at least two more distinct causes there (an +openpyxl date/time auto-coercion leak, and an unexplained value-shift +in `product_units_released` that isn't the same row-shift bug already +fixed), not one. Per the map's "split rather than sprawl" rule, chasing +those inline would have pushed this ticket well past its one-session +scope; ticket 24 carries them instead. + +**Evidence**: executed throughout -- the blank-row fix was verified via a +regression test and a live re-run of `a4d run product` against the full +248-tracker USB-drive dataset (not just unit tests), and both +normalizations were verified against `just compare-outputs` run against the +real `output_r`/`output_python` before-and-after. Full suite (571 passed, 1 +skipped), ruff, `ty check src/` all pass. + +**Tense**: all claims above describe current, executed behavior -- not a +proposed design. + +**Incidental fix**: a prior run of this session accidentally wrote a +Python-output run to `a4dphase2_upload/output_python/` (a relative +`A4D_OUTPUT_DIR` combined with `data_root`) instead of the drive's +top-level `output_python/` that `output_r/` and prior sessions' runs live +next to; caught mid-session and removed, no lasting effect on the drive. diff --git a/docs/wayfinder/tickets/23-triage-patient-arm.md b/docs/wayfinder/tickets/23-triage-patient-arm.md new file mode 100644 index 0000000..2585c65 --- /dev/null +++ b/docs/wayfinder/tickets/23-triage-patient-arm.md @@ -0,0 +1,106 @@ +--- +id: 23 +title: Triage every flagged R/Python difference for the patient arm (raw and cleaned) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-12b +claimed_at: 2026-08-12T09:20:00+00:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 18 +--- + +## Premise + +Rests on [Build and run the R/Python output comparison script, then triage +every flagged difference](15-build-and-run-comparison-script.md), closed: +the patient arm's row-alignment key (`patient_id` + `sheet_name`) was +confirmed sound (duplicate keys in only 5/172 files) and a full comparison +ran successfully, but the flagged differences were never triaged — all of +this map's triage attention (tickets 15, 17, 18) went to the product arm, +whose row-alignment key needed fixing first. [Triage every flagged R/Python +difference for both arms, and resolve the 189-vs-155-tracker +discrepancy](18-triage-comparison-flagged-differences.md), also closed, +reconfirmed the patient arm was still untouched and split it out explicitly +rather than starting it in an already-large session. + +Also rests on ticket 18's 189-vs-155-tracker finding for the product arm +specifically — check whether an analogous tracker-count gap exists on the +patient side (R: 172 files, Python: 174 files per ticket 18's session) before +assuming patient's per-column counts are directly comparable to any prior +baseline; the gap here is much smaller (2 files) than product's (34), so it +may not need the same treatment, but confirm rather than assume. + +**Updated same day, per ticket 18's addendum:** R and Python were both +re-run against the current 248-tracker production set (up from 177) — +`output_r/` now holds fresh R output (243 patient-cleaned files, 81,859 +rows; old baseline preserved at `output_r_155_frozen_backup_2025-11-14`) +and `output_python/` holds a fresh 248-tracker run (84,407 monthly rows). +The 172-vs-174 gap above is from the old snapshot and needs re-checking +against these new file counts. A fresh comparison already exists at +`output/comparison/2026-08-11T232419Z/` (`snapshot_patient_data_raw.json`, +`snapshot_patient_data_cleaned.json`) — use that rather than the `/tmp` +reports the original session produced, which no longer reflect the current +tracker set. + +## Question + +Run `just compare-outputs` against the USB drive's `output_r`/`output_python` +for `patient_data_raw` and `patient_data_cleaned` (a fresh run already +exists at `output/comparison/2026-08-11T232419Z/` — re-run only if working +this ticket long after that point, since the tracker set may have grown +further), and go through the flagged per-column differences the way +ticket 18 did for product: look for systematic patterns, fall back to the +real source Excel trackers as the arbiter, and add named causes to +`src/a4d/migration/compare.py`'s classifier registry as patterns are +confirmed (there is currently no `PATIENT_*_CLASSIFIERS` registry at all — +this ticket would create the first one, if warranted). This ticket is large +(patient has more columns and a longer history than product) — if it +doesn't converge in one session, split further rather than leaving it +open-ended, per the pattern ticket 18 itself used. + +## Resolution + +**Decision:** the patient raw-stage mismatch dominant cause is the same +date-representation artifact ticket 20 fixed for product's raw +`product_entry_date`. Extended `normalize_date_column` to the patient +`Patient (raw)` stage in `scripts/compare_outputs.py`, using the cleaned +schema's own `get_date_columns()` helper (18 `pl.Date` columns) rather than +a hand-maintained list. Did not touch pipeline code — comparison tooling +only. + +**Because:** a re-run of the fresh 248-tracker comparison (a newer run than +the premise's referenced `2026-08-11T232419Z` snapshot already existed +locally at `output/comparison/2026-08-12T085925Z/`, so that was used +instead of triggering another full pipeline pass) showed patient raw-stage +mismatches concentrated almost entirely in date columns (`dob`: 79,836; +`last_clinic_visit_date`: 73,490; `fbg_updated_date`: 61,832; +`hba1c_updated_date`: 61,769; `bmi_date`: 61,765; `recruitment_date`: +51,605; `t1d_diagnosis_date`: 47,047; etc. — 17 of 18 schema date columns +appeared in the top of the list). A direct file check (`dob` on +`06 00GS Quirino Memorial Medical Center A4D Tracker_Jun_26`) confirmed the +exact pattern: R holds the unparsed Excel serial (`"39490"`), Python holds +the already-ISO-parsed date (`"2008-02-12 00:00:00"`). + +**Rejected:** doing the full per-column triage for both raw and cleaned in +one session, as the ticket originally scoped — killed by scale, not by a +premise problem. After the date fix, raw-stage mismatches dropped from +564,096 to 46,788 (91.7%) across 70 -> 67 columns, but 46,788 across 67 +columns is still not a one-session triage, and the cleaned stage (120,639 +across 61 columns, confirmed unaffected by this fix) is untouched and +larger still. Split into [ticket 27](27-triage-patient-raw-residual.md) +(raw residual, with two leads already noted: `complication_screening` +looks like a genuine multi-select extraction gap, and `meter_received_date` +is a raw-only date column the schema-derived list didn't catch) and +[ticket 28](28-triage-patient-cleaned.md) (cleaned stage, untouched). + +**Evidence:** executed — verified end-to-end against the real 248-tracker +drive comparison (before/after snapshots diffed directly), a direct +parquet-level check of the raw representation difference, `uv run pytest +tests/test_migration` (83 passed), and `ruff check` clean. No pipeline code +changed, only `scripts/compare_outputs.py` (comparison tooling). + +**Tense:** all claims describe current, verified behaviour — not a +proposed design. diff --git a/docs/wayfinder/tickets/24-triage-remaining-raw-column-residual.md b/docs/wayfinder/tickets/24-triage-remaining-raw-column-residual.md new file mode 100644 index 0000000..5b24d09 --- /dev/null +++ b/docs/wayfinder/tickets/24-triage-remaining-raw-column-residual.md @@ -0,0 +1,174 @@ +--- +id: 24 +title: Triage the residual product_units_received/product_units_released/product_received_from raw-stage mismatches +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-12e +claimed_at: 2026-08-12 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 22 +--- + +## Premise + +Rests on [Triage the remaining product raw-stage column +mismatches](22-triage-product-raw-columns.md), closed: a real blank-row +extraction bug and two comparison-tool representation artifacts (float +string formatting, whitespace/line-ending) together cut raw-stage product +mismatches from 2,007 to 105 (95%), fully resolving `product`, +`product_balance`, `product_entry_date`, `product_remarks`, +`product_released_to`, and `product_units_returned`. The 105-row residual +is what's left: `product_units_received` (75), `product_units_released` +(24), `product_received_from` (6), on the real `output_r`/`output_python` +comparison against the current 248-tracker production set +(`output/comparison/2026-08-12T074911Z/compare_report_product_data_raw.xlsx` +in the repo's `output/` directory as of this writing, or re-run `just +compare-outputs` against the drive for a fresh count). + +A first look during ticket 22 found at least two more distinct, +not-yet-root-caused patterns in this residual, neither the same as any +cause ticket 22 already fixed: + +1. **Possible openpyxl date/time auto-coercion leak** (majority of + `product_units_received`'s 75, a handful of `product_received_from`'s + 6): R's raw value is a plain number or `"0"`; Python's is a + parsed-looking date/time string (`"00:00:00"`, `"1900-03-15 00:00:00"`, + `"2019-04-11 00:00:00"`). Distinct from ticket 20/22's date-column + representation fix -- these aren't `product_entry_date`, they're columns + that shouldn't hold dates at all. Working theory (not verified): the + source Excel cell carries date/time number-formatting that openpyxl + auto-converts to a Python `datetime`/`time` object regardless of the + cell's logical column, while R's raw extraction doesn't. If confirmed, + this could be a systemic raw-extraction issue (any column, any arm), not + specific to these two. +2. **Unexplained value pattern** in `product_units_released` (24 rows, + concentrated in Mandalay/Vietnam files): e.g. R has `"2 MM_MD023"`, + Python has `"2 MM_MD023-1"` -- values that look like `product_released_to` + content (a patient ID), not a unit count, appearing in the + `product_units_released` column. Not explained by ticket 22's blank-row + fix (row counts already match for at least one sample file checked) and + not the wide-format cell-splitting logic in + `src/a4d/extract/wide_format.py::handle_wide_format_cells` (that function + only fires for 2017-2019 Mandalay files and splits `Released To`, it + doesn't append `-1`/`-2` suffixes to `Units Released`). Needs a fresh + look at the source Excel for one of these rows to see whether this is a + genuine content divergence, another duplicate-ordinal-key pairing + ambiguity (like the `product` column mismatches ticket 22 found and + fixed via the `\r\n` normalization -- verify row order/uniqueness within + the `add_row_ordinal` group before assuming content itself differs), or + something else. + +## Question + +For `product_units_received`, `product_units_released`, and +`product_received_from`'s residual mismatches: confirm or refute the +date/time-coercion theory above (check the source Excel cell's number +format directly, not just the parquet output), and root-cause the +`product_units_released` value-shift pattern. Decide, per column, whether +each finding is a real Python bug (fix it), a comparison-tool +representation artifact (normalize it, following `normalize_date_column`/ +`normalize_numeric_column`/`normalize_whitespace_column`'s precedent in +`src/a4d/migration/compare.py`), or something else. If the date/time-coercion +theory is confirmed and looks systemic (not just these two columns), flag +that explicitly rather than only patching the two columns it happens to +surface in here -- it may need its own ticket covering the raw extraction +approach more broadly (patient arm included). + +## Resolution + +**Decision:** All 105 residual mismatches across the three columns are now +fully explained by comparison-tool cause classifiers -- none required a +pipeline code change. `product_units_received` (75) and +`product_received_from` (5) are entirely the openpyxl date/time-coercion +pattern; `product_units_released` splits into 19 float-precision rows +(fixed by the same normalization) and 5 wide-format-parsing-ambiguity rows +(a new classifier). + +**Because:** +- **Date/time coercion (80 rows, confirmed against real source Excel, not + just the data pattern):** R's readxl infers a column's type from its + majority values, so a lone Excel date/time-formatted cell in an + otherwise-numeric column (a genuine, if rare, source-data anomaly -- e.g. + someone typed a date into a "Units Received" cell) still gets coerced to + that column's numeric type, the raw serial. Python's openpyxl reads each + cell individually and honors its own format, returning a `datetime`/ + `time` object instead. Verified directly: Penang General Hospital 2019 + Apr19!E36 (a "Units Received" cell) is genuinely formatted `d/m/yy` and + holds a date value; CDA/NPH/Yangon 2023 files hold `time(0, 0)`-formatted + cells reading `"0"` on R's side. Neither side is wrong -- both faithfully + extract what the cell holds per their own library's semantics -- so this + is a comparison-tool classifier + (`STRAY_DATE_CLASSIFIERS`/`_is_openpyxl_date_typed_stray_cell`, + `src/a4d/migration/compare.py`), not a pipeline fix. It converts the + R-side serial via openpyxl's own `from_excel` (not hand-rolled epoch + math) specifically to replicate the Excel/Lotus 1900-leap-year bug + (serials 59 and 60 both resolve to 1900-02-28) -- caught by a real + flagged row (r_value 59) during verification, not anticipated up front. + Per the ticket's own instruction, this is flagged as systemic rather than + scoped to just these two columns: the same + numeric-column-with-a-stray-date-cell shape could appear anywhere, + including the patient arm. Not spawning a new ticket for it -- [ticket + 27](27-triage-patient-raw-residual.md) already covers "patient raw-stage + residual" and is still open, so the lead (and the reusable classifier) is + noted there instead as an addendum. +- **Float precision (20 rows, `product_units_released` 19 + + `product_received_from` 1):** the same R/Python float-to-string rounding + difference `normalize_numeric_column` already fixed for `product_balance` + (ticket 22) -- just not yet wired for these three quantity columns. + Extended `PRODUCT_RAW_NUMERIC_NORMALIZE_COLS` in + `scripts/compare_outputs.py` to include all three. +- **Wide-format fragment truncation (5 rows, `product_units_released` + only, 2017-2019 Mandalay files):** confirmed against real source Excel + (`2017_Mandalay Children's Hospital A4D Tracker.xlsx` Jul17!K15, + `2019_Mandalay Children's Hospital A4D Tracker.xlsx` Feb19!K48-49) that + the source "Released To" cells hold messy free-text notes with embedded + parentheses and extra hyphens + (e.g. `"MM_MD019-2(Error-1),MM_MD043-1,..."`) that + `handle_wide_format_cells`'s comma/hyphen split doesn't fully anticipate. + In every flagged case R's value is a strict text prefix of Python's + (`"2(Error"` vs `"2(Error-1)"`) -- R's equivalent step drops content + after a second hyphen, Python's keeps the full remainder. Python is + verified more faithful to the source text, not R -- this is a genuine R + parsing gap on inherently messy human-entered notes, not a Python bug to + fix, and not something to chase toward R's more-truncated answer. New + classifier `WIDE_FORMAT_FRAGMENT_CLASSIFIERS`/ + `_is_wide_format_fragment_truncated`. + +**Rejected:** +- Fixing `handle_wide_format_cells` to parse the 5 ambiguous Mandalay cells + "correctly" -- rejected because there is no single correct answer for + genuinely ambiguous human-entered free text (a missing comma between two + name-qty pairs), and R's own answer for the same cells is demonstrably + more lossy, not more correct. Classifying and documenting the divergence + is more honest than writing brittle guessing logic for 5 rows. +- Spawning a new ticket for the date/time-coercion pattern's patient-arm + implications -- rejected in favor of noting it as a lead on the + already-open, already-scoped [ticket 27](27-triage-patient-raw-residual.md) + rather than fragmenting the map further. + +**Evidence:** executed. Every root cause was verified directly against the +real 248-tracker drive data and the real source Excel files (not just the +data pattern) -- see the specific cell coordinates cited above. The fix was +verified end-to-end by re-running `just compare-outputs` against the real +drive data after implementation: all 105 residual rows across the three +columns now carry a named cause (0 `unclassified`), and the `Product +(cleaned)`/`Patient (raw)`/`Patient (cleaned)` reports are confirmed +byte-identical to the prior run ("no change from the previous run"), +i.e. this ticket's changes didn't touch anything outside its raw-stage +product-column scope. Full test suite (592 passed, 1 skipped), ruff, +`ruff format --check`, and `ty check src/` all pass. + +**Tense:** all claims above describe current, executed behavior (the fix +is implemented and pushed), not a proposed design. + +Full change: `src/a4d/migration/compare.py` (new +`_excel_serial_to_datetime`, `STRAY_DATE_CLASSIFIERS`, +`WIDE_FORMAT_FRAGMENT_CLASSIFIERS` + their classifier functions), +`scripts/compare_outputs.py` (new `PRODUCT_RAW_NUMERIC_NORMALIZE_COLS`, +updated `CLASSIFIERS_BY_COLUMN`, reordered whitespace-before-numeric +normalization to avoid a dtype conflict), `tests/test_migration/test_compare.py` +(10 new classifier tests, including a regression test for the 1900 +leap-year edge case). diff --git a/docs/wayfinder/tickets/25-triage-product-units-released-cleaned.md b/docs/wayfinder/tickets/25-triage-product-units-released-cleaned.md new file mode 100644 index 0000000..fc71507 --- /dev/null +++ b/docs/wayfinder/tickets/25-triage-product-units-released-cleaned.md @@ -0,0 +1,165 @@ +--- +id: 25 +title: Triage the product_units_released cleaned-stage column mismatches +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-12h +claimed_at: 2026-08-12T22:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 21 +--- + +## Premise + +Rests on [Triage the remaining product cleaned-stage column +mismatches](21-triage-remaining-product-columns.md), closed: five sibling +cleaned-stage columns (`product_balance`, `product_received_from`, +`product_released_to`, `product_remarks`, `product_units_received`) turned +out to be dominated by one root cause — R's per-(clinic, sheet, product) +row sort falling back to raw input-row order whenever +`product_entry_date` fails to parse (a near-universal failure for several +major clinics, confirmed against real data), while Python correctly sorts +chronologically. Since the row-alignment key (`add_row_ordinal`) is purely +positional, that legitimate order difference cascades into value-level +mismatches on every column compared through it. A `row_order_divergence` +classifier (`PRODUCT_ROW_ORDER_CLASSIFIERS` in +`src/a4d/migration/compare.py`) now detects this directly for value columns +whose mismatched R value reappears elsewhere in Python's own group. + +`product_units_released` was never assigned to any ticket: ticket 21 named +six specific columns and this wasn't one of them, and [ticket +24](24-triage-remaining-raw-column-residual.md) only covers this column's +much smaller *raw*-stage residual (24 rows, a distinct "unexplained value +pattern" ticket 24 is still chasing) — not the cleaned stage. On the current +248-tracker `output_r`/`output_python` comparison +(`output/comparison/2026-08-12T084709Z/` as of this writing, or re-run `just +compare-outputs`), `product_units_released` has 2,144 cleaned-stage +mismatches, the second-largest count among all product columns after +`product_entry_date`. + +## Question + +Pull `product_units_released`'s flagged cleaned-stage mismatch rows and +determine whether they're explained by the same row-order-divergence root +cause ticket 21 found for its five sibling columns (plausible but +unverified — check whether wiring `product_units_released` into +`PRODUCT_ROW_ORDER_CLASSIFIERS` explains most of the count, and whether +end-of-group balance/totals are preserved the way ticket 21 verified for +`product_balance`), or whether it's a distinct pattern requiring its own +explanation. If it's the same cause, wiring the existing classifier may be +enough; if not, fall back to the real source Excel trackers +(`a4dphase2_upload`) as the arbiter per the destination's standing +verification bar. + + +## Standing bar (added 2026-08-12g, applies to this ticket) + +Per the map's **triage means deciding, not labelling** preference: it is not +enough to explain a difference and name a cause. Each one must also carry an +explicit verdict on **whether Python is doing the right thing**, with the +evidence that establishes it (source Excel, or R's/Python's own code). Noting +"Python has A where R has B" and adding a classifier is not a decision in +favour of A. Where Python turns out to be wrong or to be losing information +the source file carried, fix the pipeline rather than labelling the symptom +— see [ticket 27](27-triage-patient-raw-residual.md), where exactly that +turned a labelling job into a real extraction fix. A cause genuinely +undecidable on available evidence is recorded as an open question, not +closed with a label. + +## Resolution (session-2026-08-12h) + +**Decision.** All 2,144 cleaned-stage `product_units_released` mismatches are +the same row-order divergence [ticket 21](21-triage-remaining-product-columns.md) +root-caused for this column's five siblings — not a distinct pattern. +**Python is doing the right thing**; R is not. No pipeline change. The +comparison tool's column-to-registry map was the only defect: this column +carried `WIDE_FORMAT_FRAGMENT_CLASSIFIERS` alone (raw-stage-only, ticket 24) +and so reported every cleaned-stage row as `unclassified`. Wiring +`PRODUCT_ROW_ORDER_CLASSIFIERS` alongside it takes the column from +2,144 `unclassified` to 2,144 `row_order_divergence` / **0 unclassified**. + +**Because.** Three independent checks, each executed against the real +248-tracker `output_r`/`output_python` pair on the drive: + +1. **Nothing is added, lost or altered.** Across all 229 files and all 2,283 + `(clinic_id, product_sheet_name)` groups, R's and Python's *multisets* of + `product_units_released` values are byte-identical (0 differing groups). + Per-file totals agree everywhere too — the report's `totals` sheet lists + no `product_units_released` row for any of the 229 files, while listing 34 + for other columns. +2. **Row identity survives the re-sort.** Across all 11,649 + `(clinic_id, product_sheet_name, product)` groups, the multiset of + `(product_units_released, product_released_to)` *pairs* is identical + between R and Python (0 differing groups) — every unit release keeps its + recipient. The difference is purely which position within the group each + row occupies. (Group keys whitespace-normalized for this check, including + embedded line breaks, per ticket 21's `\r\n`-vs-`\n` product-name finding; + without that normalization 430 groups appear to differ under different + names for the same group.) +3. **Python's order is the correct one, per source.** In the worst-affected + file/sheet (`2024_Mahosot Hospital A4D Tracker`, `Jul24`, product + `Accu-Chek Performa Test Strips (50s/ bottle)`, 69 differing rows), R's + output has `product_entry_date = null` on **every** row of the group, + while Python has real parsed dates. Read directly from the source Excel + (`a4dphase2_upload/Laos/MHS/2024_Mahosot Hospital A4D Tracker.xlsx`, + sheet `Jul24`, header row 15): the "Entry Date" column is fully populated + with genuine `datetime` values (2024-07-01, 2024-07-23, 2024-07-19, ...). + So R falls back to raw input order because of its own date-extraction gap + — the same gap already root-caused as `r_value_missing` (ticket 18) — and + Python sorts chronologically on dates that really are in the file. Both + sides implement the same documented rank algorithm; only the date input + differs. + +**Rejected.** +- *Closing on the classifier hit alone.* 100% of the rows set + `row_order_candidate`, which looks conclusive but isn't: that flag only + asks whether R's value appears **anywhere** in Python's group, and on a + numeric column full of repeated small values (0, 2, 4, 8) it false-fires + easily — a risk `CLASSIFIERS_BY_COLUMN` already documents for + `product_received_from`. Checks 1 and 2 above are what actually establish + the claim; the classifier is only how the finding is recorded. +- *"Fixing" Python toward R's order.* R's order is an artifact of failing to + read dates the source file plainly contains. Sorting to match it would + discard correct information — the ticket-27 precedent inverted. +- *Ordering row-order ahead of wide-format in the registry.* Measured both + orderings against the real data: identical results (the wide-format prefix + heuristic matches nothing at the cleaned stage, and the raw stage's 5 + `wide_format_fragment_truncated` rows are unaffected either way). + Wide-format kept first anyway, since it is source-verified and + stage-specific while row-order is the loose one. +- *Treating the `product_balance` re-pairing as this ticket's problem.* + Investigating check 2 surfaced that `product_balance` does **not** travel + with its row (R has `(released 8.0, balance 314.0, LA-MH063)` where Python + has `(released 8.0, balance 168.0, LA-MH063)`). Traced to + `_compute_running_balance` (`src/a4d/clean/product.py` step 2.15): balance + is *derived* as `balance[i-1] - released[i] + received[i]` on both sides, + never carried from the source cell, so it cannot follow a row under a + re-sort by construction. That is ticket 21's known, unowned + `product_balance` residual rather than new data loss — handed to [ticket + 36](36-triage-product-cleaned-unclassified-residual.md) with the mechanism + recorded, not resolved here. + +**Evidence: executed.** Every count above comes from running against the real +drive data, not from reading the previous run's report: three ad-hoc analysis +passes over all 229 file pairs, one source-Excel read, and a full +`just compare-outputs` re-run confirming the before/after +(`output/comparison/2026-08-12T220324Z/`). The one *read* claim is that R's +rank algorithm matches Python's, inherited from ticket 21 rather than +re-derived from R's source here; it is not load-bearing — checks 1-3 stand +without it. + +**Tense.** All claims describe current behaviour of the code on disk and of +the frozen R baseline. The only change made is to the comparison tool's +classifier wiring; no pipeline code was touched, so no pipeline output moved. + +**Changes.** `scripts/compare_outputs.py`: `product_units_released` now maps +to `WIDE_FORMAT_FRAGMENT_CLASSIFIERS | PRODUCT_ROW_ORDER_CLASSIFIERS`. +`tests/test_migration/test_compare.py`: new `TestClassifiersByColumnWiring`, +written first and confirmed failing — one test pins that *every* product +column aligned through the positional row-ordinal key carries the row-order +classifier (so the same silent omission cannot recur on another column), one +pins this column's own classification. Full suite 606 passed / 1 skipped, +`ruff check`, `ruff format --check`, `ty check src/` all pass. diff --git a/docs/wayfinder/tickets/26-triage-product-column-divergence.md b/docs/wayfinder/tickets/26-triage-product-column-divergence.md new file mode 100644 index 0000000..1336a14 --- /dev/null +++ b/docs/wayfinder/tickets/26-triage-product-column-divergence.md @@ -0,0 +1,155 @@ +--- +id: 26 +title: Triage the product pipeline's column-existence and dtype divergence (Column divergence) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-12d +claimed_at: 2026-08-12 +resolution: decided +evidence: executed +closed_by: null +spawned_by: null +--- + +## Premise + +Rests on [Build and run the R/Python output comparison script, then triage +every flagged difference](15-build-and-run-comparison-script.md), closed: +`compare_columns` (`src/a4d/migration/compare.py`) — column existence +(only-in-R / only-in-Python) plus dtype mismatches on common columns — was +built as one of the comparison tool's four structural layers and has been +computed and shown in the CLI summary table (`Column divergence`) on every +`just compare-outputs` run since. It was never triaged: every subsequent +triage ticket ([17](17-fix-product-row-alignment-and-triage.md), +[18](18-triage-comparison-flagged-differences.md), +[20](20-fix-raw-entry-date-representation.md), +[21](21-triage-remaining-product-columns.md), +[22](22-triage-product-raw-columns.md)) worked exclusively off the +`cell_mismatches` sheet (per-value diffs), not this structural layer — a gap +against the destination's "every Python/R difference documented and +explicitly decided" bar, not a deliberate scope decision. + +Surfaced while closing ticket 21 (a user question about the CLI's `Column +divergence` count), on the current 248-tracker `output_r`/`output_python` +pair. Cleaned-stage `product_data_cleaned` shows 4 divergences in every one +of 229 files, plus a 5th in 103 of them: + +- `orig_product_released_to` present only in Python's output (229/229) — a + Python-only helper column (`clean/product.py`, holds the pre-cleaning + `product_released_to` value) that R never emits. +- `product_table_month`: R `Float64` vs Python `Int32` (229/229) +- `product_table_year`: R `Float64` vs Python `Int32` (229/229) +- `product_unit_capacity`: R `Float64` vs Python `Int32` (229/229) +- `product_remarks`: R `Boolean` vs Python `String` (103/229) — only in + files where the column is entirely null for that tracker; a likely + all-null-column dtype-inference artifact of R's parquet writer, not + wired into any normalizer. + +Raw-stage `product_data_raw` shows a smaller, different set (51 files +missing `product_returned_by`, 25 missing `product_units_returned`, plus a +handful of one-off `product_released_to`/`product_remarks`/ +`product_units_released` — all only-in-Python, no dtype mismatches), not +yet looked at either. Patient's own column divergence (if any) hasn't been +checked at all. + +## Question + +For both product stages (and patient, if it turns out to have any): is each +column-existence or dtype divergence a genuine content gap, or a +representation/schema artifact (following `normalize_date_column`/ +`normalize_numeric_column`/`normalize_whitespace_column`'s precedent) that +should be normalized or simply documented as expected? In particular: +confirm whether `product_table_month`/`product_table_year`/ +`product_unit_capacity`'s Float64-vs-Int32 split is purely R's lack of a +native integer type (harmless) or hides an actual value difference (e.g. +truncation/rounding); confirm `orig_product_released_to`'s absence from R is +expected (a Python-only helper column, not something R should ever have +had) rather than a naming/step gap; and root-cause the raw-stage +only-in-Python columns (`product_returned_by`, `product_units_returned`, +etc.) — are these real R-side columns R simply never populates for these +files, or a genuine extraction gap. Add findings to `compare_columns`' +report output (currently CLI-only, not written to any report sheet) if +that's the fix, or to the classifier/normalization layer if it's a +representation issue. + +## Resolution + +**Decision:** column-existence/dtype divergence now has a dedicated +`column_divergence` sheet (one row per file/column, `only in R` / `only in +Python` / `dtype mismatch` with both dtypes) written to every stage's Excel +report by `build_mismatch_rows()` (`src/a4d/migration/compare.py`) — closing +the "CLI-only, no report sheet" gap the premise named. `compare_columns()` +itself is deliberately left unchanged: an existing test +(`test_flags_dtype_mismatch_on_common_column`) already established that even +a value-equal Int64-vs-Float64 pair must be flagged, so the structural check +stays a strict, unfiltered signal — severity/cause judgment belongs in the +report and this ticket's findings, not in the check. + +Every divergence named in the premise, plus patient's (previously unchecked), +was root-caused directly against the real 248-tracker USB-drive R/Python +output pair (`output_r`/`output_python`) via `just compare-outputs`, and spot +verified against real source Excel: + +- `product_table_month`/`product_table_year`/`product_unit_capacity` + (product cleaned, R `Float64` vs Python `Int32`, 229/229 files): confirmed + harmless — 0 non-integer values across all 198,060 R-side values checked. + R's own `script3_create_table_product_data.R` declares these `integer`, + but its error-substitution path assigns the *double* `ERROR_VAL_NUMERIC` + (999999) into the integer branch on any coercion failure, which silently + widens the whole column to double in R even when no error actually fires + on this data — a pure representation artifact, not a value difference. +- `orig_product_released_to` (product cleaned, only-in-Python, 229/229): + confirmed expected — 0/229 R files ever have it; it's a Python-only helper + column (`clean/product.py`) R was never going to emit, not a gap. +- `product_remarks` (product cleaned, R `Boolean` vs Python `String`, + 103/229 files): confirmed an all-null-column dtype-inference artifact — + every affected file's R-side column is 100% null; R's parquet writer + infers `Boolean` for an entirely-empty column instead of the declared + `character` type. +- `product_returned_by` (51 files) / `product_units_returned` (25 files) + (product raw, only-in-Python): confirmed a **genuine R extraction gap**, + not an artifact — spot-checked `2017_Mahosot Hospital A4D Tracker.xlsx`'s + `INV` sheet directly: its header row has a literal "Units Returned" + column, and Python's raw extraction holds 11 real non-null values for it + (range -1 to 2) that R's raw output drops entirely for this file. R is + wrong here, consistent with the destination's "R can be wrong" stance — + no Python fix needed, but it's now a documented, real divergence rather + than an assumed artifact. +- Patient cleaned stage: essentially clean — one single-file dtype mismatch + (`testing_frequency`, R `Float64` vs Python `Int32`), same harmless + integer-widening pattern as the product columns above. +- Patient raw stage: **did not converge** — 18,783 column_divergence rows + across 245 files, dominated by two large, distinct patterns neither + matching anything else on this map: hundreds of uniquely-numbered + only-in-R junk columns (`na`, `na1`, `na2`, ... `na10064`, one file + contributing 185 of them) that look like R's header-deduplication scheme + for blank/duplicate Excel headers, and a large only-in-Python set of + literal, unmapped source header text (`"Phone Number"`, `"Date"`, `"Home + Visit"`, etc.) that Python's raw extraction passes through verbatim where + R's apparently doesn't. Root-causing this needs its own session — split + into [ticket 30](30-triage-patient-raw-column-divergence.md) rather than + forcing convergence, per this map's "split rather than sprawl" rule. + +**Rejected:** normalizing the dtype-mismatch numeric/boolean cases away +inside `compare_columns()` itself (e.g. treating value-equal numeric dtypes +or all-null Boolean-vs-String as a non-divergence) — rejected because an +existing test explicitly pins the opposite policy (flag every dtype +difference, unfiltered), and folding a value-equality check into a purely +structural comparison would blur that boundary for no real benefit: these +five are now fully documented and explained instead, which satisfies the +destination's "every difference explicitly decided" bar without touching +established, tested behavior. + +**Evidence:** executed — every claim above was checked directly against the +real drive data (`output_r`/`output_python`, 248 trackers) or the real +source Excel (`2017_Mahosot Hospital A4D Tracker.xlsx`), not read from R +source alone. `src/a4d/migration/compare.py`'s R-source citation +(`script3_create_table_product_data.R`'s `ERROR_VAL_NUMERIC` coercion) is +read, used only to explain *why* the artifact happens, not to establish that +it does — the 0/198,060 non-integer check is what establishes that. + +**Tense:** all current-behaviour, from the current 248-tracker output pair. + +Full suite (584 passed, 1 skipped, incl. 1 new test), ruff, `ty check src/` +all pass. diff --git a/docs/wayfinder/tickets/27-triage-patient-raw-residual.md b/docs/wayfinder/tickets/27-triage-patient-raw-residual.md new file mode 100644 index 0000000..16e8433 --- /dev/null +++ b/docs/wayfinder/tickets/27-triage-patient-raw-residual.md @@ -0,0 +1,210 @@ +--- +id: 27 +title: Triage the residual patient raw-stage column mismatches after date normalization +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-12f +claimed_at: 2026-08-12 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 23 +--- + +## Premise + +Rests on [Triage every flagged R/Python difference for the patient +arm](23-triage-patient-arm.md), closed: the patient arm's raw-stage +mismatches were dominated (564,096 of the total, across 70 columns) by the +same representation artifact [ticket +20](20-fix-raw-entry-date-representation.md) found and fixed for product's +raw `product_entry_date` — R's raw extraction stores the unparsed source +text (an Excel serial for date-formatted cells) while Python's raw +extraction already ISO-formats parsed dates. Extending +`normalize_date_column` to every one of the patient cleaned schema's 18 +`pl.Date` columns (via the already-existing, schema-derived +`get_date_columns()` helper, wired into `scripts/compare_outputs.py`'s +`STAGES` table for the `Patient (raw)` stage only) confirmed the hypothesis: +raw-stage patient mismatches dropped from 564,096 to 46,788 (91.7%), with +every one of the 18 date columns individually collapsing by 99%+ (e.g. `dob` +79,836 -> 2). No pipeline code changed — this was a comparison-tool fix +only, mirroring ticket 20's precedent exactly. + +## Question + +Triage the remaining 46,788 raw-stage mismatches across 67 columns (per +`output/comparison/2026-08-12T092205Z/snapshot_patient_data_raw.json`), +the way ticket 18/20/21/22 did for product: look for systematic patterns, +fall back to the real source Excel trackers as the arbiter, and add named +causes to a new `PATIENT_*_CLASSIFIERS` registry in +`src/a4d/migration/compare.py` as patterns are confirmed (none exists yet +for patient). Two leads already surfaced, not yet chased to a cause: + +- `complication_screening` (12,566 mismatches, the largest residual column) + — a spot check on `2021_NPH A4D Tracker` found Python's raw value is a + comma-joined list of multiple selections (`"Dilated Eye + Examination,Foot Examination"`) where R's raw value holds only the first + selection (`"Dilated Eye Examination"`) — looks like a genuine multi-select + extraction gap on one side, not yet confirmed as systematic or judged + against the source Excel for which side is right. +- `meter_received_date` (1,761 mismatches) is a raw-only column absent from + the cleaned schema (so `get_date_columns()` didn't pick it up) but is + named like a date column — worth checking whether it needs the same + normalization treatment applied by hand, or has a different cause. + +This ticket is large (67 columns) — if it doesn't converge in one session, +split further rather than leaving it open-ended, per the pattern ticket 18 +established. + +## Resolution + +**Decision:** Fixed two comparison-tool representation gaps, made one real +pipeline change (extraction now preserves the source trackers' own Excel +formula-error strings; cleaning nulls and logs them), and named two causes. +Raw-stage patient mismatches went 46,788 -> 18,813 on the tool fixes alone, +then to 28,033 after the pipeline change -- the rise is Python becoming +*more* faithful than R, not a regression, and every one of those rows is +classified rather than unexplained (`unclassified` is unchanged at 14,981 +throughout). **Cleaned-stage output is byte-for-byte unchanged in both arms** +(patient 99,478; product 23,043; product raw 118), so no production data +moved. The remaining ~50 columns didn't converge and are split into [ticket +31](31-triage-patient-raw-residual-2.md). + +**Because:** +- `hba1c_updated`, `hba1c_baseline`, `height`, `weight`, `fbg_baseline_mg`, + `fbg_baseline_mmol`, `fbg_updated_mg`, `fbg_updated_mmol`, `bmi`, and + `t1d_diagnosis_age` were dominated by the same float-to-string + representation gap ticket 22 already fixed for product's raw numeric + columns (R's own float formatting rounds trailing digits differently, + e.g. `"8.800000000000001"` vs `"8.8"`). `PATIENT_RAW_NUMERIC_NORMALIZE_COLS` + (`scripts/compare_outputs.py`), derived from `get_numeric_columns()` + mirroring `PATIENT_RAW_DATE_NORMALIZE_COLS`'s own precedent rather than + hand-listed, wires `normalize_numeric_column` into the `Patient (raw)` + stage. `hba1c_updated` 5,928 -> 88, `hba1c_baseline` 5,748 -> 2, `height` + 1,366 -> 39, `weight` 856 -> ~0. +- `meter_received_date` (1,761 mismatches, the ticket's own second lead) is + a raw-only column absent from the cleaned schema, so `get_date_columns()` + never saw it; appended it by hand to `PATIENT_RAW_DATE_NORMALIZE_COLS` + (the one genuinely non-derivable list entry, since it has no cleaned-stage + counterpart to derive from). Collapsed to 0. +- `bmi` and `t1d_diagnosis_age` are formula-derived in the source trackers + (BMI from weight/height; age at diagnosis from DOB and diagnosis date). + Where a required input was never recorded -- verified against real source + Excel: `height` blank -> `#DIV/0!`, no `Date of T1D Diagnosis` -> `#NUM!`, + the cell's openpyxl `data_type` genuinely `'e'` -- the tracker's own + formula writes a literal error string into the cell. + + **This became a pipeline change, not just a classifier** (see Correction + below): extraction was silently nulling those strings, so the raw layer + misreported what the source file contained and the "source formula could + not compute this" signal was lost rather than recorded anywhere. + `clean_excel_errors` was removed from all four extraction call sites + (both arms) and replaced by `normalize_excel_formula_errors` + (`src/a4d/clean/converters.py`), which nulls them at the *cleaning* stage + and logs each under a new `source_formula_error` error code. `null` was + chosen deliberately over the `999999` numeric sentinel: the sentinel + means "a value was recorded but is invalid", whereas here no value could + be computed at all because a required input was never entered. + + `EXCEL_FORMULA_ERROR_CLASSIFIERS` (`excel_formula_error`) documents the + resulting raw-stage divergence for `bmi`, `t1d_diagnosis_age`, and `age`. + It matches in **both** directions, because R is the inconsistent side: + readxl guesses a column's type from its majority values, so the same + error cell survives as a string where the column is guessed character but + becomes `NA` where it is guessed numeric -- the same readxl type-guessing + mechanism `STRAY_DATE_CLASSIFIERS` documents for a different symptom. + Verified in both directions against real source Excel: Penang General + Hospital `Jun_26` (R keeps `#NUM!`) and Baguio General Hospital `Jun_26` + sheet `May26` (R drops a `#DIV/0!` that Python keeps). +- New systemic pattern, not in this ticket's original leads: several raw + date columns (`hba1c_updated_date`, `bmi_date`, `blood_pressure_updated`, + `fbg_updated_date`, `last_clinic_visit_date`, `hospitalisation_date`, and + four `complication_screening_*_date` columns) show R holding the sentinel + (`9999-09-09`) against Python holding a real date with an implausible + year (2567-2569). Verified directly against the real source Excel (06 + Nakornping Hospital A4D Tracker_Jun_26.xlsx, patient TH_NK004, + `hba1c_updated_date`, row 87): the cell is genuinely date-typed + (`dd-mmm-yyyy` number format) but holds year 2569 -- a clinician typed a + Thai Buddhist-Era year (BE = CE + 543) directly into a Gregorian date + field. R's raw extraction already rejects the implausible year and emits + the sentinel; Python's raw extraction is a faithful, by-design + pass-through of the cell's literal value. Confirmed **not** a pipeline + bug: the cleaned stage's own future-date guard (`_validate_dates`, + `clean/patient.py`) independently replaces the same cell with the + identical sentinel (verified against the real cleaned parquet for + TH_NK004 -- the June row already reads `9999-09-09`), so both pipelines + already agree from the cleaned stage onward regardless of this raw-stage + representation gap. New `PATIENT_BUDDHIST_ERA_CLASSIFIERS` + (`buddhist_era_typo`) reuses the product pipeline's own + `BUDDHIST_ERA_THRESHOLD` (2400) rather than inventing a new one. Explains + 369 of the 371 sentinel-vs-implausible-year rows found. +- Both new comparison-tool normalizations and both new classifiers are + covered by unit tests (`tests/test_migration/test_compare.py`); full + suite (599 passed, 1 skipped), ruff, `ty check src/` all pass. + +**Rejected:** Chasing `complication_screening` (12,566 mismatches, the +single largest residual column, already flagged in this ticket's premise as +a probable multi-select extraction gap) in this session -- it's a genuinely +different investigation (a content/extraction-logic question, not a +representation gap fixable by a normalization or a mechanical classifier) +and would have meant sprawling past the two leads this ticket was actually +scoped to chase. Left for [ticket 31](31-triage-patient-raw-residual-2.md). + +**Evidence:** executed -- the pipeline change was validated by a full +re-run of both arms (`a4d run patient|product --force`) against the real +248-tracker drive dataset, followed by a fresh `compare_outputs` run; +raw-stage output was confirmed to now carry `#NUM!`/`#DIV/0!` verbatim, +cleaned-stage output confirmed unchanged, and the new `source_formula_error` +log entries confirmed present across 145 tracker log files. Both formula-error +directions and the Buddhist-era finding were verified against real source +Excel cells (openpyxl `data_type`), not reasoned about. + +## Correction (same session, after the first close) + +This ticket was first closed treating the formula-error finding as a +comparison-tool classifier only, on the claim that Python "has no cached +value to carry" for an errored formula. **That was wrong on the mechanism +and wrong on the scope**, caught when the user asked what actually happens +downstream: + +- The mechanism is not openpyxl. `clean_excel_errors` (`extract/common.py`) + was deliberately nulling the error strings *during extraction*, by its own + design. openpyxl reads them faithfully. +- Downstream, R's cleaned output carries `999999` for these cells where + Python carries `null` -- checked across all 2,073 affected + `t1d_diagnosis_age` rows: 0 agree, 2,073 disagree. The first close had + asserted this was harmless without checking. +- Scoping the fix to `t1d_diagnosis_age` alone (on the grounds that `bmi`'s + cleaned value is recomputed from weight/height anyway, so nothing + downstream changes) was rejected by the user as optimizing the raw layer + for its current consumer: if extraction's job is to record what the source + cell contained, `#DIV/0!` in `bmi` is as real as `#NUM!` elsewhere. The + fix applies to every column in both arms. + +The user's design call: keep the error strings through extraction (raw layer +tells the truth), null them in cleaning (a calculation that could not run is +absent, not invalid -- so `null`, not the `999999` sentinel), and log them so +"the source formula failed" stays distinguishable from "the field was blank" +even though both end as `null`. + +**Tense:** current behaviour throughout -- every number above is from a +real comparison run against the real drive data (`output/comparison/2026-08-12T200529Z/`), +not a projection. + +## Addendum (ticket 24, closed same day) + +[Ticket 24](24-triage-remaining-raw-column-residual.md) confirmed a +possibly-systemic raw-extraction pattern it had flagged as a theory: R's +readxl infers a column's type from its majority values, so a lone Excel +date/time-formatted cell in an otherwise-numeric raw column gets coerced to +that column's type (the raw serial), while Python's openpyxl honors the +individual cell's own format and returns a `datetime`/`time` object — +verified against real source Excel cells (Penang General Hospital 2019 +Apr19!E36, a "Units Received" cell literally formatted `d/m/yy`). A reusable +classifier (`STRAY_DATE_CLASSIFIERS` / `_is_openpyxl_date_typed_stray_cell`, +`src/a4d/migration/compare.py`) now exists for this pattern — not yet +applied to any patient column. Worth checking whether any of this ticket's +67 residual columns show the same r_value-is-a-plausible-Excel-serial / +py_value-is-a-datetime-or-time-string shape before assuming a different +cause. diff --git a/docs/wayfinder/tickets/28-triage-patient-cleaned.md b/docs/wayfinder/tickets/28-triage-patient-cleaned.md new file mode 100644 index 0000000..7c9be51 --- /dev/null +++ b/docs/wayfinder/tickets/28-triage-patient-cleaned.md @@ -0,0 +1,128 @@ +--- +id: 28 +title: Triage the patient cleaned-stage column mismatches +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-12c +claimed_at: 2026-08-12 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 23 +--- + +## Premise + +Rests on [Triage every flagged R/Python difference for the patient +arm](23-triage-patient-arm.md), closed: that ticket's raw-stage date- +representation fix ([ticket 27](27-triage-patient-raw-residual.md) inherits +the residual) is raw-only and confirmed to leave the cleaned stage +completely unaffected (`patient_data_cleaned`'s snapshot is byte-identical +before and after the fix). The cleaned stage's 120,639 mismatches across 61 +columns are therefore a separate, untouched body of work — both sides +should already hold parsed dates and cleaned values there, so these are not +expected to be the same representation-artifact class as ticket 27's. + +## Question + +Triage the patient cleaned-stage mismatches (per +`output/comparison/2026-08-12T092205Z/snapshot_patient_data_cleaned.json`) +the way ticket 18/21 did for product's cleaned stage: look for systematic +patterns, fall back to the real source Excel trackers as the arbiter, and +add named causes to a `PATIENT_*_CLASSIFIERS` registry in +`src/a4d/migration/compare.py` (shared with, or separate from, whatever +[ticket 27](27-triage-patient-raw-residual.md) creates for the raw stage — +that's this ticket's call once both are in view). The largest columns are +`recruitment_date` (28,512), `t1d_diagnosis_age` (25,968), +`insulin_total_units` (16,985), `insulin_subtype` (15,724), and +`fbg_baseline_mg` (9,041) — none yet spot-checked against source. This +ticket is large (61 columns, and patient has more history/format variation +than product per ticket 23's own premise) — if it doesn't converge in one +session, split further rather than leaving it open-ended, per the pattern +ticket 18 established. + +## Resolution + +**Decision:** root-caused and resolved three of the five dominant columns; +the rest (56 smaller columns plus residuals on the three resolved ones) +didn't converge and split into [ticket 29](29-triage-patient-cleaned-residual.md), +per this ticket's own pre-authorization to split. + +**t1d_diagnosis_age (25,968 mismatches, the single largest column) was a +real Python bug, now fixed.** `_fix_t1d_diagnosis_age` +(`src/a4d/clean/patient.py`) unconditionally recomputed the value from +`dob`/`t1d_diagnosis_date`, discarding any real recorded age whenever either +date failed to parse -- its docstring's justification ("Matches R... like R +pipeline does") is false: R's own `fix_t1d_diagnosis_age` +(`script2_helper_patient_data_fix.R`) is dead code, never called from +`script2_process_patient_data.R` (the call site is commented out), so R +always just keeps the raw recorded value untouched. Confirmed via the +dominant mismatch pattern: 16,874 of 25,968 rows had R holding a plausible +1-20 age with Python null, plus 3,188 rows off by exactly 1 -- both point at +Python discarding/overriding real recorded data, not R being wrong. Fixed to +prefer the raw recorded value whenever present and not the Excel error +sentinel (999999), falling back to date-based calculation only when the raw +value is genuinely missing. Two new regression tests added +(`tests/test_clean/test_patient.py`); the existing sentinel-replacement test +still passes unchanged. Verified against the real 248-tracker drive data: a +full patient pipeline re-run (`a4d run patient --force`) and comparison +re-run confirm the column's mismatches dropped 25,968 -> 4,807 (81.5%), with +the full pipeline test suite (585 tests), ruff, and `ty check src/` all +passing. The 4,807 residual is unclassified and left for ticket 29. + +**recruitment_date (28,512) and insulin_subtype (15,724) are genuine, already- +intentional Python-correct divergences -- classified, not "fixed", since +there's nothing to fix.** `recruitment_date`: spot-checked directly against +the real source Excel (Quirino Memorial Medical Center, patient PH_QM001) -- +the tracker's own "Patient List" sheet plainly records +"Date of Recruitment" = 2025-12-01; Python extracts it correctly, R's static +extraction leaves it null. A new `r_extraction_gap` classifier +(`PATIENT_RECRUITMENT_DATE_CLASSIFIERS` in `src/a4d/migration/compare.py`) +explains 28,010/28,512 (98.2%) of this column's mismatches. +`insulin_subtype`: already documented as a deliberate, verified-correct +Python divergence in `_derive_insulin_fields`'s own docstring +(`src/a4d/clean/patient.py`) -- R's own allowed-values validator rejects the +multi-insulin CSV string R's own derivation logic produces for 2024+ +trackers, replacing it with the "Undefined" sentinel; Python's derivation is +correct and fixes a known R typo besides. A new +`r_validator_rejects_multivalue` classifier +(`PATIENT_INSULIN_SUBTYPE_CLASSIFIERS`) explains 15,656/15,724 (99.6%). +Neither classifier changes pipeline behavior -- both are compare-tool-only, +wired into `CLASSIFIERS_BY_COLUMN` in `scripts/compare_outputs.py`. Four new +unit tests added (`tests/test_migration/test_compare.py`, 91 passing). + +**insulin_total_units (16,985) and fbg_baseline_mg (9,041) were +investigated but did NOT converge to a confirmed root cause** -- both show +R null / Python real-value for the large majority of rows (16,985/16,985 +and 8,883/9,041 respectively), consistent with the same "Python fills an R +extraction/conversion gap" class as the two resolved columns above, but +unlike them this wasn't verified against the real source Excel or code +(`insulin_total_units` has no cleaning-stage transform at all on either +side, and its raw-stage comparison shows 0 mismatches, meaning whatever +causes the divergence happens during R's or Python's *type conversion* step, +not extraction -- not yet traced to a specific line). Left unclassified; +carried into ticket 29 rather than guessed at. + +**Rejected:** closing this ticket only when all 61 columns converge -- the +ticket's own premise pre-authorized splitting, and forcing convergence here +would mean either rushing the unconfirmed `insulin_total_units`/ +`fbg_baseline_mg` hypotheses into classifiers without source verification +(against this map's own verification standard) or leaving genuine progress +on the table waiting for the other 56 columns none of which were even +sampled yet. + +**Evidence:** executed -- the `t1d_diagnosis_age` bug fix was verified with +a real 248-tracker pipeline re-run and comparison re-run (not just unit +tests); the `recruitment_date` root cause was verified against the real +source Excel tracker directly, not inferred from the data pattern alone. +`insulin_subtype`'s root cause is `read`-tier evidence (confirmed via +existing code comments/docstring, not independently re-derived this +session) but is itself already load-bearing on a prior, executed +verification (the code comment cites a specific R typo and validator +behavior). `insulin_total_units`/`fbg_baseline_mg` have no classifier and +carry no evidence claim. + +**Tense:** all pipeline-code claims (the fix, the docstring's incorrect +justification, R's dead code) describe current committed behavior as of this +session, not a proposal. diff --git a/docs/wayfinder/tickets/29-triage-patient-cleaned-residual.md b/docs/wayfinder/tickets/29-triage-patient-cleaned-residual.md new file mode 100644 index 0000000..d162584 --- /dev/null +++ b/docs/wayfinder/tickets/29-triage-patient-cleaned-residual.md @@ -0,0 +1,275 @@ +--- +id: 29 +title: Triage the residual patient cleaned-stage column mismatches +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-13b +claimed_at: 2026-08-14T00:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 28 +--- + +## Premise + +Rests on [Triage the patient cleaned-stage column mismatches](28-triage-patient-cleaned.md), +closed: that ticket root-caused and resolved three of the five dominant +cleaned-stage columns (`t1d_diagnosis_age` -- a real Python bug, now fixed; +`recruitment_date` and `insulin_subtype` -- genuine Python-correct +divergences, now classified), dropping total cleaned-stage mismatches from +120,639 to 99,478. It explicitly split off the rest rather than force +convergence, per its own pre-authorization. + +## Question + +Triage what's left of the patient cleaned-stage mismatches (rerun `just +compare-outputs` against the current `output_r`/`output_python` on the USB +drive to get a fresh snapshot -- ticket 28's own run is now the baseline). +In priority order: + +1. **`insulin_total_units` (16,985) and `fbg_baseline_mg` (9,041)** -- + ticket 28 found both are dominated by R-null/Python-has-value (matching + the class already confirmed for `recruitment_date`), but didn't verify + against source or trace the actual conversion-step code path. Both have + no cleaning-stage transform on either side and 0 raw-stage mismatches, + meaning the divergence is introduced during *type conversion*, not + extraction -- worth checking whether R's `as.numeric()`-equivalent + conversion step fails more aggressively than Python's on some value + format (the parallel to `product_units_received`'s ticket-22 finding is + worth checking first). +2. **`t1d_diagnosis_age`'s residual (4,807, post-fix)** -- unclassified; + the fix only addressed the null-discard/off-by-a-few pattern, not + whatever's left. +3. **`recruitment_date`'s residual (502) and `insulin_subtype`'s residual + (68)** -- small, but worth a quick look since the dominant cause is + already known and these are what's left after removing it. +4. **The other 56 columns** (up to ~4,504 mismatches each: `blood_pressure_updated`, + `status`, `hospitalisation_date`, `hba1c_updated_date`, `fbg_updated_date`, + `insulin_regimen`, `insulin_type`, ...) -- none sampled yet. + +Same method as ticket 28: look for systematic patterns, fall back to the +real source Excel trackers as the arbiter (mount at `/Volumes/USB SanDisk +3.2Gen1 Media/a4d/`), and add named causes to +`PATIENT_*_CLASSIFIERS` registries in `src/a4d/migration/compare.py` +(`scripts/compare_outputs.py`'s `CLASSIFIERS_BY_COLUMN` wires them to +columns) -- or fix a real Python bug directly, as ticket 28 did for +`t1d_diagnosis_age`, when one turns up. This is likely to be large -- +split further rather than leaving it open-ended if it doesn't converge in +one session. + + +## Standing bar (added 2026-08-12g, applies to this ticket) + +Per the map's **triage means deciding, not labelling** preference: it is not +enough to explain a difference and name a cause. Each one must also carry an +explicit verdict on **whether Python is doing the right thing**, with the +evidence that establishes it (source Excel, or R's/Python's own code). Noting +"Python has A where R has B" and adding a classifier is not a decision in +favour of A. Where Python turns out to be wrong or to be losing information +the source file carried, fix the pipeline rather than labelling the symptom +— see [ticket 27](27-triage-patient-raw-residual.md), where exactly that +turned a labelling job into a real extraction fix. A cause genuinely +undecidable on available evidence is recorded as an open question, not +closed with a label. + + +## Resolution (session-2026-08-13b) + +**Decision.** The two priority columns are both R defects, not Python +divergences, and the 999999-sentinel family that runs through the whole +report is a deliberate and correct Python representation choice. Three +classifiers were added and two real Python bugs -- found while sampling the +long tail -- were fixed in the pipeline. Cleaned-stage mismatches fell +99,408 -> 95,490 and unclassified rows 55,670 -> 16,698 (-70%). What was +not chased is split into [ticket 37](37-triage-patient-cleaned-residual-2.md). + +**Because.** + +1. **`insulin_total_units` (16,985, all R-null) -- R never reads the column + at all.** R's `extract_patient_data` + (`r-archive/R/script1_helper_read_patient_data.R`) carries a hack for a + merged-cell artefact that can produce two "insulin regimen" columns: it + greps the source headers for the literal `Insulin` and drops every match + after the first. Since the 2024 tracker redesign the monthly sheet also + carries "TOTAL Insulin Units per day", so the grep matches two real, + unrelated columns and the hack deletes the second. The sibling insulin + columns survive only because their sub-headers read + "Pre-mixed"/"Short-acting" (the "Human Insulin" group header sits in a row + R does not merge here), and "Number of insulin injections" survives on a + lowercase "i" the case-sensitive grep misses. Measured: R's cleaned output + has **0 non-null values across all 81,859 rows**, and the column is absent + from R's raw parquet entirely -- not even as an unmapped passthrough. + Python has 18,341. Verified against the real source Excel (06 500 NPT + Children's Hospital, `Jan26` header row 93 col 27 "TOTAL Insulin Units ", + data rows 94-98 = 20, 48, 24, 30, 20 -- exactly Python's values). Python + is not diverging; it is the only side that reads the column. + Classifier: `r_insulin_dedup_drop`. + +2. **`fbg_baseline_mg` (9,041) and `fbg_baseline_mmol` (1,156) -- R's + Patient List join renames both sides on a name collision.** + `reading_patient_data` (`r-archive/R/script1_read_patient_data.R`) joins + the monthly sheets to the static "Patient List" sheet with + `suffix = c(".monthly", ".static")`. Where a column exists on both sheets + dplyr renames *both*, so the unsuffixed name disappears and R's cleaning + stage -- which reads `fbg_baseline_mg` -- finds nothing and leaves the + schema column null for the whole file. R avoids exactly this for + `hba1c_baseline` by dropping the monthly copy before the join; it never + does the same for the baseline FBG columns. Polars suffixes only the + right-hand frame, so Python keeps the monthly value under the base name. + Measured across the real drive data: 21 files carry + `fbg_baseline_mg.monthly` and 3 carry `fbg_baseline_mmol.monthly`, and + those files account for 8,883 of 9,041 and 1,101 of 1,156 mismatches + respectively. Across their ~10,000 rows the monthly and static copies are + identical or numerically equal in 92.4%, so the column Python keeps is + very nearly the one R lost. Classifier: `r_join_suffix_collision`. + +3. **The 999999 sentinel (8,085 rows across ~30 columns) -- R stamps a magic + number on a cell that never held a usable number; Python nulls it.** R has + no missing-value normalization before `as.numeric()`, so a clinician's + "-", "NA" or free text fails to parse and R's cleaning substitutes + `ERROR_VAL_NUMERIC`. `safe_convert_column` + (`src/a4d/clean/converters.py`) normalizes those markers to null *before* + conversion, deliberately. Checked exhaustively rather than sampled: every + one of the 8,085 rows was traced back to Python's own raw stage, and **not + one** had a clean number behind it -- 3,530 missing-value markers, 2,331 + Excel formula-error strings, 1,719 unparseable free text (e.g. "2 months" + as an age), 505 that could not be joined back to a raw row. Python loses + no data; the only thing R keeps is the fact of a failed parse, which + Python records in the error log instead of in the data. This alone covered + 4,247 of `t1d_diagnosis_age`'s residual -- item 2 of this ticket's own + question. Classifier: `r_numeric_error_sentinel`, wired from + `get_numeric_columns()` rather than hand-listed, since the sentinel is a + property of R's conversion step and not of any one column. + +4. **`insulin_regimen` (1,348) -- a real Python bug, now fixed.** R's + `extract_regimen` uses `sub(..., ignore.case = TRUE)`, which matches + without case and leaves a value none of its four patterns match exactly as + the source wrote it. Python emulated the case-insensitivity by calling + `.str.to_lowercase()` on the column first, which permanently rewrote every + unmatched value: "NPH" -> "nph", "Other" -> "other", "Glargine" -> + "glargine". Fixed by moving the flag into the patterns (`(?i)`). This was + a live data-quality regression in the production BigQuery output, not a + comparison artefact. 1,348 -> 41. + +5. **`status` (2,661) -- a config duplicate plus arbitrary dict ordering, now + deterministic.** `reference_data/data_cleaning.yaml` lists both + "Active - Remote" and "Active Remote" as allowed values for `status`, and + `sanitize_str` reduces both to `activeremote`. R's + `setNames`-list lookup returns the *first* entry; Python's dict + comprehension kept the *last*, so 2,611 rows emitted "Active Remote" where + R emitted "Active - Remote". Fixed to first-wins, which matches R and, + more importantly, stops the emitted spelling depending on the order the + config happens to list them in. Derived rather than assumed: this is the + **only** sanitize-colliding pair in the entire config, checked by + enumerating every `allowed_values` block. 2,661 -> 50. + +**Rejected.** + +- *Changing Python to take the Patient List (`.static`) copy of baseline FBG + instead of the monthly one.* R's dropping of monthly `hba1c_baseline` + hints its author thought the static sheet authoritative, but R never made + that choice for FBG, and there is no evidence the static copy is truer. + The two disagree on only 663 of ~10,000 rows, and on those the **source + contradicts itself** -- the Patient List and the monthly sheet record + different baselines for the same patient. Per this map's Notes that is a + legitimate terminal answer, not something a pipeline can resolve. Left as + an open question below rather than changed silently. +- *De-duplicating the `status` allowed-values list in + `reference_data/data_cleaning.yaml`.* That would decide which spelling is + canonical for production output, which is the user's call, and the config + is shared reference data. First-wins makes the behaviour deterministic + without making that decision. Raised for the user instead. +- *Normalizing the 999999 sentinel away inside `compare_cells`.* Follows + ticket 26's precedent: the comparison tool documents divergence, it does + not hide it. A classifier explains the rows; the count stays visible. +- *Forcing convergence on the remaining 16,698 rows.* They are a different + investigation -- overwhelmingly the date-column family and its four + distinct shapes -- and this ticket's own question pre-authorized a split. + +**Evidence: executed.** Every number above was measured against the real +248-tracker `output_r`/`output_python` pair on the USB drive, not reasoned +from the code. The two source-Excel checks (500 NPT's TOTAL Insulin Units +header and values; the monthly-vs-static FBG comparison) were read directly +out of the workbooks with openpyxl. Both pipeline fixes were verified by a +full `a4d run patient --force` re-run over all 248 trackers followed by a +fresh comparison run (`output/comparison/2026-08-13T221849Z`). Full suite +639 passed / 1 skipped, `ruff check`, `ruff format --check` all pass. The +one claim not fully closed: 505 of the 8,085 sentinel rows could not be +joined back to a raw row on `(patient_id, sheet_name)` and so were counted +as untraced rather than as confirmed. + +**Tense.** All statements about R describe the frozen 2025-11-14 baseline as +it exists. All statements about Python describe current behaviour *after* +this session's two fixes, except the "before" figures, which describe +behaviour as of the start of this session. + +**Open question for the user.** `reference_data/data_cleaning.yaml` lists +two spellings of the same patient status. Recommendation: delete +"Active Remote" and keep "Active - Remote", which is what both pipelines now +emit and what R has always written to production. The alternative -- +keeping "Active Remote" -- would change 2,611 rows of production output and +diverge from every historical R run. Not acted on: it is shared reference +data and the choice is the user's. + +### Addendum (same session): the canonical-label decision, settled + +The "open question for the user" above was answered in-session, and the +answer **reversed this ticket's original recommendation**. Checking the +source rather than the two pipelines showed the spellings split by tracker +generation, not at random: + +| Years | Spelling | Rows | Files | +|---|---|---|---| +| 2020-2023 | `Active - Remote` | 1,421 | 30 | +| 2024-2026 | `Active Remote` | 1,354 | 28 | + +Only `2024_Mahosot` mixes both, on a single row. The 2024 template redesign +introduced a `Lookup List` sheet defining the dropdown; 2022 and 2023 +trackers **have no such sheet at all**, and every tracker that has one +defines `Active`, **`Active Remote`**, `Active Monitoring`, `Query`, +`Lost Follow Up` -- no hyphen. So `Active Remote` is the spelling the +current template sanctions and `Active - Remote` is the retired one. Keeping +the hyphenated form (the original recommendation, made from R's behaviour +before the source was checked) would have written a two-years-dead spelling +into production. + +**Decision (the user's).** One canonical label per status, with known +aliases folded into it, and the canonical form **declared in config** rather +than implied by list order. + +`reference_data/validation_rules.yaml` -- a Python-only file; R reads +`data_cleaning.yaml`, so this touches nothing R depends on -- now reads: + +```yaml +status: + allowed_values: ["Active", "Active Remote", ...] # "Active - Remote" removed + aliases: + "Active Remote": # canonical label + - "Active - Remote" # the retired spellings it absorbs +``` + +`validate_allowed_values` gained an `aliases` parameter of that shape and now +**raises** when two `allowed_values` sanitize to the same key, or when +aliases are declared for a label that is not itself an allowed value. The +first-wins ordering hack this ticket originally introduced is gone: a silent +ambiguity is now a loud config error, which is what made the original bug +invisible. + +**Result, verified on a full 248-tracker re-run:** cleaned output carries one +label -- `Active Remote` 2,776, no `Active - Remote` at all. Against R's +frozen baseline this is a *deliberate* divergence of 2,611 rows (R collapses +everything onto the retired spelling via its own first-match lookup), so +cleaned-stage mismatches rise 95,490 -> 98,101 while **unclassified rows stay +at 16,698** -- the new rows are explained, not added to the backlog. The new +`python_canonical_label` classifier reads the alias map out of the same YAML +that produced the values, and its column wiring is derived from that config +too, so a future alias needs no code edit. The 50 residual `status` rows are +genuinely different statuses (e.g. R `Discontinued` vs Python `Active`) and +belong to [ticket 37](37-triage-patient-cleaned-residual-2.md). + +**Rejected:** reordering `allowed_values` so the canonical spelling sits +first. It gets the right answer today but encodes the decision as list +position -- insert order by another name -- which is exactly the fragility +that produced the original defect. diff --git a/docs/wayfinder/tickets/30-triage-patient-raw-column-divergence.md b/docs/wayfinder/tickets/30-triage-patient-raw-column-divergence.md new file mode 100644 index 0000000..72bcbf1 --- /dev/null +++ b/docs/wayfinder/tickets/30-triage-patient-raw-column-divergence.md @@ -0,0 +1,319 @@ +--- +id: 30 +title: Triage the patient pipeline's raw-stage column-existence divergence +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-15 +claimed_at: 2026-08-15T10:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 26 +--- + +## Premise + +Rests on [Triage the product pipeline's column-existence and dtype +divergence](26-triage-product-column-divergence.md), closed: that ticket +added a `column_divergence` sheet to every stage's comparison report and +triaged product's (both stages) and patient's cleaned-stage divergences +fully — patient cleaned stage is essentially clean (one single-file dtype +mismatch). Patient's raw stage did not converge and was split off here +rather than forced. + +On the current 248-tracker `output_r`/`output_python` pair +(`just compare-outputs` against the USB drive), patient raw stage's +`column_divergence` sheet has 18,783 rows across 245 common files — +two large, distinct patterns, neither yet root-caused: + +- Hundreds of uniquely-numbered only-in-R columns (`na`, `na1`, `na2`, ... + up to `na10064` in one file alone), which look like R's own + deduplication scheme for blank or duplicate Excel header cells (R + appends a numeric suffix to disambiguate repeated/empty column names on + read) rather than anything Python's extraction produces. +- A large only-in-Python set of literal, unmapped source header text + (`"Phone Number"`, `"Date"`, `"Home Visit"`, `"Home Visit 1"`, `"BGM + A4D"`, etc.) — apparent raw passthrough of Excel headers that didn't + match any entry in `reference_data/synonyms/synonyms_patient.yaml`, + present in Python's raw output but not R's. + +Unlike product raw's `product_returned_by`/`product_units_returned` +divergence (ticket 26, confirmed a genuine R extraction gap via direct +source-Excel spot check), neither pattern here has been spot-checked +against a real source file yet, and it isn't yet known whether they're +related to each other (e.g. R's blank-header dedup swallowing a header +Python matches to real text) or independent. + +## Question + +Root-cause both patterns against real source Excel and R's actual +extraction code (`r-archive/R/read_patient_data.R` or equivalent), the same +way ticket 26 did for product's raw-stage gap: for the `na`/`naN` columns, +confirm what R is actually doing on read (blank-header dedup, a fixed-width +header-row assumption, something else) and whether Python's raw extraction +is silently dropping equivalent columns or correctly has no equivalent to +drop; for the only-in-Python header-text columns, confirm whether these are +genuinely un-synonym-mapped columns Python passes through by design (raw +stage isn't schema-enforced) while R drops anything it can't map, or +whether R is failing to extract real data the same way it did for +product's "returned" columns. Decide, per file/column-family rather than +row-by-row given the volume: genuine content gap (needs a synonym-file +addition or an R-side note), or an expected raw-stage passthrough +difference (needs documenting only, per ticket 26's precedent for the +harmless cases). + + +## Standing bar (added 2026-08-12g, applies to this ticket) + +Per the map's **triage means deciding, not labelling** preference: it is not +enough to explain a difference and name a cause. Each one must also carry an +explicit verdict on **whether Python is doing the right thing**, with the +evidence that establishes it (source Excel, or R's/Python's own code). Noting +"Python has A where R has B" and adding a classifier is not a decision in +favour of A. Where Python turns out to be wrong or to be losing information +the source file carried, fix the pipeline rather than labelling the symptom +— see [ticket 27](27-triage-patient-raw-residual.md), where exactly that +turned a labelling job into a real extraction fix. A cause genuinely +undecidable on available evidence is recorded as an open question, not +closed with a label. + +## Premise update (session-2026-08-14, ticket 37's session) + +The tracker set is now 254 files and the frozen R baseline was renamed to +match a bulk `06 ... -> 2026_...` rename by the data analyst (see [ticket +37's addendum](37-triage-patient-cleaned-residual-2.md)). This ticket's +"18,783 rows across 245 common files" is measured against the old 248-file +set; the current figure is **18,235 rows across 245 files** +(`output/comparison/2026-08-14T200342Z`). The two patterns the Question +names are unchanged -- only the counts moved. + +## Resolution (session-2026-08-15) + +### Decision + +All 18,235 patient raw-stage `column_divergence` rows are accounted for. Two +structural causes explain 95%, both R-side artifacts with no Python defect +behind them; the residual is already-decided R defects plus columns that died +out before the current template. **One real Python data loss was found on the +way** -- headerless columns carrying data -- and it is now detected and +reported rather than silent. + +The two patterns the Question treats as possibly independent are one +phenomenon seen from two sides: the same source column mapped by one pipeline +and passed through under its raw name by the other. Both pipelines keep +unmapped columns (Python's `ColumnMapper` runs non-strict, +`reference/synonyms.py:185-217`; R's `harmonize_patient_data_columns` only +warns), so a mapping gap on either side shows up as a column "only in" the +other. + +Breakdown, measured against `output/comparison/2026-08-14T204031Z`: + +| Rows | Cause | Verdict | +|---|---|---| +| 17,132 | `r_blank_header_artifact` | R artifact; Python correct | +| 154 | `name_sanitization_only` | Representation only; both sides carry it | +| 703 | Columns last seen <= 2024 | Out of scope (latest-template rule) | +| 246 | R mapping gaps + new 2026 fields | R defects already decided, plus [ticket 41](41-decide-2026-new-patient-list-columns.md) | + +**`r_blank_header_artifact` (17,132 rows, 94%)** -- traced in R's own code and +confirmed by running R. `script1_helper_read_patient_data.R:113` calls +`make.names(header_cols, unique = TRUE)`; executed, +`make.names(c("Patient ID", NA, NA), unique = TRUE)` returns +`"Patient.ID" "NA." "NA..1"`, and `is.na()` is `FALSE` on all three. So line +116's `df_patient[, !is.na(colnames(df_patient))]` -- commented "delete +columns without a header" -- never fires, and line 131's `"NA" %in% names(...)` +misses too because the name is `"NA."`. `sanitize_str` then strips the dot, +yielding `na`, `na1`, ... `na10064`. Python drops blank-header columns in +`filter_valid_columns`, so it correctly has no equivalent. + +Measured, not assumed: across all 245 R raw parquets, 16,762 `na*` columns +exist, only 199 hold any value, and 189 of those are the trackers' unlabelled +row counter (1, 2, 3 ...). Only 10 hold real text. + +**`name_sanitization_only` (154 rows)** -- R stores the sanitized name where +Python's raw stage keeps the literal source header +(`instantmeterreceiveddate` vs `INSTANT Meter Received Date`). Both sides have +the column. + +**The 246 live rows** are: `insulin_total_units` (85, R's insulin-dedup grep -- +[ticket 29](29-triage-patient-cleaned-residual.md)), `recruitment_date` (59, +paired exactly with R's `xmlspacepreservedateofrecruitmentmmmyyyy` -- R's +header text carries an `xml:space="preserve"` XML artifact so R cannot map it), +`edu_occ`/`edu_occ_updated` (4, [ticket 37](37-triage-patient-cleaned-residual-2.md)), +the 2026 template's Thai-bilingual complication-screening/blood-pressure +headers (R fails to map them, Python maps them correctly), and the five new +2026 Patient List fields split into [ticket 41](41-decide-2026-new-patient-list-columns.md). + +### The real finding: headerless columns carrying data + +`filter_valid_columns` drops any column whose header cell is blank. Verified +against the source workbook: in `2021_Kantha Bopha Hospital A4D Tracker.xlsx` +sheets `Mar21` and `Apr21`, column Q holds real insulin-regimen values from row +87 down and its header cell (row 85) is **empty**, where the same column in +`May21` reads "Insulin Regime". Python therefore has `insulin_regimen` null for +all 194 rows. R keeps the values but only as junk-named `na1`, so R's +`insulin_regimen` is null too -- **both pipelines lose it**, and Python +irrecoverably. + +A sweep of all 254 trackers (executed) found **26 files, 4,572 values** in +headerless columns. Of those, 1,150 duplicate the column to their left, and a +further large share are the 2022 template's hidden merged-cell insulin column: +for `2022_Kantha Bopha`, measured across all 12 month sheets, the hidden column +never holds a value where the visible one is empty (0 of 1,932), agrees on +1,382 rows, and where it differs the **visible** column is richer +("Basal-bolus MDI (AN)" vs "Basal-Bolus"). R deletes this same column +deliberately (`script1_helper_read_patient_data.R:118-125`). Python losing it +is correct. + +`find_dropped_data_columns` (`src/a4d/extract/patient.py`) now reports each +site under the new `blank_header_with_data` error code, naming sheet, column +letter and value count. All-numeric columns are excluded -- measured, that is +the trackers' own row counter (189 of the 199 populated ones). This makes the +defect visible in the logs/errors table instead of silent, and feeds the +source-correction report split into [ticket 40](40-source-defect-findings-report.md). + +**And the data is recovered where the workbook itself settles the answer.** +`recover_blank_headers` names a headerless data column from the month sheets +that *do* label it -- a tracker's sheets share one layout, so the sibling +sheets are evidence, not inference. It abstains unless the answer is +unambiguous: the siblings must agree (2021 Putrajaya offers three candidate +names at one position, so those 12 values are left alone), a position no +sibling names is untouched (which is what puts the 2022 merged-cell column out +of reach), and a name the sheet already uses is refused -- 2017 Mahosot's +column S would otherwise collide with the `Estimated Testing Strips per month` +already sitting at column T, so its 97 values stay reported rather than +guessed. + +Measured over the full 254-tracker set: **17 sheet-sites across 5 trackers +recovered, 201 sites left reported**. Three of the five recovered headers map +to real schema columns -- `Insulin Regime` -> `insulin_regimen` (the 194 +Kantha Bopha rows, verified: that file's `insulin_regimen` goes 1,273 -> 1,467 +non-null), `Patient Observations` -> `observations` (7), `Clinic Visit` -> +`clinic_visit` (5). The other two (`foot`, `eye`, 13 values) have no synonym +entry and remain raw-stage passthrough, dropped at cleaning as before. + +### A tracker that changes shape mid-year is now flagged + +Set by the user this session: **a tracker is one workbook for one clinic-year +and should not change mid-year under normal circumstances; where it does, that +should be flagged.** `find_layout_changes` compares every month sheet's headers +position by position and reports each position whose *canonical* column is not +stable, under the new `tracker_layout_changed` error code. Positions where the +spelling varies but the canonical column does not (75 of them) log at info +only, since the synonym file already handles those. + +Verified against the real set: **37 trackers, 284 positions**. The output makes +the cascade legible -- `2018_CDA` flags columns P, Q, R and S in sequence, +which is one missing column in `Apr18` shifting everything right of it. + +This is the mechanism behind two of the defect classes [ticket +40](40-source-defect-findings-report.md) will report, and it found a live one: +`2020_CDA`'s Jan-May baseline FBG is currently dropped and its updated FBG +currently filed under the wrong unit, both from a single bad header edit. + +`collect_sheet_layouts` resolves every sheet's start row and headers once per +tracker, since recovery needs all sheets' headers before any one sheet is +final. Its results are handed back to `extract_patient_data` rather than +recomputed, which matters because `find_data_start_row` is the hot spot +[ticket 10](10-performance-profiling.md) rewrote: the first, naive version of +this change ran a second full column-A scan per sheet and cost **+74%** on a +serial 254-tracker extraction (63.5s -> 110.7s). Reusing the layout brings that +to **+15.6%** (63.5s -> 73.4s serial; the production arm runs parallel +workers). That residual is the honest price of the extra header pass. + +**Per the user (this session): the current tracker template is the golden +rule.** A column that appears in one year and is gone the next was very likely +a test that did not survive; the trackers are kept if it is ever wanted. So +703 residual rows across 68 columns last seen in 2024 or earlier +(`TSH`, `tTG-IgA`, `Home Visit 1`, `Age (year)`, `INSTANT Meter Received Date`) +are ruled out of scope rather than triaged. + +### Because + +The Question asked for a per-column-family decision rather than row-by-row +triage, and a named cause on 95% of the rows plus an explicit verdict on the +rest achieves that. Classifying rather than filtering follows [ticket +26](26-triage-product-column-divergence.md)'s precedent, which deliberately +left `compare_columns` unfiltered and documented the harmless cases. + +### Rejected + +- **Rejecting cross-sheet header recovery** -- this was the session's own first + conclusion and it was **wrong on the facts**, corrected when the user pushed + back. The stated objection was that recovery would import the 2022 template's + merged-cell duplicates; it cannot, because the rule only fires where a + sibling sheet supplies a header at that position, and the 2022 hidden column + is blank in *every* month sheet. Measured rather than argued: a sibling names + the column for 316 of the 4,572 values and nothing at all for the other + 4,256. The rule was then implemented (see below). Recorded here because the + reasoning, not just the conclusion, was the thing that failed: a plausible + objection was accepted without measuring it. +- **Recovering on a majority or nearest-sheet donor.** Rejected in favour of + requiring unanimity: 2021 Putrajaya offers three different candidate headers + at one position, which is a guess rather than a recovery. Unanimity drops + those 12 values and keeps the 7 that are unambiguous. +- **Deriving one header set per tracker from all its month sheets and applying + it to every sheet** (the user's proposal, by analogy with the meta schema). + Rejected on measurement, though the user was right about the dominant case. + **66 of 249 trackers (27%) have at least one position their month sheets + disagree on** (359 of 7,595 positions). Splitting those by whether the + canonical column actually changes: **75 are pure renames** -- every spelling + maps to the same column, exactly the short-to-descriptive drift the user + described, and the synonym file already absorbs them -- but **284 across 37 + trackers change meaning**, and those a unified positional set would get + silently wrong. Two verified against source: + + - `2018_CDA`: `Apr18` is 22 columns wide where the other eleven sheets are 23 + -- it has **no `Insulin Regimen` column at all**, so everything from + position 16 rightward sits one column left (Mar18 position 16 reads + `Insulin Regimen` with the value `Basal-bolus`; Apr18's reads + `BASAL Insulin Dose (IU)` with the value `18`, same patient row). A unified + header set would file that month's basal dose as the regimen, its bolus + dose as basal, and so on across seven columns. + - `2020_CDA`: position 13 changes from `Baseline FBG (mmol/dL)` to + `(mg/dL)` in June while **the values do not change** (median 233, range + 67-500 on both sides, i.e. mg/dL throughout -- mmol/L would be ~3-28). So + the header was edited over unchanged data, and `mmol/dL` is not even a real + unit. This one already costs data today, in two different ways: + `Baseline FBG (mmol/dL)` has **no synonym entry**, so Jan-May's baseline + FBG is dropped entirely, while `Updated FBG mmol/dL` *does* map, to + `fbg_updated_mmol` -- filing five months of mg/dL readings under mmol. + + The meta-schema analogy does not carry: the meta schema is a target columns + are mapped onto **by name**, whereas a unified positional set assumes + position -> meaning is stable within a tracker. This is also what makes the + unanimity guard load-bearing rather than cautious -- at `2018_CDA` position + 16 the donor set has two members, so recovery abstains -- and it never + overrides a header a sheet actually states. +- **Filtering `na*` out of `compare_columns`.** Would hide the 10 columns that + do hold real content, which is how the data loss was found. +- **Adding synonyms for the 68 pre-2025 unmapped headers.** Over-fits the + synonym file to columns that did not survive into the current template. + +### Evidence + +**Executed**: the cross-sheet recovery's effect and its cost (17 sites across 5 +trackers recovered, 201 left reported, `insulin_regimen` 1,273 -> 1,467 +non-null on the Kantha Bopha file, 63.5s -> 73.4s serial); the positional +agreement census behind rejecting a unified header set (66 of 249 trackers +conflict, split 75 renames / 284 meaning changes) and both source workbooks +behind it, including the 2020 CDA FBG value distributions either side of the +header edit; the `tracker_layout_changed` flag firing on 37 trackers / 284 +positions; the synonym-lookup check showing `Baseline FBG (mmol/dL)` maps to +nothing while `Updated FBG mmol/dL` maps to `fbg_updated_mmol`; R's +`make.names` behaviour (ran `Rscript`); every count in the +table above (computed from the real comparison report and the 254-tracker +drive data); the `na*` content census across 245 R parquets; the 2021 Kantha +Bopha source-Excel header inspection and the 194-row loss on both sides; the +full-tracker blank-header sweep; the 2022 Kantha Bopha hidden-column +measurement; the new detection firing on exactly `Mar21`/`Apr21` column Q, 97 +values each. Full suite 673 passed / 1 skipped, `ruff format --check`, +`ruff check`, `ty check src/` all pass. + +**Read**: R's extraction and harmonization code, Python's `ColumnMapper` +non-strict behaviour. + +**Tense**: every count describes current behaviour on the 254-tracker set. The +`blank_header_with_data` warning is new in this session and has not yet run in +production. diff --git a/docs/wayfinder/tickets/31-triage-patient-raw-residual-2.md b/docs/wayfinder/tickets/31-triage-patient-raw-residual-2.md new file mode 100644 index 0000000..20ff96b --- /dev/null +++ b/docs/wayfinder/tickets/31-triage-patient-raw-residual-2.md @@ -0,0 +1,247 @@ +--- +id: 31 +title: Triage the residual patient raw-stage column mismatches (round 2) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-15b +claimed_at: 2026-08-15 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 27 +--- + +## Premise + +Rests on [Triage the residual patient raw-stage column mismatches after date +normalization](27-triage-patient-raw-residual.md), closed: extending +`normalize_numeric_column` to patient's raw numeric columns (derived via +`get_numeric_columns()`) and `normalize_date_column` to the raw-only +`meter_received_date`, plus two new classifiers (`r_formula_error` for +Excel formula-error strings R's raw extraction carries through where +Python's correctly has no cached value; `buddhist_era_typo` for a +clinician-entered Thai Buddhist-Era year in a Gregorian date cell, +confirmed harmless by the cleaned stage's existing future-date guard) +together cut raw-stage patient mismatches from 46,788 to 18,813 (59.8%). +`complication_screening` (12,566, the single largest residual column, 84% +of what remains) was explicitly left unchased -- ticket 27 judged it a +different kind of investigation (content/extraction-logic, not a +representation gap) rather than sprawl past its own two original leads. + +## Question + +Triage the remaining 18,813 raw-stage mismatches across roughly 50 columns +(per `output/comparison/2026-08-12T200529Z/snapshot_patient_data_raw.json`), +continuing the pattern ticket 20/21/22/24/27 established: look for +systematic shapes, fall back to the real source Excel trackers as the +arbiter, add named causes to `PATIENT_*_CLASSIFIERS` registries in +`src/a4d/migration/compare.py` as patterns are confirmed. Current shape, +largest first: + +- `complication_screening` (12,566) -- ticket 27's own lead, not yet + chased to a cause: a spot check on `2021_NPH A4D Tracker` found Python's + raw value is a comma-joined list of multiple selections + (`"Dilated Eye Examination,Foot Examination"`) where R's raw value holds + only the first selection (`"Dilated Eye Examination"`) -- looks like a + genuine multi-select extraction gap on one side, not yet confirmed as + systematic across files or judged against the source Excel for which + side is right. +- `observations` (358) -- same comma-joined-list shape observed in ticket + 24/27's exploratory samples (e.g. `"Transfer to PKH,NA"` vs + `"Transfer to PKH"`); worth checking whether it's the same root cause as + `complication_screening` or a separate one. +- `latest_complication_screenning` (308), `fbg_baseline_mg.static` (192), + `fbg_baseline_mmol.static` (110) -- not yet looked at. +- A long tail of ~46 columns each under 100 mismatches, several showing a + swapped-adjacent-row shape in early sampling (`status`, `insulin_regimen`, + `age` each showed r/py values that looked like two rows' values traded + places) -- worth checking whether this is a genuine row-alignment issue + (patient's key is `patient_id` + `sheet_name`, confirmed sound by ticket + 15, but duplicate keys existed in 5/172 files at that time) rather than a + per-column content bug. +- `hba1c_updated` (88) and `fbg_updated_mg` (44) show a same-value + spacing-only variant in early sampling (`"8.8(20.9.16)"` vs + `"8.8 (20.9.16)"`) -- may be another readxl-vs-openpyxl whitespace + convention, similar in spirit to ticket 22's `normalize_whitespace_column` + but on a substring rather than the whole value. +- The remaining date columns' non-Buddhist-Era residual + (`last_clinic_visit_date` 89, `fbg_updated_date` 75, `hba1c_updated_date` + 39, `bmi_date` 43, etc.) -- not yet characterized. + +This ticket is large -- if it doesn't converge in one session, split further +rather than leaving it open-ended, per the pattern ticket 18/27 established. + + +## Standing bar (added 2026-08-12g, applies to this ticket) + +Per the map's **triage means deciding, not labelling** preference: it is not +enough to explain a difference and name a cause. Each one must also carry an +explicit verdict on **whether Python is doing the right thing**, with the +evidence that establishes it (source Excel, or R's/Python's own code). Noting +"Python has A where R has B" and adding a classifier is not a decision in +favour of A. Where Python turns out to be wrong or to be losing information +the source file carried, fix the pipeline rather than labelling the symptom +— see [ticket 27](27-triage-patient-raw-residual.md), where exactly that +turned a labelling job into a real extraction fix. A cause genuinely +undecidable on available evidence is recorded as an open question, not +closed with a label. + +## Premise update (session-2026-08-14, ticket 37's session) + +Two things moved under this ticket without changing what it asks: + +- **The tracker set is now 254 files, not 248**, and the frozen R baseline + was renamed to match a bulk `06 ... -> 2026_...` rename by the data analyst + (see [ticket 37's addendum](37-triage-patient-cleaned-residual-2.md)). The + numbers below supersede this ticket's original ones, and the baseline run + is `output/comparison/2026-08-14T200342Z`, not the 2026-08-12 snapshot the + Question cites. +- **Ticket 37's date-parser fixes reach this stage too.** The raw-stage + comparison normalizes both sides with `normalize_date_column`, which calls + the same `parse_date_flexible` ticket 37 fixed three times over (the broken + month-name truncation, the 2-digit-only month-year branch, and the new + longest-parseable-prefix fallback). So some of this ticket's residual has + already moved without anyone triaging it. + +Current raw-stage figures: **27,921 mismatches, 14,844 unclassified** (was +28,033 / 14,981). `complication_screening` remains the dominant single +column and this ticket's main question is unchanged. + +Also inherited from ticket 37: **13 cleaned-stage rows were found where both +pipelines hold the same two rows in a different order**, because +`patient_id` + `sheet_name` is duplicated for that patient in that sheet -- +the same "swapped adjacent row" shape this ticket's Question already lists in +its long tail. [Ticket 38](38-triage-patient-cleaned-date-family.md) now owns +deciding whether patient's row-alignment key needs product's positional +treatment; whatever it decides settles this ticket's version too, so do not +re-derive it here. + +## Premise update (session-2026-08-14b, ticket 38's session) + +**The row-alignment question this ticket deferred to [ticket +38](38-triage-patient-cleaned-date-family.md) is answered: patient's key stays +as it is.** Measured across all 254 cleaned files, 84 of 85,325 rows (0.10%), +in 7 files, sit on a duplicated `patient_id` + `sheet_name`. Product's key was +broken in a different league (up to 35-way duplication) before ticket 17 gave +it `add_row_ordinal`. Do not re-derive this; the "swapped adjacent row" shape +in this ticket's long tail is that bounded 84-row population. + +**Two of ticket 38's changes move this stage's numbers without anyone having +triaged them.** The raw-stage comparison normalizes both sides with +`normalize_date_column`, which calls `parse_date_flexible` -- and ticket 38 +widened its missing-marker handling (`MISSING_VALUE_MARKERS`, +`DATE_ABSENCE_MARKERS`, `DATE_PLACEHOLDER_FRAGMENTS`), so cells recording "-", +"Nil" or the template's placeholder text now normalize to null on both sides +rather than to the error sentinel. Current raw-stage figures: **27,893 +mismatches, 14,816 unclassified** (was 27,921 / 14,844); the baseline run is +`output/comparison/2026-08-14T204031Z`. `complication_screening` remains the +dominant single column and this ticket's main question is unchanged. + +## Resolution (session-2026-08-15b) + +### Decision + +**The dominant column is resolved, and it was a bug-fixing session, not a +labelling one.** `complication_screening` (12,566 mismatches, 84% of the +residual) had two entirely separate causes stacked on top of each other, and +the ticket's own headline lead was neither of them. + +**Cause 1 -- a real Python data loss, now fixed.** `ColumnMapper.rename_columns` +(`src/a4d/reference/synonyms.py`) handled several source columns mapping to one +canonical name by keeping the first and dropping the rest, under a comment +calling it "an edge case from discontinued 2023 format". It is not an edge +case: the 2023 template splits complication screening across B.P./Kidney/Eye/ +Foot/Lipids sub-columns that all map to `complication_screening`, and each +carries an independent value. Python kept B.P. and silently discarded the rest. +Measured by instrumenting the real extraction over all 254 trackers: +**2,489 recorded values dropped across 27 trackers** (2,457 +`complication_screening`, 32 `observations`). Concretely, `KH_KB023` in +`2023 Kantha Bopha`'s `Jan'23` has `JAN` in the Kidney column and nothing +else, and Python emitted `null` for that patient-month. + +The fix merges the group -- comma-joining non-empty values in column order -- +which is what R's `tidyr::unite()` does and what `merge_duplicate_columns_data` +in the same pipeline already does for repeated *raw headers*. The +keep-first branch's misleading `invalid_tracker` warning is replaced by a +`duplicate_source_columns` warning naming the group, so a new duplicate group +is visible rather than silent (ticket 30's `blank_header_with_data` precedent). +Verified against the real files: `KH_KB023` now carries `JAN`. + +**Cause 2 -- an R rendering artifact, correctly a classifier.** R's +`tidyr::unite(sep = ",")` leaves `na.rm` at its `FALSE` default, so every empty +cell in the group becomes the literal string `NA`: a patient screened in +January reads `JAN,NA,NA,NA,NA` and an unscreened patient reads +`NA,NA,NA,NA,NA` rather than being null. 99.3% of the column's mismatches carry +an `NA` token. This is R's rendering and carries no information -- +source-verified against `2023 Kantha Bopha`'s `Jan'23!AB98-AF98`, where the +padded sub-columns are genuinely empty in the workbook. **Python is the correct +side.** New `r_na_unite_padding` classifier +(`PATIENT_NA_UNITE_PADDING_CLASSIFIERS`, `src/a4d/migration/compare.py`), wired +to the three canonical columns that form a duplicate group anywhere in the +254-tracker set -- derived by measurement, not hand-listed. It fires only when +stripping the `NA` tokens leaves exactly Python's value, so a real disagreement +inside a group stays unclassified. + +**The ticket's headline lead was a 2-row phenomenon.** Ticket 27 flagged +`2021_NPH`'s "Python has a comma-joined multi-select where R has only the +first selection" as the likely systematic cause. Measured: **2 rows out of +12,566.** Do not re-propose it as the explanation for this column. + +### Numbers (real 254-tracker drive data, baseline run `2026-08-15T214447Z`) + +- Patient raw-stage **unclassified: 14,903 -> 1,879 (-87%)**. +- Total raw-stage mismatches 28,092 -> 27,980; the count barely moves because + cause 1 recovers data on Python's side while cause 2's mismatches remain + mismatches until classified. The two must not be judged by the same number. +- `complication_screening` 12,566 -> 2 unclassified; + `latest_complication_screenning` 308 -> 0; `observations` 325 -> 61. +- Full suite 696 passed, 1 skipped; ruff and `ty check src/` clean. + +### Rejected + +- **Coalesce instead of concatenate** (take the first populated value per row). + Killed on measurement: 1,393 rows have two or more populated sub-columns, so + coalescing would still lose data. +- **A declared config list of mergeable canonical columns.** Rejected as a + hand-maintained list that would drift; the merge rule is general, and the + measurement showing only three targets ever collide is what makes it safe. +- **Keeping keep-first for identity columns as a guard.** No identity or scalar + column forms a duplicate group anywhere in the 254 real trackers -- only + `complication_screening`, `observations` and `latest_complication_screenning` + (the last never populated). The `patient_id` scenario in + `test_harmonize_patient_data_columns_multiple_synonyms` is synthetic, and its + stated rationale ("This matches R behavior") was factually wrong: R unites. + The test was updated rather than kept. What this gives up: if a future tracker + ever populates two spellings of an identity column, they will be concatenated + rather than one silently chosen. The new `duplicate_source_columns` warning is + what makes that visible if it happens. +- **Chasing the long tail in the same session.** ~50 columns, largest 235, at + least three distinct shapes -- a different investigation. Split out rather + than sprawled, per the pattern tickets 18/27 set. + +### Evidence + +**Executed** for every load-bearing claim: the 2,489-value loss and the +1,393-multi-populated-row count come from instrumenting `read_all_patient_sheets` +over all 254 real trackers; the mechanism was read directly from +`r-archive/R/script1_read_patient_data.R` (`tidyr::unite`) and +`src/a4d/reference/synonyms.py`; the `KH_KB023` case was read cell-by-cell from +the real source workbook and re-checked in Python's output after the fix; all +before/after counts come from real pipeline and comparison runs against the +drive, not from unit tests. + +**Tense:** every number above describes current behaviour after this session's +commits, except the 2,489-value loss and the pre-fix per-column counts, which +describe the behaviour this session removed. + +### Not fixed, deliberately + +The 1,879 remaining unclassified rows across ~50 columns, spawned as [ticket +43](43-triage-patient-raw-residual-3.md). Three shapes are already visible and +recorded there. In particular `insulin_regimen` (235) was **not** wired to +`r_extraction_gap` even though its shape matches: 194 of its rows are ticket +30's source-verified blank-header recovery, but the other 41 are not verified, +and firing a whole-column classifier on an R-null/Python-present shape would +label them without deciding them -- exactly what this map's standing bar +forbids. diff --git a/docs/wayfinder/tickets/32-audit-classifiers-against-decision-bar.md b/docs/wayfinder/tickets/32-audit-classifiers-against-decision-bar.md new file mode 100644 index 0000000..21c662a --- /dev/null +++ b/docs/wayfinder/tickets/32-audit-classifiers-against-decision-bar.md @@ -0,0 +1,272 @@ +--- +id: 32 +title: Re-audit every existing cause classifier — is Python actually right, or was the diff merely labelled? +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-22 +claimed_at: 2026-08-22 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 27 +--- + +## Premise + +Rests on [Triage the residual patient raw-stage column mismatches after date +normalization](27-triage-patient-raw-residual.md), closed: that ticket was +first closed treating a formula-error divergence as a comparison-tool +labelling job, on an unverified claim that both sides were "just +representing no-value differently". Checking downstream showed R and Python +disagreed in 2,073 of 2,073 cleaned rows, and the real mechanism turned out +to be Python's own extraction silently discarding data the source file +carried — a pipeline bug the classifier would have cemented as "explained". + +That prompted the map's standing preference **triage means deciding, not +labelling** (see Notes): every flagged difference must both (1) be explained +by a mechanism traced to source Excel or to R's/Python's own code, and (2) +carry an explicit decision on whether Python is doing the right thing. +Naming a cause is not a decision in favour of Python's value. + +Every classifier now in `src/a4d/migration/compare.py` was written *before* +that bar was stated, across tickets 15, 18, 20, 21, 22, 24, 26, 27 and 28. +Some clear it comfortably (the mechanism was traced to R's source or to a +real source-Excel cell); others were shape-matching heuristics seeded from +the parity-presentation PDF and never verified against anything. + +## Question + +Audit all nine classifier registries against the two-bar standard, and for +each one record: what the mechanism actually is, what evidence establishes +it, and the explicit verdict on whether Python is correct. Where Python is +wrong or lossy, fix the pipeline (ticket 27's precedent) rather than +keeping the label. + +**Known weak, in priority order — these are the reason this ticket exists:** + +- `off_by_one_day` (`PRODUCT_ENTRY_DATE_CLASSIFIERS`) — the clearest + failure of the bar: it matches any pair of dates one day apart and says + nothing about which side is right. A one-day gap is exactly the shape a + real timezone/serial-origin bug would take. Never traced to a mechanism, + never checked against source Excel. 10 cleaned + 2 raw rows currently. +- `r_value_missing` (same registry, **11,468 cleaned product rows** — the + single largest classified cause on the map) — deliberately lumps together + "R rescued a source typo" and "R plainly failed to extract a clean value", + because `classify()` only sees the parsed value pair. Ticket 18 verified + *one* instance against real source Excel (2018 Mahosot, Jan18) and + generalized from it. At this volume that generalization needs testing, and + the two sub-causes need separating. +- `sentinel_null` and `ce_typo` (same registry) — both seeded from the + parity-presentation PDF's four named causes rather than from evidence; + the PDF's own baseline is now known stale (ticket 18). Neither has been + re-checked against the current 248-tracker data. +- `row_order_divergence` (`PRODUCT_ROW_ORDER_CLASSIFIERS`) — the mechanism + *is* established (R falls back to input order when its date parse fails; + Python sorts chronologically) and Python is very likely right, but the + "is Python correct" half rests on a 98.3%-of-groups end-balance spot + check, and `product_balance` is knowingly only 27.9% detected, left as + "future work". Finish the verdict or state the residual as an open + question. + +**Likely fine, confirm briefly rather than re-derive:** +`r_category_lookup_miss` (traced to `read_product_data.R`'s unnormalized +join), `openpyxl_date_typed_stray_cell` (source-Excel verified), +`wide_format_fragment_truncated` (R's value verified a strict prefix of +Python's), `r_extraction_gap` (source-Excel verified), +`r_validator_rejects_multivalue` (documented R validator bug), +`buddhist_era_typo` (source-Excel verified + cleaned-stage reconciliation +checked), `excel_formula_error` (ticket 27, both directions verified). + +If this doesn't converge in one session, split by registry rather than +leaving it open-ended. + +## Addendum (session-2026-08-12h, from [ticket 25](25-triage-product-units-released-cleaned.md)) + +`row_order_divergence`'s "is Python correct" half is now substantially +stronger than the 98.3% end-balance spot check this ticket was written +against — don't re-derive it, extend it. Ticket 25 established, against the +real 248-tracker drive pair: + +- Per-column multiset equality for `product_units_released` across **all** + 2,283 `(clinic_id, product_sheet_name)` groups (0 differing) — the + divergence provably adds, loses and alters nothing. +- Row-identity preservation: the multiset of + `(product_units_released, product_released_to)` pairs is identical across + all 11,649 `(clinic_id, product_sheet_name, product)` groups. +- Direct source-Excel confirmation that R's fallback trigger is real: in the + worst-affected group, R has `product_entry_date` null on every row while + the source file's "Entry Date" column is fully populated with genuine + datetimes. + +Two caveats this ticket should still carry: + +1. The `row_order_candidate` flag itself remains a **loose** membership test + ("R's value appears somewhere in Python's group"), which false-fires on + repeated small numeric values. It is adequate as a *record* of a cause + established by other means, but it is not evidence on its own — ticket 25 + deliberately did not close on it. +2. `product_balance`'s under-detection is now **explained**, not just + observed: balance is *derived* (`_compute_running_balance`, step 2.15, + `balance[i-1] - released[i] + received[i]`, mirrored in R), so it cannot + travel with a row under a re-sort and value-membership cannot detect it in + principle. That residual now has its own home in [ticket + 36](36-triage-product-cleaned-unclassified-residual.md); this ticket's + `row_order_divergence` entry can point there instead of holding it open. + +## Inherited from session-2026-08-14 (ticket 37) + +This ticket also now inherits `r_ifelse_na_propagation` (new), plus three +changes to existing causes: + +- `PATIENT_RECRUITMENT_DATE_CLASSIFIERS` was renamed + `PATIENT_R_EXTRACTION_GAP_CLASSIFIERS` and now serves three columns, its + single `r_extraction_gap` cause documenting three separately-verified R + mechanisms. That lumping is deliberate -- `classify()` cannot see the file, + so it cannot tell a 2022 tracker from a 2026 one -- but it is exactly the + kind of generalization this ticket exists to re-check. +- `buddhist_era_typo` is now symmetric (either side may hold the sentinel). +- `python_future_date_sentinel` is now wired to a patient column as well as + product ones. + +All were verified against the real drive data or the source Excel when +written, per the map's Notes. + +## Premise update (session-2026-08-15b, ticket 31's session) + +This audit now also inherits **`r_na_unite_padding`** +(`PATIENT_NA_UNITE_PADDING_CLASSIFIERS`, `src/a4d/migration/compare.py`), added +by [ticket 31](31-triage-patient-raw-residual-2.md). It is source-verified +against the real 2023 Kantha Bopha workbook rather than sampled, and it is +deliberately narrow: it fires only when stripping R's literal `NA` tokens +leaves exactly Python's value. Ticket 31 is also worth reading as a worked +example of what this audit is looking for -- the column it covers had a real +Python data-loss bug sitting underneath a classifiable R artifact, and +classifying first would have cemented the bug as "explained". + +## Resolution (session-2026-08-22) + +**Decision.** The audit is scoped to the classifiers written *before* the +two-bar standard, and the four the ticket named as known-weak are resolved: +`off_by_one_day` and `ce_typo` are **deleted** as shape-matching labels that +named no mechanism, `sentinel_null` is **renamed** to the mechanism it actually +describes, and `r_value_missing` is **bounded** so it can no longer absorb cells +where Python is the questionable side. Two pipeline defects found underneath +them are fixed. The remaining eight pre-bar causes are split into +[the second half of the audit](60-audit-remaining-pre-bar-classifiers.md), per +this ticket's own "split by registry rather than leaving it open-ended". + +**Because.** The ticket's premise -- "every classifier now in `compare.py` was +written before that bar was stated" -- was **false when this session opened**, +and measuring it is what set the scope. Dating all 50 causes by first commit +(`git log -S` over `compare.py`) puts **12** before the bar (2026-08-11/12) and +**38** after it, each of the 38 argued in its own triage ticket against real +data. Auditing all 50 would have re-derived a week of reviewed work; auditing +the 12 is the ticket's actual subject. + +Findings, each traced rather than pattern-matched: + +- **`off_by_one_day` (12 rows) named a coincidence.** Its 10 cleaned-stage rows + are consecutive daily entries in VNCH's 2022 and PKH's 2025 trackers. Pulling + the whole `(VNC, May22)` group shows R holding null entry dates at ordinals + 42, 47 and 51 -- its sort falls back to input order -- so its rows sit one + position behind Python's chronological sort and the "one day apart" is just + what neighbouring rows of a daily sequence look like. This is + `row_order_divergence`; `off_by_one_day` only won because + `PRODUCT_ENTRY_DATE_CLASSIFIERS` merges ahead of `PRODUCT_ROW_ORDER_CLASSIFIERS`. + The 2 raw-stage rows are a real and different mechanism: source serials `25` + and `12`, where openpyxl reproduces Excel's phantom 1900-02-29 (serial 25 -> + 1900-01-25) and `normalize_date_column`'s epoch arithmetic does not + (1900-01-24). Named `excel_1900_leap_serial`, bounded to serials below 61. +- **`ce_typo` (2 rows) fired on the comparison tool's own sentinel and hid a + production defect.** It tested `r_value.year > 2100`; both rows have R at + 9999-09-09, which is what `normalize_date_column` produces when R's raw + string will not parse. The source cells are junk Excel serials -- 1,339,576 + and 411,384 -- and **Python was publishing them as `5567-08-19` and + `3026-04-30` in cleaned output**, because `_validate_entry_dates` exempted + every year >= 2400 so Buddhist-era dates could flow through. Replaced by + `python_absurd_excel_serial`, which judges the *Python* side and excludes a + genuine BE year. +- **`sentinel_null` (78 rows) was a shape, but Python is right.** Traced to + source: Sarawak's 2023 tracker genuinely carries December **2024** dates in + its `Dec23` sheet, so Python's beyond-tracker-year guard fires while R's + cleaned side is null for the reason `r_value_missing` already describes. + Renamed `python_sentinel_r_extraction_gap` -- the intersection of two + already-decided mechanisms, not a third one. +- **`r_value_missing` (11,468 rows) holds for 11,436 and absorbed 32 it should + not have.** Filtering the population to rows whose Python date is implausible + for its own `product_table_year` found 29 year-floor cells -- `0202-06-20` + (Surat Thani), `0205-12-02` (QMC), `1935-04-30`, seven `2009-12-04` in a 2019 + Mahosot tracker -- which `_validate_entry_dates` logs and deliberately + preserves, plus the 3 era cells above. The cause asserts Python read the cell + correctly, so it may not cover these: `python_out_of_window_date_preserved` + now names them ahead of it. + +**Two pipeline fixes, both measured against the real 254-tracker data:** + +1. `_validate_entry_dates` (`clean/product.py`) no longer exempts everything + above 2400. The exemption is now the tracker's own BE band + (`product_table_year + 543`, minus `YEAR_FLOOR_DELTA`), so all 22 genuine BE + dates from the Thai clinics survive untouched while `5567-08-19`, + `3026-04-30` and Hat Yai's `2525-10-02` (BE for 2025 is 2568) are + sentinelled and logged under a new `implausible_era_date` error code. Cleaned + product rows with a year >= 2400 that is not the sentinel: **25 -> 22**. Total + rows unchanged at 74,689. +2. **`a4d run` -- the production entry point behind the Cloud Run Job -- was + publishing an errors table containing the patient arm only.** The patient arm + writes the table from inside `run_patient_pipeline`; nothing wrote the product + arm's, so `table_errors.parquet` and BigQuery's `errors` table have been + patient-only. Measured: 63,295 records, zero `balance_reconciliation` (116 + exist) and zero `implausible_era_date`. `run` now writes the table once after + both arms: **63,295 -> 97,326 records**. Found because the fix in (1) emitted + records that never arrived. + +**Rejected.** +- *Auditing all 50 causes* -- the premise that made that the job is false; the + 38 post-bar causes were each verified when written. +- *Converting Buddhist-era dates to Gregorian* -- the pipeline publishes + `2567-11-11` for a November 2024 transaction, which downstream reads as a + stock movement 543 years out. Correct in principle and deliberately **not** + decided here: it moves 22 rows of real data and R does not do it. Spawned as + [ticket 61](61-decide-buddhist-era-date-conversion.md). +- *Leaving the >= 2400 exemption alone and only bounding the classifier* (the + option B put to the user) -- rejected by the user, who asked for the source + and pipeline to be addressed rather than the comparison alone. +- *Fixing `parse_date_flexible`'s serial epoch to match openpyxl* -- it is + pipeline code, and the cleaned stage already nulls tiny-int residue, so the + change would move production output to fix two raw-stage comparison rows. +- *`product_balance`'s 11 unclassified rows* -- untouched; they belong to + [ticket 36](36-triage-product-cleaned-unclassified-residual.md), which already + argued them as a real signal worth surfacing. + +**Evidence: executed.** All counts come from a full pipeline run over the real +254-tracker set on the USB drive and a full `just compare-outputs` against the +frozen `output_r/`, not from reasoning. Cause deltas (previous run -> this run), +product cleaned: `off_by_one_day` 10 -> 0, `sentinel_null` 78 -> 0, +`python_sentinel_r_extraction_gap` 0 -> 81, `python_out_of_window_date_preserved` +0 -> 29, `r_value_missing` 11,468 -> 11,436, `row_order_divergence` 113 -> 121. +Product raw: `ce_typo` 2 -> 0, `off_by_one_day` 2 -> 0, +`excel_1900_leap_serial` 0 -> 2, `python_absurd_excel_serial` 0 -> 2. Both +patient stages are byte-identical (114,284 and 26,171 unchanged), which is the +check that this session moved nothing it did not intend to. Full suite 913 +passed, ruff and `ty check src/` clean. + +One measurement scare worth recording: an intermediate run showed six patient +`fbg_baseline_mg` cells losing their `python_glucose_unit_corrected` label. That +was not a regression -- it was defect (2) biting the measurement itself. Running +the product arm alone had overwritten the shared errors table, and +`load_glucose_unit_swaps` reads that table. It disappeared once both arms wrote +the table together. + +**Tense.** Every count above describes current behaviour on this branch after +the two fixes, except the "was publishing" statements about the >= 2400 +exemption and the patient-only errors table, which describe behaviour before +this session. + +**Residual, deliberately left open.** Product cleaned `unclassified` moves 20 -> +21 (`product_entry_date` 9 -> 10): the PKH row that `off_by_one_day` used to +cover has no mechanism yet, and it sits with a group of rows where R holds a +date a year ahead of Python's (`2028-02-26` vs `2026-02-26` at 2026 NOGH, +`2022-12-12` vs `2021-12-30` at 2021 VNCH). Naming that population is +[ticket 60](60-audit-remaining-pre-bar-classifiers.md)'s to pick up; it is a +question stated, not a label applied. diff --git a/docs/wayfinder/tickets/33-fix-red-ci-ruff-format-markdown.md b/docs/wayfinder/tickets/33-fix-red-ci-ruff-format-markdown.md new file mode 100644 index 0000000..3de98e6 --- /dev/null +++ b/docs/wayfinder/tickets/33-fix-red-ci-ruff-format-markdown.md @@ -0,0 +1,128 @@ +--- +id: 33 +title: Fix red CI — ruff format --check fails on Python snippets inside markdown docs +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-12g +claimed_at: 2026-08-12 +resolution: decided +evidence: executed +closed_by: null +spawned_by: null +--- + +## Premise + +Rests on [Diagnose and fix why CI is red at migration +HEAD](04-fix-migration-ci.md), closed: that ticket fixed a *different* +red-CI cause (Typer's `FORCE_TERMINAL` freezing at import time under +`GITHUB_ACTIONS=true`, breaking `--help` substring assertions) and left CI +green on the PR #6 merge commit — run `31286057254`, 2026-08-09, the last +success on `migration`. + +CI has failed on **every** push to `migration` since, starting with +`31343020352` (2026-08-09, "Give run product parity with run patient's +console output"). This is a new, unrelated cause; ticket 4's fix is not +regressed. + +Also rests on the map's rule that the destination requires **CI green** +before [ticket 6](06-promote-migration-to-dev.md) can promote `migration` +into `dev`. Every triage ticket closed since 2026-08-09 was verified with a +locally-green suite while CI itself was red the whole time. + +## Question + +CI's `Run ruff formatting check` step fails — and only that step; the test +suite never runs, which is why each run dies in ~15s. Diagnosed +(2026-08-12g, read from `gh run view 31643782401 --log-failed`), the single +current cause is: + +**`ruff format --check` wants to reformat the Python code blocks embedded in +`docs/archive/MIGRATION_GUIDE.md`** (4 blocks: the `settings.*` +configuration example around line 158, the vectorized `with_columns` +example around 194, and the GCS/BigQuery/loguru snippets around 214-230). +Ruff formats fenced Python in markdown, and this file has never been run +through it. + +Nothing else is failing. An earlier run (`31591513197`) additionally showed +`src/a4d/migration/compare.py`'s `except TypeError, ValueError:` — that is +**valid** Python 3.14 (PEP 758 permits unparenthesized except tuples), ruff +was normalizing *to* it, and it is already applied. Do not "fix" that line. + +**Why this survived four days of green local runs:** `uv run pytest`, `ruff +check`, and `ty check src/` all pass locally; only `ruff format --check` +fails, and it is easy to run `ruff format` (which silently rewrites the +markdown) without noticing. Twice during ticket 27's session the resulting +`MIGRATION_GUIDE.md` diff was actively reverted as "incidental +reformatting unrelated to this work" — which is exactly the change CI was +asking for. Any fix must make that failure mode impossible to repeat, not +just clear the current red. + +Decide between: + +1. **Accept the reformatting** — run `ruff format`, commit the + `MIGRATION_GUIDE.md` changes. Simplest; keeps doc snippets in the same + style as real code. Check the reformatted blocks still read as intended + (the "R (slow) vs Python (fast)" comparison in particular loses some of + its visual contrast when the Python side is rewrapped). +2. **Exclude markdown from `ruff format`** — via `extend-exclude` or a + format-specific setting in `pyproject.toml`. Keeps illustrative snippets + free to diverge from formatter rules (they are prose, not code, and are + never executed or imported). + +**Recommendation: (1)**, plus a guard so this cannot silently recur — the +docs are meant to mirror real pipeline code, so formatting them the same way +is consistent rather than arbitrary, and option 2 would let doc snippets +drift stylistically from the code they document. The guard matters more than +the choice: add `ruff format --check` to whatever local pre-commit/`just` +path is actually run before pushing, so the checks run locally are the same +set CI runs. + +Verify by pushing and confirming a green run on `migration`, not by a local +`ruff format --check` alone — the point of this ticket is that local and CI +checks had diverged. + +## Resolution + +**Decision:** Option 2 — `docs/archive` added to ruff's `extend-exclude` +in `pyproject.toml`, alongside the existing `r-archive` entry. The user +overrode this ticket's recommendation of option 1, and was right to: +`MIGRATION_GUIDE.md` is a **working spec document**, and there is nothing in +it for ruff to validate. Its fenced Python is illustrative prose — never +imported, never executed — so formatting it enforces code rules on something +that is not code, and only rewraps examples whose line breaks were chosen for +readability. + +**Because:** scoped to the whole `docs/archive/` directory rather than +just the one failing file. Both markdown files in the repo containing +`” ```python ”` fences live there (`MIGRATION_GUIDE.md`, +`PYTHON_IMPROVEMENTS.md`, confirmed by grep across `docs/` and root); the +second is incidentally well-formatted today and would have re-broken CI on +any future edit. Excluding the directory fixes the cause rather than the +instance. + +**Rejected:** option 1 (accept the reformatting and commit it), this +ticket's own recommendation, on the reasoning that docs should mirror real +code style. That confuses "code shown in a document" with "code" — the +snippets are explanatory and are allowed to prioritize clarity over +formatter rules. Also rejected: excluding all markdown repo-wide, which +would silently cover future docs that might genuinely want formatting; +`docs/archive/` is the meaningful boundary. + +**Evidence:** executed — every CI step reproduced locally and passing: +`ruff check .` (all checks passed), `ruff format --check .` (144 files, was +"1 file would be reformatted"), `ty check src/`, `pytest -m "not slow and +not integration"` (555 passed, 1 skipped), and the product coverage gate +(88%, floor 85). `git status docs/archive/` confirms the markdown itself +is untouched. Final confirmation is a green run on `migration` after push — +the whole point of this ticket being that local and CI check sets had +diverged, so a local pass alone is not the proof. + +**Tense:** current behaviour; all numbers from this session's actual runs. + +**Guard, not yet done:** this ticket asked for a mechanism so the failure +cannot silently recur. The exclusion removes *this* trigger, but local and +CI check sets can still drift — nothing local runs the full CI set +automatically. Spawned as [ticket 34](34-local-ci-parity-guard.md) rather +than left as an untracked intention. diff --git a/docs/wayfinder/tickets/34-local-ci-parity-guard.md b/docs/wayfinder/tickets/34-local-ci-parity-guard.md new file mode 100644 index 0000000..6ae093b --- /dev/null +++ b/docs/wayfinder/tickets/34-local-ci-parity-guard.md @@ -0,0 +1,82 @@ +--- +id: 34 +title: Make the local pre-push check set actually match CI, and make running it automatic +labels: [wayfinder:grilling] +status: open +blocked_by: [] +assignee: null +claimed_at: null +resolution: null +evidence: null +closed_by: null +spawned_by: 33 +--- + +## Premise + +Rests on [Fix red CI — ruff format --check fails on Python snippets inside +markdown docs](33-fix-red-ci-ruff-format-markdown.md), closed: CI was red on +`migration` for four days and nobody noticed, because the checks actually +run locally between commits (`pytest`, `ruff check`, `ty check src/`) were a +*subset* of what CI runs. The missing one — `ruff format --check` — was the +one failing. Excluding `docs/archive` removed that specific trigger, but +not the drift that let it go unseen: the same class of failure recurs the +moment CI gains a step, or an existing step's arguments change. + +## Question + +A `just ci` recipe already exists and is meant to be this guard: + +``` +ci: format-check lint check test +``` + +Two things are wrong with relying on it as things stand. + +**1. It does not actually match CI.** Compared against +`.github/workflows/`: + +- `just test` runs `pytest -m "not slow"`; CI runs + `pytest -m "not slow and not integration" --cov --cov-report=xml`. The + local set therefore *includes* integration tests (which are + USB-drive-gated and skip or fail depending on whether the drive is + mounted) and CI does not — so the two can disagree in both directions. +- CI additionally enforces a **product-code coverage floor** (`coverage + report --include=<7 product files> --fail-under=85`) that `just ci` never + runs. A change that drops product coverage below 85% passes `just ci` and + fails CI. + +**2. Nothing makes anyone run it.** It is a recipe you have to remember, +and ticket 33 is the evidence that remembering does not happen — including +across a session that ran `ruff format` (silently rewriting the very file +CI was failing on) and then reverted the diff twice as unrelated noise. + +Decide: + +- **How to keep `just ci` honest about CI.** Options: (a) hand-maintain it + and accept drift, (b) restructure the workflow so CI *calls* `just ci`, + making the recipe the single definition, (c) generate one from the other. + (b) is the standing-rules-compliant answer — never hand-maintain a list + that can be derived — but it changes how CI is invoked and needs the + coverage/codecov steps thought through, since those are CI-only concerns. +- **What makes it run without being remembered.** Options: a git pre-push + hook, a pre-commit framework config, or accepting that the agent working + this map runs `just ci` before every push as a standing rule in the map's + Notes. The last is the cheapest and needs no tooling, but is exactly the + kind of "remember to" that already failed once. + +**Recommendation:** (b) plus a pre-push hook, but the hook's cost is real — +it runs the full suite on every push, and this repo's suite plus coverage +takes ~30s. Worth confirming that trade before building it, which is why +this is a grilling ticket rather than a task. + +Scope note: this is about the *check set*, not about CI's own reliability. +Whether CI should also run the USB-drive-gated integration tests (it +cannot — no drive) is settled and out of scope. + +## Standing bar (applies to this ticket) + +Per the map's **triage means deciding, not labelling** preference: name the +mechanism, and decide explicitly whether the chosen approach actually +prevents recurrence rather than merely documenting the intention. "Add it to +the Notes" is a decision only if it is honestly judged sufficient. diff --git a/docs/wayfinder/tickets/35-polars-2-deprecation-warnings.md b/docs/wayfinder/tickets/35-polars-2-deprecation-warnings.md new file mode 100644 index 0000000..420e0fd --- /dev/null +++ b/docs/wayfinder/tickets/35-polars-2-deprecation-warnings.md @@ -0,0 +1,87 @@ +--- +id: 35 +title: Resolve the Polars 2.0 deprecation warnings — decide the behaviour each one is asking about +labels: [wayfinder:task] +status: open +blocked_by: [] +assignee: null +claimed_at: null +resolution: null +evidence: null +closed_by: null +spawned_by: 33 +--- + +## Premise + +Rests on [Fix red CI — ruff format --check fails on Python snippets inside +markdown docs](33-fix-red-ci-ruff-format-markdown.md), closed: CI is green +again on `migration` (run `31644262255`, 2026-08-12), so the test suite runs +end to end on CI for the first time since 2026-08-09 and its output is +readable again. + +Also rests on [Audit and update all dependencies and library +versions](13-dependency-audit.md), closed: dependencies were moved to +current latest, which is what surfaced these warnings — they are +forward-looking (Polars **2.0** behaviour), not breakage in the pinned +version. + +## Question + +`pytest -m "not slow and not integration"` emits **17 warnings**, all +`DeprecationWarning` from Polars, from exactly **three** source sites. +Each is a real behavioural decision, not a silencing job — resolve them per +the map's **triage means deciding, not labelling** bar. + +**1. `empty_as_null` on `str.split` — two sites, same call shape:** +- `src/a4d/clean/product.py:215` — `pl.col("product").str.split("; ")` then + `.explode("product")`, in `_split_multi_product_cells` (R step 2.1) +- `src/a4d/validate/source_vs_output_product.py:38` — the same split, in the + source-vs-output validator that must mirror the cleaning step + + > In Polars 2.0, the default behavior for `empty_as_null` will change to + > `False`. To keep the current behavior, explicitly set + > `empty_as_null=True`. + + The real question: when a product cell is `"Insulin; "` or `"; Lancets"`, + today's implicit `empty_as_null=True` turns the empty fragment into + `null`; under 2.0 it becomes an empty string `""` and survives `explode` + as a phantom product row. **Decide which is correct against real tracker + data** (do such trailing/leading separators actually occur in the 248-file + set?), then set the flag explicitly to that answer rather than + reflexively pinning today's behaviour. Note the two sites must stay in + lockstep — the validator exists to check the cleaning step, so a silent + divergence between them would make the validator lie. + +**2. String -> Date cast — one site:** +- `src/a4d/clean/converters.py:182` — the `pl.lit(error_value).cast(target_type)` + branch in `safe_convert_column`, where `error_value` is the + `"9999-09-09"` date sentinel string and `target_type` is `pl.Date` + + > Casting from String to Date is deprecated and will be removed in Polars + > 2.0. Use `str.to_date()` instead. + + Mechanical in principle (`pl.lit(error_value).str.to_date()`), but check + the failure semantics match: `cast(strict=False)` yields null on an + unparseable string where `str.to_date()` raises by default. The sentinel + is a known-good constant so it should be equivalent, but confirm rather + than assume — this is the sentinel-assignment path for *every* failed date + conversion in the patient pipeline. + +**Verify by re-running the suite and confirming a 0-warning run**, and (for +site 1, if the decision changes behaviour) by a real-data pipeline re-run +plus `just compare-outputs` showing no unintended movement — the cleaned +product output is currently at 23,043 cell divergences and any change here +would show up there. + +Consider also whether the suite should run with `-W error` (or +`filterwarnings = ["error"]` in `pyproject.toml`) once clean, so the next +deprecation is caught at introduction rather than accumulating unnoticed — +that is the same class of drift [ticket +34](34-local-ci-parity-guard.md) is about. + +## Standing bar (applies to this ticket) + +Per the map's **triage means deciding, not labelling** preference: pinning +current behaviour to silence a warning is only a valid resolution if +current behaviour is confirmed correct. Say what was checked. diff --git a/docs/wayfinder/tickets/36-triage-product-cleaned-unclassified-residual.md b/docs/wayfinder/tickets/36-triage-product-cleaned-unclassified-residual.md new file mode 100644 index 0000000..6d4ec3c --- /dev/null +++ b/docs/wayfinder/tickets/36-triage-product-cleaned-unclassified-residual.md @@ -0,0 +1,397 @@ +--- +id: 36 +title: Triage the product cleaned-stage mismatches no ticket owns (product_balance, sheet_name, entry_date, units_received, file_name) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-13 +claimed_at: 2026-08-13 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 25 +--- + +## Premise + +Rests on three closed decisions: + +- [Triage the remaining product cleaned-stage column + mismatches](21-triage-remaining-product-columns.md) — established the + product arm's dominant cleaned-stage cause (R's row sort falls back to + input order whenever `product_entry_date` fails to parse, while Python + sorts chronologically on correctly parsed dates), added the + `row_order_divergence` classifier, and explicitly left + `product_balance`'s residual as "future work" without giving it a ticket. +- [Triage the product_units_released cleaned-stage column + mismatches](25-triage-product-units-released-cleaned.md) — closed the last + column that *did* have a ticket, and in doing so found that the remaining + `unclassified` rows on the product cleaned-stage report belong to no + ticket at all. It also established the mechanism behind `product_balance`'s + poor classification rate: `product_balance` is *derived*, not carried — + both pipelines recompute it as a running total + (`balance[i] = balance[i-1] - released[i] + received[i]`, Python's step + 2.15 `_compute_running_balance`, R's iterative equivalent) — so under a + legitimate re-sort the balance cannot travel with its row, and + `row_order_divergence`'s "does R's value appear elsewhere in Python's + group" membership test structurally under-detects it. +- [Triage every flagged R/Python difference for both arms](18-triage-comparison-flagged-differences.md) + — established that per-column counts are judged by pattern, not by + reproducing the parity-presentation PDF's numbers. + +Current counts, measured against the real 248-tracker drive comparison +(`output/comparison/2026-08-12T220324Z/`, `Product (cleaned)` stage): + +| Column | `unclassified` | +|---|---| +| `product_balance` | 1,976 (of 2,740; 764 already `row_order_divergence`) | +| `product_sheet_name` | 275 | +| `product_entry_date` | 169 (of 11,727) | +| `file_name` | 60 | +| `product_units_received` | 8 (of 305) | + +Two of these have a documented lead but no verdict: `product_sheet_name` is +believed to be R's un-trimmed sheet names (noted in passing by [ticket +17](17-fix-product-row-alignment-and-triage.md), whose count of 201 +reproduced the PDF exactly — the count is now 275 against the refreshed +baseline), and `product_units_received`'s 8 are believed to be the +Excel-date-serial-leak pattern [ticket +24](24-triage-remaining-raw-column-residual.md) confirmed for the raw stage, +reaching the cleaned stage too. Neither was ever decided under the standing +bar. `file_name` and `product_entry_date`'s residual have no lead at all. + +This ticket exists because the destination requires **every** Python/R +difference documented and explicitly decided; these rows were reported as +`unclassified` on every run while each triage ticket worked only the columns +its own title named. + +## Question + +Explain and decide each of the five columns above. `product_balance` is the +bulk of the work and the one with a known mechanism but no adequate +detection: decide whether its residual is fully accounted for by the +already-established sort-order divergence (in which case the open question is +how to *evidence* that for a derived running total — e.g. comparing +end-of-group balances, or reconstructing R's order and re-deriving — rather +than by value membership), or whether some part of it is a genuine divergence +the sort-order story hides. + +If the work does not converge in one session, split rather than sprawl. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: explaining +a difference and naming a cause is only half the job. Each must carry an +explicit verdict on **whether Python is doing the right thing**, with the +evidence that establishes it (source Excel, or R's/Python's own code). Where +Python turns out to be wrong or to be losing information the source file +carried, fix the pipeline rather than labelling the symptom. A cause +genuinely undecidable on the available evidence is recorded as an open +question, not closed with a label. + +## Addendum (session-2026-08-12h): the balance mechanism, measured + +Follow-up investigation after ticket 25 closed, prompted by the user asking +whether `_compute_running_balance` is itself wrong. Findings are verified, +the decision is **not** made — that is still this ticket's job. + +**There is no defect in `_compute_running_balance`.** It mirrors R exactly. +R's `compute_balance` (`r-archive/R/helper_product_data.R:437-481`) loops +over rows and, for every `"change"`/`"end"` row, *overwrites* +`product_balance` with `previous_balance - released + received`; Python's +step 2.15 does the same via a per-group cumsum seeded from the group's first +(`"start"`) row. **Both pipelines discard the source spreadsheet's recorded +balance on non-start rows by design** — this is the original R design, not a +migration artifact. + +**What actually differs is the order the ledger accumulates in**, which is +downstream of step 2.7's sort, not of the balance step. Measured on +`2024_Mahosot Hospital`, sheet `Jul24`, `Accu-Chek Performa Test Strips +(50s/ bottle)` (43 rows), against the real source Excel: + +- The source's own Balance column is internally consistent **in data-entry + order** (322, 314, 310, 306, 302, 300, 293, 290, ... — each equals the + previous minus that row's Units Released). +- R, accumulating in entry order, reproduces the source's balances **exactly** + (multiset equal, row for row). +- Python, accumulating in chronological order, emits 322, 314, 306, 302, + 298, 294, ... — a self-consistent ledger whose intermediate values appear + **nowhere in the source file** (source has 172/182/186/202...; Python has + 176/178/183/187...). +- **Both end at 139.0**, matching the source's closing balance. + +Note the source's entry order here is *not* chronological (the clinician +entered 2024-07-23 before 2024-07-19), so the source's own Balance column is +a ledger in data-entry order, not a stock history over time. + +**Breadth, across all 11,649 `(clinic, sheet, product)` groups:** + +| Measure | Groups | Share | +|---|---|---| +| Balance multiset identical R vs Python (no divergence at all) | 11,117 | 95.4% | +| Closing (end-of-group) balance identical | 11,491 | 98.6% | +| Closing balance differs | 158 | 1.4% | + +Of those 158, **153 are float-accumulation noise where Python is the cleaner +side** (R `-1.5999999999999999` vs Python `-1.6`; Python's cumsum applies +`.round(10)`, R's loop rounds nothing). The other **5 are R being corrupted +by an Excel date serial leaking into the arithmetic** — e.g. 2019 Penang +General Hospital, `Accu-Chek Performa Glucometer Set`: R's closing balance is +**43,572** where Python has **6.0** (43,572 is a 2019 date serial); same +shape for four Sultanah Bahiyah groups. This is the stray-date-typed-cell +pattern [ticket 24](24-triage-remaining-raw-column-residual.md) already +root-caused, now shown to corrupt R's *balance totals*, not just individual +cells. **Python's closing stock is correct in every group.** + +**The open question this ticket must decide** is therefore not "is Python +buggy" but **what `product_balance` is supposed to mean on a non-closing +row**, given it is written per-row to BigQuery's `product_data` table and has +no consumer inside the Python codebase (verified by grep — only +`clean/product.py` and the schema reference it): + +1. *Keep current behaviour* — balance accumulates in chronological order, so + it is coherent as a stock-over-time series and the closing figure is + right, but an individual row will not match the tracker cell a human + opens next to it. +2. *Carry the source's recorded balance*, computing only where absent — + maximally faithful per row, but propagates the source's own arithmetic + mistakes and abandons R's design. +3. *Accumulate in entry order, output in chronological order* — each row + keeps the balance the tracker recorded while rows still display + chronologically; the balance column then reads non-monotonically down the + output. + +Whichever is chosen, consider logging a data-quality error when the +recomputed balance disagrees with the source's recorded balance: nothing +currently surfaces that, and it is the signal that would have exposed the R +date-serial corruption above from the tracker side. + +### Decision on the balance question (user, 2026-08-12h) — implemented + +**Option 1 chosen: keep the current recompute-after-chronological-sort +behaviour, and add a clear log warning.** The source's balance column +accumulates in data-entry order, which is not a meaningful stock history when +entries are out of order, so reverting to it would trade a coherent series +for a faithful-looking but less useful one. Python's closing stock is already +correct in every group. What was missing was any signal when the ledger and +the tracker's own total disagree. + +**Implemented.** `_compute_running_balance` (`src/a4d/clean/product.py` step +2.15) now takes an optional `ErrorCollector` and, via +`_report_balance_reconciliation`, reports **per (sheet, product) group** when +the computed closing balance contradicts the balance the tracker itself +recorded. New error code `balance_reconciliation` in `src/a4d/errors.py`; it +flows into the errors/logs tables and BigQuery unchanged, since nothing +filters by code. + +**Granularity was chosen by measurement, not by taste** — this is the part a +later session should not re-litigate: + +| Candidate signal | Fires on real 248-tracker data | Verdict | +|---|---|---| +| Every row where computed != recorded | 15,301 rows (31.4% of recorded rows) | Noise — this is the re-sort working as designed | +| Per-group closing balance, incl. start-only groups | 781 groups (7.0%) | Still mostly false: a group whose only recorded balance is its start has no closing figure to reconcile | +| **Per-group closing balance, groups that recorded a non-start balance** | **113 groups (1.1%), 21 files** | **Chosen** — order-independent, so a disagreement is a real source problem | + +Verified end-to-end: all 248 cleaned outputs re-run and confirmed +**byte-identical** to the previous run (no pipeline behaviour changed, only +reporting), and the live run produces exactly the 113 warnings predicted. +Example message: + +> Closing balance mismatch for product 'NovoRapid FlexPen 3ml (singles/ 5s)' +> in sheet 'Mar26': tracker recorded 34.0, but the recorded transactions add +> up to 68.0 (difference 34.0). The stock movements and the tracker's own +> total do not agree. + +Warnings concentrate rather than scatter — 54 of the 113 are in +`2021_Mahosot Hospital`, 12 in `2021_Mandalay Children's Hospital` — so they +point at specific trackers worth investigating. + +**Still open for this ticket:** the five columns' classification (including +`product_balance`'s 1,976 `unclassified` rows) is untouched by the above; the +new warning is an operability improvement, not a triage verdict. Whether the +113 flagged groups are source data-entry errors or something the pipeline +mishandles has **not** been investigated. + +## Progress (session-2026-08-13) + +Worked small-columns-first at the user's direction. Three of the five columns +are decided and implemented; the whitespace pair awaits a user decision. + +Counts are against the real 248-tracker drive pair, +`output/comparison/2026-08-13T212807Z/`, `Product (cleaned)` stage. + +### `product_units_received` — 8 -> 0 `unclassified`. Python correct. + +Source Excel `2019_Sultanah Bahiyah Hospital A4D Tracker_DC.xlsx`, sheet +`Aug19`: the header row reads `D=Date, E=Units Received`, and on rows 10-12 +the clinician typed the entry date into **E** with **D** left empty. R's +readxl coerces the column to numeric and carries the raw serial (`43708`) +into its output — the same serials that corrupt R's closing balance in the +five groups this ticket's addendum measured. Python's `_clean_units_received` +(step 2.11) fails the float cast, emits one `type_conversion` error per row, +and zeroes the cell. + +**Verdict: Python is right**, and observably so — the error is logged, not +swallowed. New `stray_date_zeroed` classifier +(`STRAY_DATE_ZEROED_CLASSIFIERS`): the cleaned-stage face of ticket 24's +`openpyxl_date_typed_stray_cell`, which could not see these because it +expects Python's side to still hold a datetime. + +### `product_entry_date` — 169 -> 9 `unclassified`. Python correct. + +Root cause found in source Excel `2022_Vietnam National Children_s Hospital`, +sheet `Apr22`: the "Entry Date" column holds **mixed types** — some cells are +real `datetime`, others are text (`'20/04/2022'`, `'14/04/2022'`, ...). +readxl guesses the column as date and nulls every *text* cell; Python's +`parse_date_flexible` reads both. In that one sheet-group R loses 25 real +dates, which drops it back to input-order sorting, so positional alignment +pairs R's July date against Python's April date — both non-null, so +`r_value_missing` never fires. + +Three sub-causes, all decided: + +| Sub-cause | Rows | Verdict | +|---|---|---| +| `row_order_divergence` (113) | 113 | R's readxl-nulled text dates drop it to input order; Python sorts on dates that really are in the file | +| `python_future_date_sentinel` (34) | 34 | Python substitutes R's own `9999-09-09` sentinel for an out-of-tracker-year date (`_validate_entry_dates`); R propagates the typo | +| `summary_residue_nulled` (13) | 13 | Python nulls end-of-block residue (`30` -> 1900-01-30); R keeps the junk date | + +The future-date verdict is source-verified: for Preah Kossamak's 2023 +tracker, sheet `Aug23`, **both** pipelines independently read `2029-08-29` +out of a sheet whose every other row is August 2023 — so the source really +does carry the typo. Python flags it; R passes it through. + +The 9 remaining rows (8 real-vs-real dates, 1 R-date-vs-Python-null in +`2023_Taunggyi`, `Nov'23`) are positional-alignment residue where the +membership heuristic misses. Checked directly for the Taunggyi row: Python's +raw extraction holds `2023-11-30` for that group's last row, i.e. R's row +pairs against a Python row that is null — an alignment artefact, not a +content divergence. Deliberately left `unclassified` rather than given a +label that would decide nothing. + +Also wired `PRODUCT_ROW_ORDER_CLASSIFIERS` onto every remaining positionally +aligned product column (`product_units_notes`, `product_units_returned`, +`product_returned_by`, `product_balance_status`, `orig_product_released_to`, +`product_unit_capacity`, `product_category`), and **replaced ticket 25's +hand-written `positional_columns` test list with one derived from +`get_product_data_schema()`** minus a new +`GROUP_INVARIANT_PRODUCT_COLUMNS` exclusion set. The hand-written list is +precisely what let `product_entry_date` sit unwired: the test written to +prevent this class of omission had itself omitted the column. + +### `product_balance` — 1,976 -> 11 `unclassified`. Python correct. + +The addendum above established the mechanism but not the detection. The +missing piece is that a derived running total needs *order-independent* +evidence, and the group's **closing** balance is exactly that. + +New `group_endpoint_matches` diagnostic on `CellMismatch`, computed by +`compare_cells` (only for `add_row_ordinal`'s positional key), plus a +`derived_running_total_row_order` classifier keyed off it. Measured +independently of the comparison tool, straight off the parquet pair, and +then reproduced by the tool exactly: + +| | Cells | Groups | +|---|---|---| +| Balance mismatches in groups whose closing balance **agrees** | 2,729 | 2,281 | +| Balance mismatches in groups whose closing balance **differs** | 11 | 2 | + +Both differing groups are `2019_Sultanah Bahiyah ... DC`, sheets `Jun19` and +`Aug19` — the same stray-date-serial groups above, where R's ledger is +corrupted by `43644`/`43708` leaking out of "Units Received". **Python's +closing stock is correct in both.** + +Those 11 are left `unclassified` **on purpose**: `unclassified` on +`product_balance` now means "the two ledgers disagree on closing stock", +which is a real signal worth surfacing. Absorbing them into the same label +would have been labelling, not deciding. + +### `product_sheet_name` (275) + `file_name` (60) — 335 -> 0. Pipeline fixed. + +All 335 are R keeping trailing whitespace Python strips: `trim(r_value) == +py_value` for 275/275 of the sheet-name rows. The source really carries it — +the Excel tab is literally named `'Dec24 '` and the file is literally +`06 Pahol Polpayuhasena Hospital A4D Tracker_Jun_26 - final .xlsx`. R trims +only `product_released_to` (`read_product_data.R:204`); Python's step 2.16 +strips every string column. + +**The finding is not about R.** Python is inconsistent with itself: + +| | `file_name` | `sheet_name` | +|---|---|---| +| `patient_data_cleaned` | `'... - final '` | `'Dec24 '` | +| `product_data_cleaned` | `'... - final'` | `'Dec24'` | +| `tracker_metadata` (`tracker_path.stem`) | `'... - final '` | — | + +Measured across all 248 trackers: 1 `file_name` value and 9 `sheet_name` +values (`Apr26 `, `Dec24 `, `Feb21 `, `Jun19 `, `May19 `, `May20 `, `May26 `, +`Nov21 `, `Sept20 `) differ between the arms, so any BigQuery join of +`product_data` to `patient_data_*` or `tracker_metadata` on those keys +silently drops them. + +**User decision:** trim everywhere. *"I see no reason why the pipeline should +not consistently trim whitespace at the end of all strings, filenames etc. +there is never a meaning in that"* — the source files should be corrected +too, but that is a human task, not the pipeline's. + +**Implemented as a real pipeline change:** + +- `strip_string_whitespace()` (`src/a4d/clean/transformers.py`), applied at + **step 0.5** of `clean_patient_data` — deliberately *before* validation, + not after, so a value is never rejected for whitespace alone. Product's + own step 2.16 already did the equivalent and is unchanged. +- `tables/metadata.py` now strips `tracker_path.stem`, so + `tracker_metadata.file_name` joins the cleaned outputs' own `file_name`. + Output parquet names still carry the untrimmed stem, and the trimmed name + remains a prefix of them, so the presence check is unaffected. +- Comparison tool: `product_sheet_name`/`file_name` added to the product + cleaned whitespace normalization, and a patient equivalent added for both + patient stages, **derived from `get_string_columns()`** rather than + hand-listed. `sheet_name` is half the patient row-alignment key, so + without this R's untrimmed `"Dec24 "` would fail to join Python's + `"Dec24"` and silently drop those rows from the comparison rather than + showing them as equal. + +**The trim recovered real data that both pipelines were losing.** A test +written before the change (`test_whitespace_does_not_defeat_allowed_value_validation`) +predicted it and failed as expected: `sex = "F "` validated to `"Undefined"`. +Confirmed on the real dataset and against the source Excel — +`2019_Kantha Bopha Hospital`, sheet `Jan19`, patient `KH_KB023`, cell reads +literally `'F '`. R's validator rejects it and sentinels it; Python used to +copy that behaviour and now keeps the value. **72 rows across 6 trackers**, +classified as `r_validator_rejects_untrimmed` — Python recovering data, not +diverging. + +Patient re-run (`a4d run patient --force`, all 248 trackers) and comparison +re-run confirm the rest is representation-only: + +| Patient cleaned column | Before | After | +|---|---|---| +| `complication_screening_remarks` | 57 | 0 | +| `family_history` | 24 | 0 | +| `observations` | 360 | 327 | +| `blood_pressure_dias_mmhg` | 315 | 303 | +| `testing_frequency` | 154 | 144 | +| `sex` | 4 | 76 (72 of them the recovery above) | + +Patient cleaned total: **99,478 -> 99,408**. The two numeric columns improved +because trimming lets their values parse. Patient raw is byte-unchanged +(28,033), as expected — raw deliberately preserves what the source contained. + +### Where the numbers stand + +Product cleaned-stage `unclassified`: **2,488 -> 20** (11 `product_balance`, +9 `product_entry_date`), every one explained with a verdict, and each +remaining as a deliberate signal rather than a label: + +- the 11 mean "the two ledgers disagree on closing stock" — both are the + 2019 Sultanah Bahiyah groups where R is corrupted by a date serial; +- the 9 are positional-alignment residue. + +Both are terminal conclusions of the "the source file is corrupt, a human +should look at this tracker" kind, which the user confirmed mid-session is a +valid place for triage to stop. + +Full suite 629 passed / 1 skipped, `ruff check`, `ruff format --check`, +`ty check src/` all pass. diff --git a/docs/wayfinder/tickets/37-triage-patient-cleaned-residual-2.md b/docs/wayfinder/tickets/37-triage-patient-cleaned-residual-2.md new file mode 100644 index 0000000..bf7fd4a --- /dev/null +++ b/docs/wayfinder/tickets/37-triage-patient-cleaned-residual-2.md @@ -0,0 +1,230 @@ +--- +id: 37 +title: Triage the residual patient cleaned-stage mismatches (round 2) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-14 +claimed_at: 2026-08-14 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 29 +--- + +## Premise + +Rests on [Triage the residual patient cleaned-stage column +mismatches](29-triage-patient-cleaned-residual.md), closed: that ticket +root-caused the two priority columns its own question named and, along the +way, the whole 999999-sentinel family. Two real Python bugs were found and +fixed (`extract_regimen` lowercasing every value it did not match; +`validate_allowed_values` picking the last of two colliding config +spellings), and three classifiers were added +(`r_insulin_dedup_drop`, `r_join_suffix_collision`, +`r_numeric_error_sentinel`). Cleaned-stage mismatches fell 99,408 -> +95,490 and unclassified rows 55,670 -> 16,698 (-70%), verified against a +fresh 248-tracker pipeline re-run and comparison. + +It split the rest off rather than force convergence, per its own +pre-authorization. + +## Question + +Triage the 16,698 remaining unclassified cleaned-stage mismatches. Re-run +`just compare-outputs` first -- ticket 29's run +(`output/comparison/2026-08-13T221849Z`) is the baseline. Current shape, +largest first: + +- **`blood_pressure_updated` (4,504)** -- 4,442 of these are Python-null + where R holds a value, plus 48 where Python holds the future-date + sentinel against a Thai Buddhist-Era year in R (2568/2569). The + Buddhist-Era half is the *inverse* direction of the existing + `buddhist_era_typo` classifier, which was written for the raw stage + (R sentinels, Python passes through); at the cleaned stage Python's own + future-date guard sentinels it and R carries the bad year through. The + py-null majority is not characterized at all and is the real question. +- **The date-column family: `hospitalisation_date` (2,149), + `hba1c_updated_date` (1,809), `fbg_updated_date` (1,800), + `t1d_diagnosis_date` (1,082), `last_clinic_visit_date` (193)** -- each + mixes four shapes in different proportions: Python-null vs R-value, + Python-sentinel vs R-date, R-sentinel vs Python-null, and both-real. + `fbg_updated_date` also shows a 1900-01-03 vs 1900-01-04 pair, which + looks like the Excel 1900-leap-year serial bug ticket 24 already handled + once in `STRAY_DATE_CLASSIFIERS` -- worth checking whether that + classifier's logic applies here. +- **`insulin_type` (1,265)** -- 1,252 are R-null where Python has a real + value ("Analog Insulin"). `insulin_type` is derived by the same + `_derive_insulin_fields` step whose sibling `insulin_subtype` is already + explained by `r_validator_rejects_multivalue`; check first whether this is + the same cause with a different symptom. +- **`t1d_diagnosis_age`'s remaining 560 and `recruitment_date`'s remaining + 502** -- what is left of each after ticket 28's and ticket 29's causes + were removed. +- **A tail of ~50 smaller columns**, including `observations` (327) and the + blood-pressure value pair (`blood_pressure_dias_mmhg` 303, + `blood_pressure_sys_mmhg` 240), none sampled. + +Same method: look for systematic shapes, fall back to the real source Excel +trackers as the arbiter (mount at `/Volumes/USB SanDisk 3.2Gen1 Media/a4d/`), +add named causes to the `PATIENT_*_CLASSIFIERS` registries in +`src/a4d/migration/compare.py` (`scripts/compare_outputs.py`'s +`CLASSIFIERS_BY_COLUMN` wires them to columns) -- or fix a real Python bug +directly, as ticket 29 did twice, when one turns up. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: it is not +enough to explain a difference and name a cause. Each one must also carry an +explicit verdict on **whether Python is doing the right thing**, with the +evidence that establishes it (source Excel, or R's/Python's own code). Noting +"Python has A where R has B" and adding a classifier is not a decision in +favour of A. Where Python turns out to be wrong or to be losing information +the source file carried, fix the pipeline rather than labelling the symptom. +A cause genuinely undecidable on available evidence is recorded as an open +question, not closed with a label; where the evidence shows the source file +itself is corrupt, "the source is wrong, this tracker needs human +inspection" is a legitimate final conclusion. + +## Resolution (session-2026-08-14) + +**Decision.** Four of the five named leads are resolved, three of them by +fixing a real Python bug rather than by naming a cause. Unclassified +cleaned-stage mismatches fell **16,698 -> 8,259 (-51%)**, total mismatches +98,101 -> 93,997. The date-column family did not converge and is split into +[ticket 38](38-triage-patient-cleaned-date-family.md), per this ticket's own +method note. + +**Pipeline bugs found and fixed** (all four are data-loss or +non-determinism, none was a labelling job): + +1. `merge_headers` (`src/a4d/extract/patient.py`) produced the unmappable + header `Updated 2022 Date` for every 2022 tracker, because that template + writes `Updated 2022` in the upper header row instead of repeating the + subject column's name. R carries an explicit fixup for exactly this + (`script1_helper_read_patient_data.R`, lines 47-61). Python now treats a + whole-cell `Updated ` as a continuation of the column to its left, + which reproduces R's result without hardcoding either the year or the two + affected column names. Recovered **4,442 `blood_pressure_updated` and + 2,723 `edu_occ_updated` values**; `blood_pressure_updated` mismatches + 4,504 -> 62. +2. `_apply_type_conversions` (`src/a4d/clean/patient.py`) stripped a trailing + time component by splitting on the first space and keeping the first + token, which destroyed genuine space-separated dates: `"Jun 2006"` became + `"Jun"`, which dateutil completed from **today's date**, producing a + future date that `_validate_dates` then replaced with the 9999-09-09 + sentinel. Two defects in one: real diagnosis dates lost, and output that + changes depending on the day the pipeline runs. Now matches the trailing + time pattern itself. +3. `parse_date_flexible`'s month-name truncation deleted only the *fourth* + letter (`re.sub(r"([a-zA-Z]{3})[a-zA-Z]", ...)`), so `"March"` became + `"Marh"` and `"January"` became `"Janary"` -- unparseable, and sentinelled. + Every full month name in the trackers was affected, including the + function's own docstring examples, which never worked. Now anchored on a + month-name alternation. +4. `parse_date_flexible`'s month-year branch only accepted a 2-digit year, so + `"Jun 2006"` fell through to dateutil's today-based day fill even once (2) + was fixed. Extended to 4-digit years, resolving to the first of the month + -- matching both R and Python's own 2-digit behaviour. + +Fixing (2) removed the trailing-free-text truncation it had been doing by +accident (`"16-Nov-2019 due to DKA"`), so `parse_date_flexible` gained an +explicit longest-parseable-prefix fallback, tried only after the whole string +fails, and skipping a purely alphabetic prefix (a bare month name is what +made the old behaviour depend on the run date). + +Measured on the real 254-tracker pair: date-column mismatches fell 2,344 +(`fbg_updated_date` -761, `hba1c_updated_date` -734, `recruitment_date` -422, +`t1d_diagnosis_date` -419). + +**Classifiers added or extended**, each source-verified: + +- `r_extraction_gap` extended to `blood_pressure_updated` and + `edu_occ_updated` (2,785 rows), covering two distinct R mechanisms + documented in its docstring: R's own 2022 header fixup is defeated by a + *leading space* in the source cell (` Updated \n2022`), so R's merged name + sanitizes to the junk column `xlevelofeducationoroccupationdate` that no + synonym matches -- confirmed in R's own raw parquet; and R reads nothing at + all from the 2026 template's new `Annual` sheet, verified against + `2026_YGH T1D Tracker_June_26`, `Annual!G/H/I` for MM_YG101 + (2026-02-05, 100, 60) where R's whole-file count is 0 non-null. The + registry was renamed `PATIENT_RECRUITMENT_DATE_CLASSIFIERS` -> + `PATIENT_R_EXTRACTION_GAP_CLASSIFIERS`, since it now serves three columns. +- `r_ifelse_na_propagation` (new, `insulin_type`, 1,252 rows): R's + `ifelse(pre_mixed == "Y" | short == "Y" | intermediate == "Y", ...)` + evaluates `FALSE | FALSE | NA` as NA under R's three-valued logic, so a row + whose human-insulin columns are blank but whose *analog* columns plainly + read "Y" loses its type entirely. Verified both pipelines hold the same + five input values (2026 Baguio General, May26, PH_BG001 and PH_BG004). + Python is the correct side; not the same cause as + `r_validator_rejects_multivalue` on the sibling `insulin_subtype`. +- `buddhist_era_typo` made symmetric: at the cleaned stage the direction + reverses on `blood_pressure_updated` (R carries the Buddhist-Era year + through, Python's future-date guard sentinels it). Same typo, same verdict + -- the sentinelling side is the one that recognized an unusable year. +- `python_future_date_sentinel` reused for `edu_occ_updated`'s reverse + direction (17 rows); its docstring now names patient's `_validate_dates` + alongside product's `_validate_entry_dates`. + +**Rejected.** Writing a `blood_pressure_updated` classifier for the 4,442 +Python-null rows: the shape said "Python has nothing" and the source said +otherwise (2022 Kantha Bopha, `Jan'22` col 27, KH_KB022 = 2022-01-19 +alongside a BP of 110/60), so labelling it would have cemented an extraction +bug as explained -- ticket 27's precedent exactly. Distinguishing +`edu_occ_updated`'s two R mechanisms as separate cause names: `classify()` +sees only the (r_value, py_value) pair, never the file, so it cannot tell a +2022 tracker from a 2026 one; both are documented under one name instead, +following `_is_r_value_missing`'s precedent. + +**Found, deliberately not fixed.** 13 `insulin_type` rows where R and Python +hold the *same* two rows in a different order, because the patient row key +(`patient_id` + `sheet_name`) is duplicated for that patient in that sheet. +That is a comparison-tool alignment issue, not a pipeline divergence, and the +same shape [ticket 31](31-triage-patient-raw-residual-2.md) already flagged +for the raw stage -- a classifier here would be a false explanation. Left +unclassified deliberately, and noted on ticket 38. + +**Evidence.** Executed. Every claim above rests on a run or a query against +the real drive data, not on reading: two full `a4d run patient --force` +re-runs over the whole tracker set, four `just compare-outputs` runs +(`output/comparison/2026-08-14T195310Z` immediately before the data refresh, +`2026-08-14T200342Z` after), direct openpyxl reads of the named source +workbooks, and direct queries against R's own raw and cleaned parquets. Full +suite 655 passed / 1 skipped, `ruff check`, `ruff format --check`, `ty check +src/` all pass. + +**Tense.** Every number above describes current behaviour after this +session's changes, except the "before" figures, which describe the state at +ticket 29's close. + +## Addendum (same session, after closure): tracker set refreshed to 254 + +At the user's direction, after this ticket's work was verified: production +trackers were re-downloaded (`a4d download trackers`, 47 new files) after the +data analyst renamed every `06 ...` file to a `2026_...` form and added new +clinics. Three consequences were handled so the R comparison survives: + +- The 41 stale `06 ...` local copies (absent from GCS) were **moved**, not + deleted, to `/Volumes/.../a4d/stale_06_trackers_2026-08-14/`. Left in place + they would have been processed alongside their renamed twins, double-counting + every 2026 patient. Local tracker count now matches GCS exactly (254). +- The comparison pairs R and Python outputs **by file name**, so the rename + would have silently dropped ~41 files out of the cell-by-cell comparison + (Python-only on one side, R-only on the other). The frozen R output's 322 + per-tracker files were renamed to the corrected names instead, the mapping + derived from the clinic folder (`clinic_id` is the parent folder name, so + the pairing is authoritative, not guessed) and verified 1:1 with zero + collisions. The map is saved at + `/Volumes/.../a4d/output_r_rename_map_2026-08-14.json`, so the rename is + reversible. Result: **0 R-only files** on every stage. +- Renaming the files left R's parquets holding the *old* stem in their own + `file_name` column -- the only column carrying it -- which produced 33k + phantom mismatches by construction. Rewritten in place across 160 R + parquets / 40,701 rows, restoring the counts exactly (patient cleaned + unclassified 15,643 -> 8,259). + +R was **not** re-run, per the user's decision: the 9-25 genuinely new +trackers per stage have no R counterpart and show as Python-only, which is +accepted. Post-refresh state, all four stages: patient cleaned 93,997 +mismatches / 8,259 unclassified; patient raw 27,921 / 14,844; product cleaned +22,718 / 20; product raw 118 / 0. diff --git a/docs/wayfinder/tickets/38-triage-patient-cleaned-date-family.md b/docs/wayfinder/tickets/38-triage-patient-cleaned-date-family.md new file mode 100644 index 0000000..1482a3f --- /dev/null +++ b/docs/wayfinder/tickets/38-triage-patient-cleaned-date-family.md @@ -0,0 +1,246 @@ +--- +id: 38 +title: Triage the patient cleaned-stage date-column family (round 3) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-14b +claimed_at: 2026-08-14T21:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 37 +--- + +## Premise + +Rests on [Triage the residual patient cleaned-stage mismatches (round +2)](37-triage-patient-cleaned-residual-2.md), closed: four real Python bugs +were fixed (the 2022 `Updated 2022` header, the first-space date split, the +broken month-name truncation, and the 2-digit-only month-year branch) and +three causes named (`r_extraction_gap` extended to two more columns, +`r_ifelse_na_propagation`, `buddhist_era_typo` made symmetric). Unclassified +cleaned-stage mismatches fell 16,698 -> 8,259. + +It also rests on [Triage the patient cleaned-stage column +mismatches](28-triage-patient-cleaned.md) and [its +residual](29-triage-patient-cleaned-residual.md), which between them +established that R's 999999 sentinel is a correct Python representation +choice and that R's own extraction gaps -- not Python defects -- explain the +largest cleaned-stage columns. + +Two of ticket 37's own fixes matter to this ticket's framing: the date parser +now resolves month-year strings to the first of the month deterministically, +and recovers a date followed by a free-text clause via a longest-parseable- +prefix fallback. Several date columns' remaining mismatches are what survived +those fixes, so they are **not** the same population ticket 37 opened with. + +Also rests on the tracker set having been refreshed to 254 files with the R +baseline renamed to match (ticket 37's addendum): the current baseline is +`output/comparison/2026-08-14T200342Z`, not any earlier run. + +## Question + +Triage the 8,259 remaining unclassified cleaned-stage mismatches. Re-run +`just compare-outputs` first. Current shape, largest first, with the value +shapes already measured (`r_sent`/`py_sent` = the 9999-09-09 error date): + +- **`hospitalisation_date` (2,111)** -- 1,468 are `r_sent/py_null`, 371 + `r_val/py_sent`, 179 `r_sent/py_val`, 88 `r_val/py_val`. The dominant + R-sentinel-vs-Python-null shape is the inverse of most causes named so far + and is not characterized at all. These cells carry free-text clauses + ("16-Nov-2019 due to DKA"), so ticket 37's prefix fallback is directly + implicated -- check whether Python's null is a correct "no date recorded" + or a parse that gives up where R sentinels. +- **`hba1c_updated_date` (1,074) and `fbg_updated_date` (1,032)** -- both + dominated by `r_val/py_val` (824 and 840): two *real, different* dates on + each side. That is the hardest shape on the report and the one no cause so + far explains. `fbg_updated_date` also showed a 1900-01-03 vs 1900-01-04 + pair before ticket 37's fixes -- re-check whether it survives, and whether + `STRAY_DATE_CLASSIFIERS`'s Excel 1900-leap-year logic applies. +- **`t1d_diagnosis_date` (663)** -- 530 `r_val/py_sent`, i.e. Python's + future-date guard firing where R carries the date through. Ticket 37 + confirmed one instance of this family is a real source typo, but the + population was never checked; `python_future_date_sentinel` may simply + apply, which would be a wiring change rather than an investigation. +- **`t1d_diagnosis_age` (560)** -- what is left after ticket 28's fix and + ticket 29's `r_numeric_error_sentinel`. +- **A tail of ~50 columns**, none sampled: `observations` (327), the blood + pressure value pair (`blood_pressure_dias_mmhg` 303, + `blood_pressure_sys_mmhg` 240), `bmi_date` (190), + `last_clinic_visit_date` (185), `height` (167), the `fbg_updated` pair + (163 each). + +Carry over one finding ticket 37 deliberately left open: **13 `insulin_type` +rows where both sides hold the same two rows in a different order**, because +`patient_id` + `sheet_name` is duplicated for that patient in that sheet. +Decide whether patient's row-alignment key needs the same treatment product's +did (`add_row_ordinal`, ticket 17) or whether the affected population is +small enough to record and leave -- measure it across all columns first +rather than judging from these 13. [Ticket +31](31-triage-patient-raw-residual-2.md) flagged the same shape on the raw +stage, so whatever is decided here should settle it there too. + +Same method: look for systematic shapes, fall back to the real source Excel +trackers as the arbiter (mount at `/Volumes/USB SanDisk 3.2Gen1 Media/a4d/`), +add named causes to the `PATIENT_*_CLASSIFIERS` registries in +`src/a4d/migration/compare.py` (`scripts/compare_outputs.py`'s +`CLASSIFIERS_BY_COLUMN` wires them to columns) -- or fix a real Python bug +directly, as ticket 37 did four times, when one turns up. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: it is not +enough to explain a difference and name a cause. Each one must also carry an +explicit verdict on **whether Python is doing the right thing**, with the +evidence that establishes it (source Excel, or R's/Python's own code). Noting +"Python has A where R has B" and adding a classifier is not a decision in +favour of A. Where Python turns out to be wrong or to be losing information +the source file carried, fix the pipeline rather than labelling the symptom. +A cause genuinely undecidable on available evidence is recorded as an open +question, not closed with a label; where the evidence shows the source file +itself is corrupt, "the source is wrong, this tracker needs human +inspection" is a legitimate final conclusion. + +## Resolution (session-2026-08-14b) + +### Decision + +**The date sentinel was being stamped on cells that recorded an absence, and +that is a Python defect, not an R divergence.** `parse_date_flexible` +recognized exactly four missing markers (`na`, `nan`, `null`, `none`) where +the numeric path's `safe_convert_column` already recognized eleven. Everything +outside that four-item list -- `-`, `.`, `N/A`, `Nil`, `Unknown`, `?`, and the +tracker template's own leftover instruction text -- reached the error sentinel, +which asserts "a date was recorded and it is invalid" about a cell that plainly +recorded nothing. + +Three changes, all in `src/a4d/clean/date_parser.py`: + +- `MISSING_VALUE_MARKERS` is now declared once and **shared** with + `safe_convert_column` (`clean/converters.py`), which had been carrying its own + copy. The two had drifted; that drift *was* the bug. The numeric path's + comparison also becomes case-insensitive as a side effect of sharing. +- `DATE_ABSENCE_MARKERS` covers absence written as a word (`nil`, `nill`, + `unknown`, `unknwon`, `uncertain`, `?`, `no`). Date-scoped deliberately: the + numeric path stamps 999999 on the same words, and whether it should is a + separate question this ticket did not reopen. +- `DATE_PLACEHOLDER_FRAGMENTS` catches the template's own instruction text + left in a data row (`Insert Date`, `Insert Date or NA`, + `NA or Hospitalisation Date`) -- substring-matched, because clinics copy it + with varying suffixes. + +**R's side of the same shape is now a named cause, and Python is the correct +side.** `r_date_error_sentinel` (`src/a4d/migration/compare.py`, wired across +every column in `get_date_columns()` -- derived, not hand-listed) covers R +stamping `9999-09-09` where Python nulls. R's `parse_dates` tests +`is.na(date)`, which is false for the *string* `"NA"`, so the value falls +through to `lubridate::as_date`, fails, and `convert_to` substitutes +`ERROR_VAL_DATE`. + +**Patient's row-alignment key does not need product's positional treatment.** +Measured across all 254 cleaned files: 84 of 85,325 rows (0.10%), in 7 files, +sit on a duplicated `patient_id` + `sheet_name`. Product's key was broken in a +different league (up to 35-way duplication, near-0% row-key match before +ticket 17). The identity key stays; the 84 rows are recorded as a known, +bounded population. **This settles [ticket 31](31-triage-patient-raw-residual-2.md)'s +version of the same question -- it must not be re-derived there.** + +### Because + +The source file settles the largest column outright. `hospitalisation_date`'s +own sub-header in the tracker template reads **"(Insert Date or NA)"** -- +verified directly in `2020_Mahosot Hospital A4D Tracker_DC.xlsx`, sheet +`Jan20`, column 24, where "NA" is what the clinic wrote on nearly every row. +A cell holding "NA" is the form being filled in as designed. Both pipelines' +*raw* stages agree on it (1,565 of the 1,607 cells carrying this shape were +traced back to a raw value of literal `"NA"` on both sides; the remainder +could not be joined to a raw row rather than contradicting it). The +divergence is created entirely at cleaning, by R. + +The wider marker set follows the same argument the map already accepted for +`r_numeric_error_sentinel` (ticket 29): a sentinel means "recorded but +invalid", which is the wrong claim for a blank, and it poisons every +downstream aggregate that does not know to exclude it. + +### Rejected + +- **Wiring `python_future_date_sentinel` across every date column.** This was + the ticket's own suggestion for `t1d_diagnosis_date` ("may simply apply, + which would be a wiring change rather than an investigation"), and it is + wrong. Of the 1,209 cells where Python holds the sentinel and R holds a real + date, only 722 have an R date beyond the tracker year -- the other 487 are + Python *failing to parse* something R parsed, which the future-date label + would have cemented as explained. Split out instead (see below). +- **Extending the numeric path's missing markers to the same word list.** + Correct in principle, but it moves production numeric output and would + invalidate ticket 29's exhaustive verification of `r_numeric_error_sentinel` + in the same session that depends on it. Named in **Not yet specified**. +- **Giving patient a positional row-alignment key.** Measured, not assumed: + 0.10% of rows. The cure costs more than the disease. + +### Evidence + +**Executed**, not read: + +- Full `a4d run patient --force` and `a4d run product --force` against the + real 254-tracker set on the USB drive (254/254 succeeded on both arms), then + a full `just compare-outputs` -- baseline run + `output/comparison/2026-08-14T204031Z`. +- Source-workbook reads: `2020_Mahosot Hospital A4D Tracker_DC.xlsx` (the + "(Insert Date or NA)" header and its column of "NA" values), and + `2022_Vietnam National Children_s Hospital A4D Tracker.xlsx` Patient List + (the 2023-dated cohort, below). +- Raw-vs-cleaned joins across all 254 files to attribute every sentinel-stamped + date cell to the source text behind it, before and after the fix. +- Full suite 663 passed / 1 skipped, `ruff check`, `ty check src/` all clean. + +**Judgement**, not evidence: which words count as an absence. +`nil`/`unknown`/`uncertain`/`?` are unambiguous; `unknwon`/`nill` are typo +spellings observed in this dataset and could be over-fitting. Recorded in +**Assumptions in force**. + +### Tense + +Every number below describes **current behaviour after this session's fix**, +measured on the 254-tracker set. The before-numbers are the state at +`output/comparison/2026-08-14T200342Z`. + +### Numbers + +| Measure | Before | After | +|---|---|---| +| Patient cleaned-stage `unclassified` | 8,259 | **6,652** (-19.5%) | +| Patient cleaned-stage total mismatches | 93,997 | 98,097 (+4,100, all newly *explained*) | +| Date cells carrying the sentinel in Python's own output | 6,186 | **1,994** (-68%) | +| ... of those, caused by an unrecognized missing marker | 4,377 | **0** | +| `hospitalisation_date` unclassified | 2,111 | 643 | +| `r_date_error_sentinel` rows | — | 5,707 | + +Total mismatches rising is the expected shape, not a regression: Python now +nulls where it previously agreed with R's sentinel by accident, so ~4,100 cells +became a *divergence* -- and every one of them lands in `r_date_error_sentinel` +rather than in `unclassified`. + +### A source-corruption finding, recorded not fixed + +`t1d_diagnosis_date`'s 530 unclassified rows are 43 patients x 12 sheets in a +single file, `2022_Vietnam National Children's Hospital A4D Tracker`. Its +Patient List records **2023** diagnosis dates for the whole cohort, and those +dates are internally consistent with the same sheet's D.O.B. and Age-at- +Diagnosis columns (VN_VC001: dob 2013-03-03, diagnosis 2023-07-16, age 10). +So neither pipeline misparsed anything -- the workbook is a 2022-named, +2022-sheeted tracker holding 2023 data. Python's future-date guard flags it +and R propagates it; per the map's Notes, **"the source is wrong, this tracker +needs human inspection" is the terminal answer here.** + +### Split off + +The 487-row parse-failure population and its production twin -- 1,809 date +cells across 296 distinct source texts where a real date sits inside a +clinical note (`"DKA 23 Oct 2020"`, `"Passed away 28/10/2019 due to DKA"`) and +Python discards it -- became [ticket +39](39-recover-dates-embedded-in-free-text.md). It is a production-behaviour +decision, not triage: R recovers some of these and is demonstrably wrong on +others (`"admitted ... DKA 7-15 Apr"` -> R gives 2015-07-01), so "do what R +does" is not available as an answer. diff --git a/docs/wayfinder/tickets/39-recover-dates-embedded-in-free-text.md b/docs/wayfinder/tickets/39-recover-dates-embedded-in-free-text.md new file mode 100644 index 0000000..da9ba06 --- /dev/null +++ b/docs/wayfinder/tickets/39-recover-dates-embedded-in-free-text.md @@ -0,0 +1,317 @@ +--- +id: 39 +title: Decide whether a date buried inside a clinical note should be recovered or discarded +labels: [wayfinder:grilling] +status: closed +blocked_by: [] +assignee: session-2026-08-21 +claimed_at: 2026-08-21 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 38 +--- + +## Premise + +Rests on [Triage the patient cleaned-stage date-column family (round +3)](38-triage-patient-cleaned-date-family.md), closed, which established three +things this ticket depends on: + +- **A date cell that records an absence is now nulled, not sentinelled.** + `MISSING_VALUE_MARKERS` / `DATE_ABSENCE_MARKERS` / + `DATE_PLACEHOLDER_FRAGMENTS` (`src/a4d/clean/date_parser.py`). That removed + 4,377 of the 6,186 sentinel-stamped date cells, so what remains is no longer + contaminated by blanks -- this ticket's 1,809 rows really are cells where + *something* was written. +- **R is not a usable oracle here.** R's `parse_dates` recovers a date from + some of these strings and is demonstrably wrong on others: `"admitted to + Pokaku Hosp due to DKA 7-15 Apr"` -> R gives `2015-07-01`; `"DKA: admitted + 6-12 Nov 2020"` -> R gives `2020-12-06` (the range's two numbers read as + day and month); `"9-Dce-20"` -> R gives `2020-09-01`, discarding the + transposed "Dce". So "match R" is not available as an answer, and any + divergence this ticket creates from R needs its own justification. +- **Ticket 37 already added a longest-parseable-prefix fallback**, which + handles a date at the *start* of the string (`"16-Nov-2019 due to DKA"`). + This ticket is about the other direction -- the note comes first and the + date is inside or at the end. + +Overturned only if the destination stops requiring that Python's own output be +defensible on its own terms; it survives R's retirement (ticket 12) intact, +because it is a question about Python's behaviour, not about parity. + +## Question + +**1,809 cleaned date cells across 296 distinct source texts currently become +the error sentinel because a real, legible date is embedded in a clinical +note.** Measured on the real 254-tracker set after ticket 38's fix, largest +first: `t1d_diagnosis_date` 559, `hospitalisation_date` 540, `bmi_date` 126, +`fbg_updated_date` 124, `hba1c_updated_date` 86, then the four +complication-screening date columns and a tail. + +Representative texts, all of which a human reads without hesitation: + +``` +DKA 23 Oct 2020 +Passed away 28/10/2019 due to DKA +DKA 30-Nov-2020 (Meikhtila hospital) +7/04/2021: DKA +22nd Feb.2019(DKA) +May'21: Rx at Emergency room Hyperglycemia with Ketosis +``` + +And the ones that make this a decision rather than a task: + +``` +DKA: admitted 6-12 Nov 2020 (a range -- which endpoint?) +DKA 2020: June, Aug, Nov (three events in one cell) +admitted ... due to DKA 7-15 Apr (a range, and no year at all) +DKA 2019 (a year, no month -- invent Jan 1?) +10-มค-2025 (Thai month abbreviation) +``` + +Decide, with a recommendation rather than a survey: + +1. **Do we recover these at all**, or is the sentinel plus the error log the + right home for "a date is in here somewhere"? Note what is lost either way: + discarding loses a real hospitalisation date the clinic recorded; recovering + risks asserting a precise date the source did not state. +2. **If we recover: how strict?** An anchored, unambiguous pattern (day + + month-name + 4-digit year, and nothing that looks like a range) recovers + the clean majority and refuses the ambiguous tail. dateutil's `fuzzy=True` + recovers more and invents more -- it is how R gets `2015-07-01` out of + `"7-15 Apr"`. +3. **Where does an ambiguous cell go?** Sentinel, null, or first-endpoint-with- + an-error-log-entry are all defensible; they are not the same claim. +4. **Does `hospitalisation_date` want a different answer from the rest?** It is + the only column where clinicians routinely write prose -- the template's own + sub-header says "(Insert Date or NA)" and they write a case note anyway. + `t1d_diagnosis_date`'s 559 rows may have a different shape entirely and are + not yet characterized. + +Whatever is decided, the corresponding cause needs naming in +`src/a4d/migration/compare.py` so the R-vs-Python report stops carrying these +as `unclassified` -- 487 of them currently sit there. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: it is not +enough to explain a difference and name a cause. Each one must also carry an +explicit verdict on **whether Python is doing the right thing**, with the +evidence that establishes it (source Excel, or R's/Python's own code). Where +Python turns out to be wrong or to be losing information the source file +carried, fix the pipeline rather than labelling the symptom. A cause genuinely +undecidable on available evidence is recorded as an open question, not closed +with a label; where the evidence shows the source file itself is corrupt, +"the source is wrong, this tracker needs human inspection" is a legitimate +final conclusion. + +## Addendum (2026-08-19c, from [round 6](53-triage-patient-cleaned-residual-6.md)) + +Round 6 measured `hospitalisation_date`'s whole cleaned-stage residual -- 546 +cells at the time, 489 after that round's parser fix -- by joining every +flagged cell back to Python's raw stage and grouping by the source value. +**Every distinct source is a clinical note; the "mangled date token and nothing +else" kind that earlier rounds expected alongside them does not appear in this +column at all.** So this ticket owns the entire column, not a share of it. + +The population runs in three directions, and the third is new to this ticket: + +- **Python sentinels, R has a date (302).** `DKA - Feb-2020`, `DKA; Jul 2018`, + `Passed away 28/10/2019 due to DKA`, `DKA 2020: June, Aug, Nov`. The + direction this ticket was written about. +- **R sentinels, Python has a date (179).** `April 19 (due to very high + Hba1c)` -> Python 2019-04-01, `19th-29th Aug 2019` -> 2019-08-19, + `7/2020 DKA Admit Ratchaburi Hospital 5 Days` -> 2020-07-01. **Python already + recovers dates from free text here**, via ticket 37's longest-parseable-prefix + fallback, and R does not. So the question is not "should Python start doing + this" -- it partly does, inconsistently, depending on where in the string the + date sits. +- **Both find a date and they differ (65).** `May 2019, Dec 2019 DKA, August + 2020 DKA`, `Dec 2019, Mar 2020 DKA Jan 2021 DKA`. The cell records *several* + admissions and the two pipelines pick different ones. + +That last group adds a sub-question this ticket should decide alongside the +first: **when a note records more than one date, which one does a single date +column mean** -- the first, the latest, or is a single column simply the wrong +shape for the data? Answering "recover" without answering this leaves the +recovery non-deterministic in exactly the cells clinicians cared enough to +write out. + +Round 6 deliberately left all 489 cells `unclassified` rather than naming a +cause for them: a label would read as "understood and settled" while this +decision is open, and would hide the largest single column on the cleaned-stage +report from the next round's re-measurement. + +## Resolution (2026-08-21) + +### Decision + +**A date buried inside a clinical note is recovered, by an explicit pattern +recogniser that refuses anything it does not recognise.** Seven parts, all +settled with the user in session: + +1. **Recover, rather than discard.** The choice was never "start recovering or + not" -- it was which way to make an inconsistency consistent. Python already + recovered a date at the *front* of a note (ticket 37's prefix walk) and + discarded the identical date in the middle, so `"16-Nov-2019 due to DKA"` + was published and `"DKA 16-Nov-2019"` was sentinelled. +2. **Recognition is an explicit, anchored pattern set** (`_DATE_IN_TEXT`, + `recover_date_from_text`, `clean/date_parser.py`), not `dateutil(fuzzy=True)` + and not a windowed generalisation of the prefix walk. +3. **A note naming several dates publishes the first**, and the discard is + logged and reported. +4. **Missing components**: no day -> the 1st; no year -> the row's tracker year. +5. **Everything else keeps the `9999-09-09` sentinel.** Per ticket 38's line: + null means nothing was recorded, the sentinel means something was recorded + that cannot be read as a date. Every refused cell is the second. +6. **The recogniser lives in `parse_date_flexible`**, so all 18 date columns + get it, even though prose occurs in only one. +7. **Three new warning-level error codes** -- `date_recovered_from_text`, + `date_multiple_in_cell`, `date_year_inferred` (`errors.py`, emitted by + `_log_text_recoveries` in `clean/converters.py`). Refused cells keep + `type_conversion`/`invalid_value` unchanged. + +### Because + +The pattern set was built **after** measuring the column rather than from the +six example strings this ticket was written with -- the user's call, and it +changed three of the answers. The scan (`output/analysis/ticket39_hospitalisation_date_sources.xlsx`, +745 distinct strings over 7,847 non-null cells in 240 trackers) found: + +| shape | cells | +|---|---| +| already parsed | 4,734 | +| `NA` / `-` / blank -> nulled | 1,736 | +| words, no digits (mostly `Nil` + template placeholder) | 763 | +| **digits, unparsed** | **614** | + +and within that 614: one full date inside prose 302, two or more date tokens +158, month+year only 45, day+month with no year 31, range sharing a month 30, +and **48 cells with digits and no date at all** (`3 month come back meet +Doctor`, `check 2 or 3 time per/day`, `on stamlor 5mg`). That last row is what +disqualifies fuzzy parsing on evidence rather than on taste: `dateutil` does not +decline on any of them, it returns a date built from `datetime.now()` -- the +same failure this parser has already been bitten by in tickets 50, 53 and 56. + +The header itself was read from the source workbook rather than assumed: +`2021_Preah Kossamak`, sheet `Feb21`, row 65 -- *"Hospitalisation due to +diabetes emergency or glucose control (Include Date)"*. It is a free-text +clinical field whose only instruction is to include a date, which is why this +column and no other carries prose. + +The cells are also **patient-level history, not month events**: `DKA - Feb-2020` +appears in all 12 month sheets of 2021 PKH for `KH_PK010`, `22/7/2020-26/7/2020` +in all 12 of 2021 Chiang Mai for `TH_CP019`. The clinic writes the note once and +copies it down. + +### Rejected + +- **Discard everything, remove the prefix walk too** -- consistency bought by + deleting 158 correct dates from production. +- **`dateutil(fuzzy=True)`** -- see the 48 no-date cells; it is also how R gets + `2015-07-01` out of `"7-15 Apr"`. +- **Windowed prefix search** -- sounds like the smaller change and is the + dangerous one: `_parse_date_str` ends at `dateutil`, so a window like + `"5 Days"` parses, and the invented-component problem returns by the side door. +- **Latest date rather than first**, for a multi-date note -- defensible where a + tracker's header says "Last hospitalisation" (`synonyms_patient.yaml:202`), but + that wording is a minority and choosing on it would make one string mean + different things in different files. +- **Refusing multi-date cells outright** -- removes currently-published values + and is not more honest, since the first date *is* recorded in the cell. +- **Nulling the unreadable cells** (option B/C on the sentinel question) -- the + user's reasoning: the sentinel is a flag that something was written, was not + empty, and could not be parsed, which is precisely what wants investigating. +- **Column-scoping the recogniser to `hospitalisation_date`** -- a second code + path behaving differently for reasons invisible at the call site. The + user's reasoning: the patterns are general date patterns, correct anywhere. +- **Recognising a bare year inside prose.** `"DKA 2019"` names no month and a + month cannot be recovered from anything. The whole-cell bare-year convention + (ticket 52) rests on a clinic writing *only* a year, which this is not; and + refusing it is what lets `"DKA 2020: June, Aug, Nov"` avoid becoming 1 January. + +What the chosen route gives up: each new spelling of a date needs a pattern, and +Python now recovers less than R does on a handful of cells R happens to guess +right. + +### What it moved + +On the real 254-tracker set, `hospitalisation_date`: + +- **673 sentinels -> 156** in cleaned output (5,406 non-null cells). +- **940 cells** logged `date_recovered_from_text`, **240** `date_multiple_in_cell`, + **40** `date_year_inferred`. Only 7 of the 1,223 entries land outside + `hospitalisation_date` (5 `bmi_date`, 2 `last_clinic_visit_date`). +- R/Python cell mismatches for the column: **478 -> 0 unclassified**, split + across three new causes -- `python_reads_date_in_clinical_note` (332), + `note_dates_read_differently` (134), `r_invents_january_from_bare_year` (24). +- Whole patient cleaned stage: 114,371 -> 114,240 mismatches; unclassified + 3,432 -> 2,955, of which 2,935 is ticket 44's FBG population. + +Old parser vs new over **all 18,186 distinct (date string, tracker year) pairs** +in the raw output: 117 strings changed, 553 cells. All but five are +sentinel-to-real-date. The five reading *changes* are all corrections: +`19th-29th Aug 2019` 2019-08-29 -> 2019-08-19 and `19th-28th Oct 2018 (DKA)` +2019-10-28 -> 2018-10-19 (the admission day, not the discharge day, and the +second also had the wrong *year*), `3-9 Sep 2020` 2003-09-09 -> 2020-09-03 (x2 +files), `4-13 Oct 2024` 2004-10-13 -> 2024-10-04. + +### Three defects found while building it + +- **A bare range with no prose was misread, not refused** -- and this was + pre-existing, not introduced here. `dateutil` takes the range's first number + for a year, so `"6-12 Nov 2020"` was published as **2006-11-12** and + `"3-9 Sep 2020"` as **2003-09-09**. Fixed by consulting the range patterns + first when the cell *opens* with one. +- **The prefix walk beat the recogniser to the range cells**, because it runs + earlier and ends at `dateutil`. The recogniser now runs before it: it is the + explicit reading, and the walk is the guess. +- **`_is_range` was checking `match.lastgroup`**, which names the last *inner* + group to match (`rn_year`), never the outer alternative -- so ranges were + never flagged `date_multiple_in_cell`. Caught by reading the real error table, + not by a test. + +### Evidence + +**Executed.** The 745-string scan, the 18,186-pair old-vs-new parser sweep, two +full patient-arm runs against the real 254-tracker set on the USB drive, three +comparison runs against the frozen `output_r/`, and a direct read of the source +workbook's own header cell. 888 tests pass; ruff and `ty check src/` clean. +Tense: every number above is measured current behaviour, not a projection. + +The one claim resting on **judgement** is the year-inference rule -- see +*Assumptions in force* on the map. + +### Left open, deliberately + +- **One `bmi_date` cell is newly unclassified.** Source `0ct/19` (a zero for the + letter O); typo rescue makes it `OCT/19`, which Python now reads as + 2019-10-01 and R sentinels. Python is right -- R's parse order list cannot + express a slash-separated month/year. Not classified because the honest home + for it is a widened `r_parse_order_cannot_read_cell`, whose own docstring says + the day>12 signature is what keeps it from over-claiming. One cell. +- **`25Jul-2Aug2022`** (1 cell): a range written with no separators at all. + Neither pipeline reads it correctly -- R gives 2022-02-25, Python the + discharge day 2022-08-02. The source is what needs correcting; it goes to + ticket 40. +- **7 recovered cells land past their tracker year** (`39 Aug 2022`, + `DKA Jul'29`). Both are source typos; the beyond-tracker-year guard sentinels + them, so nothing wrong reaches output. Also ticket 40's. + +### Findings routed elsewhere + +- **The template's own placeholder text is stored as a cell value in ~170 cells** + (`Insert Date` 65, `Insert Date or NA` 60, `NA or Hospitalisation Date` 44). + Harmless today -- ticket 38 already nulls them -- but a source defect, so it + goes to [ticket 40](40-source-defect-findings-report.md). +- **Buddhist-Era years appear in this column** (`18-19/11/2567`, + `19-22/5/2568`). Refused, since converting them is a separate decision about + the workbook. Also ticket 40's. +- **The ticket's own claim that `t1d_diagnosis_date` (559 cells) belongs to this + population is false**, and was checked rather than inherited: every one of its + sentinels traces to a clean, unambiguous source date rejected by the + future-date guard -- the corrupt 2022 VNCH column round 4 already decided -- + plus one BE `2560-01-01`. No free text is involved. The same check found the + other date columns' unparsed values are typos (`9-Dce-20`, `00/00/2020`), not + prose, so this ticket's population really is one column. diff --git a/docs/wayfinder/tickets/40-source-defect-findings-report.md b/docs/wayfinder/tickets/40-source-defect-findings-report.md new file mode 100644 index 0000000..75ba3e4 --- /dev/null +++ b/docs/wayfinder/tickets/40-source-defect-findings-report.md @@ -0,0 +1,386 @@ +--- +id: 40 +title: Produce one Excel of every source-tracker defect, so the trackers themselves can be corrected +labels: [wayfinder:task] +status: open +blocked_by: [] +assignee: null +claimed_at: null +resolution: null +evidence: null +closed_by: null +spawned_by: 30 +--- + +## Premise + +Rests on [Triage the patient pipeline's raw-stage column-existence +divergence](30-triage-patient-raw-column-divergence.md), closed, where the user +set the principle this ticket exists to serve: where a divergence comes from a +human error in the source workbook, **the right fix is the source tracker, not +inference in the pipeline** -- cheaper, safer, and permanent. That only works +if every such finding is written down somewhere a person can act on. + +Rests also on the map's standing preference that triage means deciding: across +tickets 18-38 this map has accumulated a large body of confirmed +source-data defects, each currently recorded only in prose on its own ticket, +or as a row in a per-run comparison report that will disappear when R is +retired ([ticket 12](12-retire-r-workspace.md)). + +Concrete findings already waiting for such a report, all source-verified: + +- **26 trackers with headerless columns holding data** (ticket 30), 4,572 + values, of which the clearest is `2021_Kantha Bopha`, sheets `Mar21`/`Apr21`, + column Q -- 194 insulin-regimen values lost because the header cell is empty. + Now emitted at runtime under the `blank_header_with_data` error code. +- **37 trackers that change shape mid-year**, 284 column positions + (`tracker_layout_changed`, ticket 30) -- a tracker is one workbook for one + clinic-year and should hold one layout throughout. Includes `2018_CDA`, whose + April sheet is missing the `Insulin Regimen` column outright, and `2020_CDA`, + whose June header edit relabels `Baseline FBG` from `mmol/dL` to `mg/dL` over + unchanged values -- costing that clinic five months of baseline FBG (the + `mmol/dL` spelling maps to nothing) and mis-filing five months of updated FBG + under `fbg_updated_mmol`. +- **A corrupt tracker**: one 2022-named, 2022-sheeted tracker whose Patient + List holds 2023 diagnosis dates for all 43 patients ([ticket + 38](38-triage-patient-cleaned-date-family.md)). +- **Buddhist-Era years typed into Gregorian date cells** ([ticket + 27](27-triage-patient-raw-residual.md)). **Superseded as a defect by [ticket + 61](61-decide-buddhist-era-date-conversion.md)**: a BE year is the calendar a + Thai clinic uses, and the cleaned stage now converts it (375 patient cells, + 22 product rows). What remains reportable is only the era years that decode + to nothing -- see the two below. +- **Excel formula errors cached in source cells** (`r_formula_error`, ticket 27). +- **Stray date/time-formatted cells in numeric columns** ([ticket + 24](24-triage-remaining-raw-column-residual.md)). +- **23 patients listed twice on the same monthly sheet** ([ticket + 45](45-patient-row-alignment-duplicate-keys.md)), with different data in each + copy: 21 on `2024_Vietnam National Children`'s `Jul24` (two lists spliced + together -- the numbered column runs 1..78 while the IDs repeat), one on + `2023_Vietnam National Children's` `Jun23` (`VN_VC026`), one on `2018_Penang + General Hospital_DC` `Oct18` (`MY_PN004`). Derived from the pipeline's own + raw output, so this one is already machine-derivable per file, sheet and + `patient_id`. +- **Two trackers whose R output carries `patient_id = "#REF!"`** -- 98 rows in + `2026_Preah Kossamak` `May26`, 5 in `2026_Quirino` `Jan26` (ticket 45). A + broken Excel reference in the source, visible because R preserves it where + Python drops the rows. +- **Numeric entries typed into date-formatted cells** ([ticket + 46](46-triage-patient-raw-residual-4.md)), 26 patient cells: `2025_Hat Yai` + `Annual!H` (12 sheets x systolic 120), `2025_YGH` (systolic 80), + `2020_Mahosot` (`testing_frequency` 2). Excel stored each as a 1900 date, so + the cell displays a date where a reading belongs -- the format is what needs + correcting, not the value. The pipeline now recovers the number, so this is + a workbook-tidiness finding rather than data loss. +- **Dates typed into numeric columns** ([ticket + 46](46-triage-patient-raw-residual-4.md)): `2023_Chiang Mai Maharaj Nakorn`, + `Patient List!H` for `TH_CP005`, "Age at Diagnosis" holds a + Buddhist-formatted date (Excel serial 20668) against a D.O.B. of 2009 -- 16 + rows. Neither pipeline can recover an age from it; the cell needs a human. + `2020_Mahosot` and `2022_Mahosot` `blood_pressure_mmhg` carry the same shape. +- **Rich-text cells whose space sits in its own formatting run** ([ticket + 46](46-triage-patient-raw-residual-4.md)): `2017_Yangon`'s + `hba1c_updated`/`fbg_updated_mg` ("8.8 (20.9.16)") and `2022_Mahosot`'s + `observations`. Python reads them correctly, so this is cosmetic for the + pipeline -- but it is also why the same value exists in two forms in one + column, which is worth flagging to whoever maintains the workbooks. +- **More dates typed into numeric columns** ([ticket + 50](50-triage-patient-raw-residual-6.md)), extending the ticket-46 entry + above with the two cells that finished the raw stage: `2023_Yangon General` + `Patient List!H76` for `MM_YC043_YG`, "Age at Diagnosis" holding a + `d-mmm-yyyy`-formatted 2017-05-04 (4 rows); `2021_Khon Kaen` `Mar21!P60` for + `TH_KN008`, "Testing Frequency (per day)" holding a `d-mmm`-formatted + 2021-02-01 (1 row). Same verdict as the ticket-46 cases: no number is + recoverable, the cell needs a human. +- **A clinical note typed into a date column** ([ticket + 50](50-triage-patient-raw-residual-6.md)): `2022_Kantha Bopha`, + `May'22!Y161` and `Jun'22!Y161` for `KH_KB089`, the hospitalisation Date + column holding the text "on stamlor 5mg" -- a medication note, not a date. + Both pipelines carry the text through the raw stage unchanged. +- **A Buddhist-Era year in a date cell that decodes to a plausible Gregorian + date** ([ticket 50](50-triage-patient-raw-residual-6.md)): `2022_Hat Yai` + `Patient List!G25` for `TH_HY013` holds 2560-01-01 in a `d-mmm-yyyy` cell. + The patient's own D.O.B. (2013-03-17), recruitment (2017-06-01) and age at + diagnosis (4) all agree the intended date is 2017. **[Ticket + 61](61-decide-buddhist-era-date-conversion.md) now recovers it**: the band + test converts 2560 to 2017 without being told, matching the answer this + report had derived by hand. Still worth reporting so the workbook is + corrected, but no longer data the pipeline loses. +- **Three more header defects Python works around and R does not** ([ticket + 50](50-triage-patient-raw-residual-6.md)), each costing R real data and each + worth correcting at source: `2022_Kantha Bopha`'s month sheets open the + hospitalisation header (`X71`) with thirteen leading spaces; `2019_Preah + Kossamak`'s `Jul19` and `Aug19` sheets have lost every header merge the other + ten sheets still carry, leaving "Date" sub-headers unqualified (and a stray + `3` typed into `L63`); `2025_LWCH` leaves the current-month clinic-visit + column (`D`) unheaded in both header rows. +- **113 product groups across 21 files where the computed closing balance + contradicts the tracker's own recorded total** (`balance_reconciliation`, + [ticket 36](36-triage-product-cleaned-unclassified-residual.md)). + +## Question + +Decide what this report is and build it. The user's stated shape: **one Excel, +every finding, with tracker file, sheet, `patient_id`, row and the exact +finding**, so the data team can go and correct the source workbooks. + +Points to settle before building: + +1. **Source of truth.** Derive it from the pipeline's own error log / errors + table (which already carries error codes, file and sheet) rather than + hand-collecting from ticket prose -- otherwise it is a hand-maintained list + that drifts. Check what the existing `errors`/`logs` tables already capture + per finding and what is missing (row number and `patient_id` in particular). +2. **Which findings belong.** Every error code, or only the ones a human can + act on in the workbook? A type-conversion failure on a cell a clinician + mistyped is actionable; an internal pipeline warning is not. +3. **Where it lives and when it runs.** A new `a4d` CLI command (this outlives + R, unlike `scripts/compare_outputs.py`), part of `create tables`, or a + separate report step. +4. **Whether it supersedes anything.** [Ticket + 16](16-log-analyzer-drill-down.md) asks for a drill-down view into one + tracker's errors; these two may be the same deliverable seen from two + angles, or complementary (per-file interactive vs. whole-set actionable). + Decide that rather than building both blind. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference and the +destination's "every difference explicitly decided": a finding in this report +must be precise enough that someone can open the named workbook, find the named +cell, and see the problem -- not a category label. + +## Findings added by session-2026-08-19 ([round 4](51-triage-patient-cleaned-residual-4.md)) + +Two more source defects, both verified in the workbook and both already +producing a pipeline error the report can derive from: + +- **2022 Vietnam National Children's Hospital: the whole `Date of T1D + Diagnosis` column is a year out.** Every cell in `Patient List!G` holds a + 2023 date -- `G13` = 2023-07-16 for VN_VC001, who was born 2013-03 and + recruited 2017-07 -- in a tracker filled in during 2022. The patients are + recorded as diagnosed a year after the tracker was written and five years + after they were recruited for having the disease. Python's `_validate_dates` + already logs each one as `invalid_value`; 530 cleaned-stage cells. +- **Misspelled month names inside otherwise well-formed dates.** `9-Dce-20` + (2021 Mahosot, 81 cells), `25-Ma4-2025` (18), `4-Okt-2023` (3), plus + malformed separators `26/102022` (5) and `19-Jan_2023` (5). Python refuses + them and stamps the date sentinel, logging `invalid_value`; R guesses and + gets them wrong (it reads `9-Dce-20` as 2020-09-01, dropping the misspelled + month and promoting the day into the month slot). The intended dates are + legible to a human, which is exactly why the fix belongs in the workbook. + +Also worth carrying, though it is a *province* rather than a date: five VNCH +trackers write `Thái Nguyễn` where the allowed list has `Thái Nguyên`, and two +more spellings (`Thai Nguyen`, `Thai nguyen`) that **neither** pipeline +recovers -- those rows are "Undefined" in both, so the comparison never flags +them but the data is lost all the same. + +## Findings added 2026-08-19c (from [round 6](53-triage-patient-cleaned-residual-6.md)) + +Two more source-workbook defects, both found while triaging the patient cleaned +stage and both deliberately left uncorrected in the pipeline: + +- **A date typed into the diagnosis-age column.** 2023 Chiang Mai Maharaj + Nakorn records `1956-08-01` in `t1d_diagnosis_age`; R carries the Excel + serial (20668) through into the age, Python nulls it. The cell needs a number + of years, not a date. +- **A date broken by a stray space inside it.** `26-05- 2007` is legible to a + human as 26 May 2007 but parses as neither pipeline's answer — R and the old + Python both produced a wrong date, and Python now produces the error + sentinel. Round 6 rejected normalizing whitespace around date separators as + an untested third change in one session, so the workbook is the fix. + +## Findings added 2026-08-19d (from [round 7](54-triage-patient-cleaned-residual-7.md)) + +Round 6 already logged 2023 Chiang Mai's date-in-the-age-column above. Round 7 +confirmed it against the workbook itself and found the wider defect behind it, +plus a second clinic with the same shape: + +- **2023 Chiang Mai's `Date of T1D Diagnosis` column is empty for every + patient**, which is why its formula-derived `Age at Diagnosis*` column reads + `#NUM!` throughout and why TH_CP005's stray `1956-08-01` had nothing to be + computed from. Fixing the one cell without filling the diagnosis-date column + leaves the whole column broken. +- **2023 Yangon General records a diagnosis a year before the birth.** + MM_YC043_YG has D.O.B. 2008-01-01 and `Date of T1D Diagnosis` 2007-06-01, + with `2017-05-04` typed into the age column on top. R carries that date's + Excel serial (42859) into the age; Python nulls it. Two defects in one row. +- **Heights that are not heights.** 2025/2026 Señor Sto. Niño, 2021-2023 Likas + and 2019 Mahosot record `6.9`, `2.52`, `2.43`, `2.72`, `13.0` in the height + column. R sentinels them; Python converts them to metres and emits `0.069`, + `0.0252`. Whether these are a mis-keyed unit or plain data entry errors is + [round 8](55-triage-patient-cleaned-residual-8.md)'s question, but no reading + of them is a plausible height. + +## Findings added 2026-08-20 (from [round 10](57-triage-patient-cleaned-residual-10.md)) + +Round 10's whole residual is source defects: 37 of its 72 remaining cells are a +date cell damaged past reading, and every one of them is a workbook fix rather +than something either pipeline can infer. + +- **`25-Ma4-2025` in a `Date Lost to Follow Up` cell.** 2025 Taunggyi, 18 cells + (one value, repeated down the months). "Ma4" is Mar or May and nothing in the + workbook decides which; R reads the embedded `4` as the month and publishes + 2025-04-25, a third answer neither spelling supports, and Python declines. + The same row also carries `status = Active` while holding a lost-date at all, + so two cells need attention. +- **Dates with a separator swallowed or a digit group glued.** `26/102022` + (2022/2023 Sunprasitthiprasong), `8/1023` (2023 Yangon General), `10/1023`, + `3/10.23` (2023 North Okkalapa), `10-Oct-2-24`, `13-Mar-0202` (2023 Penang + General). Where the missing separator belongs is a guess: R's frozen output + reads `10/1023` as 2010-10-23 while running R's own parser over that string + in isolation returns 2023-10-10 -- the same input, two answers, so R's number + is not evidence of the clinic's intent either. +- **The tracker template's own example text left in a patient row.** 2018 + Penang General DC, MY_PN001, `e.g. xxx (mth-18)` in both the HbA1c and FBG + measurement cells across Jul18/Aug18/Sep18. R turns it into 2018-01-01. +- **A fasting-glucose reading typed into the HbA1c column.** 240 and 125, plus + 299 at 2024 Preah Kossamak (KH_KB050, Jun24-Aug24). R has no range check on + this column and publishes them as HbA1c percentages; Python rejects them + against the declared 0-25 bound. The numbers look real -- they are in the + wrong column. +- **A date typed into `testing_frequency`.** 2021 Khon Kaen, TH_KN008, Mar21 + holds 2021-02-01 where a tests-per-day count belongs. R publishes the Excel + serial 44228 as the frequency. +- **Two baseline-FBG readings that cannot be read in the unit their column + turned out to hold.** 2020 Kantha Bopha's `Baseline FBG` column is 90.1% + sub-30, so the pipeline reads the whole column as mmol/L (ticket 42) -- and + KH_KB056's 47.8 and KH_KB062's 53.8 are then past the analytical ceiling and + are rejected. Read as mg/dL they would be ordinary. Either those two cells + are mg/dL in an otherwise-mmol column, or they are mis-keyed; the workbook is + the only place that can say. +- **Patient IDs that do not match the template's `XX_YY###` format**, found + while closing [ticket 47](47-patient-ids-merged-at-cleaning.md). These are + identity defects, not measurement defects, so they are worth listing first in + the report: an unrepairable one costs the clinic a whole patient's history. + - `2023_NPH A4D Tracker`, sheet `Sep23`, rows 128-131: `KH_NPH026`, + `KH_NPH027`, `KH_NPH028`, `KH_NPH029` carry a stray `H`. The same + workbook's `Patient List`, `Oct23`, `Nov23` and `Dec23` sheets all spell + the same four patients `KH_NP026`-`KH_NP029`. The pipeline now recovers + these, so the data is not lost -- but the workbook should be corrected so + it stops relying on recovery. + - `2026_NOGH T1D Tracker_June_26`: `MM_NO97`, `MM_NO98`, `MM_NO99` are + 7 characters where the template wants 8, in the `Patient List` **and** + every month sheet. The workbook has `MM_NO090`-`MM_NO096` and no + `MM_NO097`, so nothing in it says whether the intended IDs are `MM_NO097` + -`MM_NO099` or something else. **Unrepairable in the pipeline**: three + distinct patients, 18 rows, all published as `Undefined` today. This one + needs a human at the clinic. + - `2026_YGH T1D Tracker_June_26`: `MM_NO55_MW_YG` reduces to `MM_NO55`, + again 7 characters. One patient, 3 rows, published as `Undefined`. Same + situation as above. + - `2021_Mahosot Hospital A4D Tracker_DC`: `LA-MH056`, `LA-MH057`, + `LA-MH058` on the Jun-Nov sheets vs `LA_MH056`-`LA_MH058` on `Dec21` -- + the same three patients spelled two ways inside one workbook. The pipeline + merges them correctly; the workbook should still be made consistent. + - `2026_Surat Thani Hospital A4D Tracker_Jun_26`: `TH-ST029` and `TH_ST029` + both appear on `May26`, so that patient ends up with two rows for one + month once the spellings are normalized. Likely a duplicated entry. + +## Findings added by [ticket 39](39-recover-dates-embedded-in-free-text.md) (2026-08-21) + +All four sit in `hospitalisation_date` and were measured on the real +254-tracker set. None is repairable in the pipeline; each needs the workbook +changed. + +- **The template's own instruction text saved as data, ~170 cells.** + `Insert Date` (65), `Insert Date or NA` (60), `NA or Hospitalisation Date` + (44), `DKA - ` (1). The pipeline nulls them (ticket 38), so + nothing is published wrongly -- but a data row holding the form's own + placeholder means the cell was never filled in, and the clinics should be + told which rows those are. +- **Buddhist-Era years in a Gregorian date field, inside a note naming a date + *range***: `18-19/11/2567`, `19-22/5/2568`. Two cells, both + `hospitalisation_date`. [Ticket + 61](61-decide-buddhist-era-date-conversion.md) settled that a BE year is + converted, not refused -- but these carry a range as well, and picking one of + two admission days is a convention [ticket + 39](39-recover-dates-embedded-in-free-text.md) declined and this decision did + not overturn. Left unconverted on the range, not on the calendar; the fix is + the clinic writing one Gregorian date. +- **Era years that decode to no plausible date**, 6 cells, both + `fbg_updated_date` ([ticket + 61](61-decide-buddhist-era-date-conversion.md)): `2025_CDA` `3035-03-01` + (1 cell, `KH_CD016`) and `2025_Surat Thani` `5025-05-19` (5 cells, + `TH_ST003`). Subtracting 543 gives 2492 and 4482, still years in the future, + so neither is a Buddhist-era date the band can recover. Both stay sentinelled + and are logged `invalid_value`; the cells need a human. +- **A stay written with no separators at all**: `25Jul-2Aug2022`, 1 cell. + Neither pipeline reads it correctly -- R gives 2022-02-25, Python takes the + discharge day 2022-08-02 -- because with the separator missing the opening + `25Jul` cannot be told from a token whose year is unreadable. +- **Two impossible dates that a plausible-looking reading nearly hid**: + `39 Aug 2022` (2022 CDA) and `DKA Jul'29` (2021 Mahosot), 7 cells. Both + recover to a date past their own tracker year (2039, 2029) and are then + sentinelled by the beyond-tracker-year guard, so nothing wrong is published. + `39 Aug` is a day that does not exist; `Jul'29` is almost certainly `'19` + or `'20` mistyped. + +## Findings added by [the classifier re-audit](32-audit-classifiers-against-decision-bar.md) (2026-08-22) + +All three are product-side, all measured on the real 254-tracker set, and all +now emit a pipeline error record this report can derive from. + +- **Corrupt Excel serials in entry-date cells**, `implausible_era_date`, 3 cells. + `2024_Chiang Mai Maharaj Nakorn`, `Aug24`: serial **1,339,576**, which reads as + `5567-08-19`. `2026_Penang General Hospital_Jun_26`, `Apr26`: serial + **411,384**, reading as `3026-04-30`. `2025_Hat Yai Hospital`, `Oct25`: + `2525-10-02`, where the Buddhist year for 2025 is 2568 -- so this one is a + mistyped BE year rather than a serial. In each case the day and month match + the sheet, so the intended date is legible to a human; the year is not + recoverable by inference, which is why the pipeline now sentinels them. +- **Entry dates from the following year in a December sheet**, 78 cells across + 26 files, the largest being `2023_Sarawak General Hospital`'s `Dec23` sheet + (25 cells, running 2024-12-01 to 2024-12-28). Python's beyond-tracker-year + guard already flags each one under `invalid_value`. Either the rows belong in + the next year's tracker or the year was mistyped; the workbook is the only + place that says which. +- **Entry dates decades before the tracker**, 29 cells, logged by the same + guard's year-floor branch and deliberately preserved in output. + `2023_Surat Thani` holds `0202-06-20` and `2025_Quirino` `0205-12-02` -- year + 202 and 205 -- alongside `1935-04-30` (2025 NOGH), seven `2009-12-04` (2019 + Mahosot) and four `2004-05-08` (2024 Sarawak). + +**One finding about this report's own source of truth, now fixed.** Point 1 of +the Question above assumes the errors table carries every finding. It did not: +`a4d run` published an errors table holding the **patient arm only**, because +the patient arm writes it from inside `run_patient_pipeline` and nothing wrote +the product arm's. Measured before the fix: 63,295 records with zero +`balance_reconciliation` (116 exist) and zero `implausible_era_date`. So the 113 +product balance-reconciliation groups this ticket already lists were never +actually reachable from the table. `run` now writes it once after both arms: +63,295 -> 97,326 records. + +## Findings added by [rows that pair with nothing](59-triage-unmatched-row-keys.md) (2026-08-24) + +- **Patient ID cells holding a broken formula, 120 rows across 9 trackers**, + now emitted under the new `excel_error_patient_id` code. Every one of these + rows is discarded -- the patient cannot be identified -- so the clinic's + measurements for that row never reach BigQuery. Two trackers hold nearly all + of it: `2026_Preah Kossamak Hospital`'s `May26` sheet is **98 rows, the entire + month**, and they are not empty rows (50 carry an age and an updated FBG, 47 a + baseline HbA1c, weight, height and BMI, 45 an insulin regimen); and + `2026_Quirino Memorial`'s `Jan26` sheet is 5 rows carrying baseline HbA1c and + BMI. The remaining 17 are spread over `2025_Kantha Bopha II` (4), + `2024_Likas Women & Children's` (4), `2024_Sultanah Bahiyah` (3), + `2022_Mandalay Children's` (3), `2025_Phattalung` (2) and + `2025_Mandalay Children's` (1). The ID column formula needs repointing in each + workbook; until then the measurements are unrecoverable, since nothing else in + the row names the patient (the name column reads `#REF!` too). + +- **A patient ID that is two clinic codes spliced together.** `2026_YGH` + records `MM_NO55_MW_YG` in `Apr26`, `May26` and `June26` -- a transfer between + clinics written into the ID -- where every other patient in that tracker reads + `MM_YY###_YG`. It fails the template's format and no well-formed ID in the + tracker sits an edit away, so the pipeline sentinels it to `Undefined` per + [ticket 47](47-patient-ids-merged-at-cleaning.md) and the patient's three + months are unattributable. Only the clinic can say which ID is intended. + +- **A cleared patient row number, `2022_Children's Hospital 2`, `Oct22`, cell + A70.** The number was deleted but the cell kept a space, which is invisible in + Excel. Harmless to the pipeline as of this session (`find_data_start_row` now + reads through it), and listed only because the same residue in row 1 of eleven + `2024_Mandalay Children's` sheets is what makes R drop that tracker's first + patient -- a clinician cannot see the difference between an empty cell and one + holding a space, so it is worth knowing the shape exists. diff --git a/docs/wayfinder/tickets/41-decide-2026-new-patient-list-columns.md b/docs/wayfinder/tickets/41-decide-2026-new-patient-list-columns.md new file mode 100644 index 0000000..fa9d564 --- /dev/null +++ b/docs/wayfinder/tickets/41-decide-2026-new-patient-list-columns.md @@ -0,0 +1,76 @@ +--- +id: 41 +title: Decide whether the 2026 template's five new Patient List fields enter the pipeline +labels: [wayfinder:grilling] +status: open +blocked_by: [] +assignee: null +claimed_at: null +resolution: null +evidence: null +closed_by: null +spawned_by: 30 +--- + +## Premise + +Rests on [Triage the patient pipeline's raw-stage column-existence +divergence](30-triage-patient-raw-column-divergence.md), closed, which found +these columns while accounting for the only-in-Python divergences, and on the +user's rule stated in that session: **the current tracker template is the +golden rule** -- a column present in today's template is the kind that matters, +where one that appeared for a single year and vanished is very likely a test +that did not survive. These five are the former case, which is why they are +ticketed rather than ruled out. + +Verified against the source workbook +(`2026_Preah Kossamak Hospital A4D Tracker_Jun_26.xlsx`, `Patient List` sheet, +header row 8): the 2026 template carries five columns with no entry in +`reference_data/synonyms/synonyms_patient.yaml` and no home in the 83-column +cleaned schema: + +| Col | Header | Populated | +|---|---|---| +| 17 | `Phone Number` | | +| 18 | `Insurance Card Status` | | +| 19 | `Current Insulin Regimen` | 213 rows | +| 20 | `BGM A4D` | 213 rows | +| 21 | `Insulin A4D` | 213 rows | + +Python's raw stage passes them through under their literal header text (its +`ColumnMapper` runs non-strict); the cleaning stage drops them, since the +schema has no such columns. R does the same thing under sanitized names +(`currentinsulinregimen`, `bgma4d`, ...), so neither pipeline carries them +forward -- this is a **new-field question, not a Python defect**. + +**`Current Insulin Regimen` is not the monthly `Insulin Regimen`.** Measured on +the same file: both columns exist, 191 rows have both populated, and they +**disagree on 39** of those. Mapping the Patient-List column onto +`insulin_regimen` would corrupt data. It is a patient-level field beside the +month-by-month one. + +## Question + +The user said they need to look at the new 2026 trackers themselves before +deciding -- so this ticket waits on that inspection, and the decision is +theirs. + +For each of the five: does it enter the pipeline as a new cleaned-schema +column, or stay a raw-stage passthrough that is deliberately dropped? + +Things the decision turns on, worth having ready when it is taken: + +1. **How widely each column is actually used** -- only one clinic's 2026 + tracker is in the current set. Check the 2026/2025 master template rather + than this single file before concluding it is standard. +2. **Blast radius of saying yes.** A new cleaned-schema column means + `clean/schema.py`, the synonym file, the patient tables, and the BigQuery + schema -- and a BigQuery schema change touches the production load path + ([ticket 5](05-production-verification-run.md)'s territory). +3. **Whether `Current Insulin Regimen` needs a distinct name** to keep it from + ever colliding with the monthly `insulin_regimen` -- the 39 disagreeing rows + are the evidence that it would. +4. **Whether this belongs to this map at all**, or to the post-promotion + backlog. The destination is "close out the R-to-Python migration"; adding + fields R never had is arguably new development, not migration. Deciding that + is part of this ticket. diff --git a/docs/wayfinder/tickets/42-fbg-unit-headers-and-implausible-values.md b/docs/wayfinder/tickets/42-fbg-unit-headers-and-implausible-values.md new file mode 100644 index 0000000..f5258e2 --- /dev/null +++ b/docs/wayfinder/tickets/42-fbg-unit-headers-and-implausible-values.md @@ -0,0 +1,243 @@ +--- +id: 42 +title: Decide how FBG unit headers are resolved, and what to do about physiologically implausible mmol values +labels: [wayfinder:grilling] +status: closed +blocked_by: [] +assignee: session-2026-08-17 +claimed_at: 2026-08-17 +resolution: decided +evidence: read +closed_by: null +spawned_by: 30 +--- + +## Premise + +Rests on [Triage the patient pipeline's raw-stage column-existence +divergence](30-triage-patient-raw-column-divergence.md), closed, whose +`tracker_layout_changed` flag surfaced `2020_CDA` relabelling its +`Baseline FBG` column from `mmol/dL` to `mg/dL` in June while the values stayed +identical (median 233, range 67-500 -- mg/dL on both sides). Chasing that +produced a full inventory of every FBG header spelling across the 254-tracker +set: [assets/fbg_header_inventory.md](../assets/fbg_header_inventory.md), all +23 spellings with their resolution and value distributions. (An `.xlsx` copy +was generated for the advisor conversation but is not committed -- `*.xlsx` is +gitignored repo-wide to keep tracker data out of git.) + +What the inventory establishes (executed, whole-set): + +- **The mg side is clean.** Ten spellings, all mapped, all mg-scale -- + including typos the synonym file already absorbs (`mg%`, `mg/%`, `m/dL`, + `ml/dL`). +- **The malformed `mmol/dL` spelling carries two different units.** + `Updated FBG mmol/dL` (2 files, 178 values, median 9.4) is genuinely mmol, so + its existing alias to `fbg_updated_mmol` is correct. + `Baseline FBG (mmol/dL)` (1 file -- 2020 CDA, 45 values, median 233) is + mg-scale. So the same spelling cannot be blanket-aliased, and leaving it + unmapped is the safe default rather than a gap. +- **One header offers the reader a choice of units and is unmapped**: + `Baseline FBG (mmol/L or mg/dL)`, with an `Updated FBG` twin, drops 155 + mmol-scale values. Confirmed to be `2018_Kantha Bopha Hospital A4D Tracker` + (verified from the local comparison report, where both sides carry the pair + unmapped -- R under its sanitized name, Python under the literal text). Being + a 2018 tracker, the **latest-template rule puts this at low priority**: it is + historic, not current, and the trackers are kept if the values are ever + wanted. +- **Exactly one column is outright mislabelled**, not two: + `Baseline FBG (mmol/dL)` at 2020 CDA. The mixed-unit problem below is a + different kind of defect -- individual rows in the wrong unit inside a + correctly-labelled column, which no header fix can reach. +- **Unit confusion runs in both directions**, and is an order of magnitude + larger in one of them. Grouping every glucose column by when it is measured + and the unit its header states + ([assets/glucose_readings_by_unit.md](../assets/glucose_readings_by_unit.md)): + **9,264 readings sit in mg/dL-labelled columns but below 30 mg/dL** (3,714 + baseline + 5,550 updated), which cannot be an ambulatory reading and is + exactly where a mmol/L number lands; **122 sit in mmol/L-labelled columns but + above 100** (30 + 92), where a mg/dL number lands. +- **Zeros are placeholders, not hypoglycaemia**: all 244 baseline mmol/L + readings under 3 mmol/L are exactly 0. +- **The baseline distribution is broad but continuous** -- median 19.3 mmol/L, + p90 38.0, and **no values above 100 at all**. That is not the bimodal shape + bulk mg/dL contamination would produce, so the high baseline may simply be + presentation hyperglycaemia. + +Two framings were tried and discarded before this one, both recorded so they +are not retried: bucketing on **35 mmol/L** (an invented threshold, replaced by +percentiles and the two physiologically impossible cut-offs above), and +comparing **mg-recording clinics against mmol-recording clinics** (unsound -- +different clinics and populations, no dual recording of the same patient). + +Nothing in `reference_data/synonyms/synonyms_patient.yaml` was changed while +closing ticket 30: resolving a unit needs clinical input, not a code decision. + +## Status: asked, awaiting reply + +**The email went to A4D's medical advisor on 2026-08-15**, with a +clinician-facing spreadsheet (the grouped table above plus a tab listing every +exact column label; no clinic names, no pipeline vocabulary, no invented +thresholds). This ticket cannot be worked until that reply lands -- there is +nothing to derive from the code or the data that would settle it, which is why +it is not simply blocked on another ticket. + +The four questions asked, in the order sent: + +1. Are a reading below 30 mg/dL and a reading above 100 mmol/L both impossible + in this setting? If so, ~9,300 readings have their unit mixed up one way and + ~120 the other. +2. What is the credible range for a fasting glucose reading, top and bottom? +3. Is a baseline median of 19.3 mmol/L (p90 38.0) what you would expect at T1D + diagnosis in these clinics? +4. What should the pipeline do with a reading outside that range -- mark it + suspect, convert it (mg/dL / 18), or discard it? + +Plus one statement offered for correction: that readings of exactly 0 mean +"not measured". + +## Question + +The decision this ticket carries, once the reply is in. + +1. **What credible range does the advisor give**, and does it confirm that + below 30 mg/dL and above 100 mmol/L are impossible? That confirmation is + what turns ~9,400 readings from "odd" into "unit mixed up at entry". +2. **What should the pipeline do with an out-of-range value** -- flag it via + `ErrorCollector` and keep it, convert it (`mmol/L = mg/dL / 18.0182`), or + sentinel it? Converting is a data-altering guess and should not be done on a + heuristic alone. Note the asymmetry in scale: the mg/dL-labelled columns + hold 9,264 suspect readings against the mmol/L columns' 122, so whatever is + decided moves far more data in one direction than the other. +3. **Does `Baseline FBG (mmol/L or mg/dL)` get mapped**, and if so to which + column -- or is the answer that a template should never offer an ambiguous + header, making this a source fix rather than a synonym one? Note the + latest-template rule likely settles this as "leave it": the only tracker + carrying it is from 2018. +4. **Is a per-column unit check worth adding generally?** The same shape could + apply to HbA1c and other unit-bearing columns; deciding that here avoids + solving it once per column later. + +Note that a unit rule cannot be decided from R's behaviour -- R has the same +headers and the same synonym source -- so this is a Python-forward decision, +not an R/Python divergence question, and it does not block [ticket +12](12-retire-r-workspace.md). + +## Resolution (session-2026-08-17) + +### Decision + +**The advisor's reply did not confirm this ticket's central premise, and the +decision was rebuilt on what it did establish.** The ticket asked whether a +reading below 30 mg/dL is impossible; the answer came back as *analytical* +limits -- mg/dL readable from ~2-5 to ~720-800, mmol/L from ~0.1-0.3 to ~40-45 +-- which makes 5-30 mg/dL **possible but severe, not impossible**. The +"9,264 readings with the unit mixed up" figure is therefore not, on the +advisor's word, clearly wrong-unit. What he authorised is narrower and +conditional: *"if it is clearly a wrong unit mixed up, then correct it"*, plus +*"advise the data input staff"*. Defining "clearly" was the real decision. + +**"Clearly" was settled by measurement, not by a threshold.** Across 372 +file-column groups and 110,007 readings, the confusion is overwhelmingly *per +column* rather than per row, and it clusters by clinic: + +| share of the column below 30 mg/dL | groups | reading | +|---|---|---| +| >=90% | 29 | the column is mmol under an mg/dL header | +| 50-90% | 7 | per-patient mixing, all baseline at two clinics | +| 10-50% | 10 | mostly one clinic (Preah Kossamak) | +| <10% | 26 | scattered single rows | +| none | 300 | clean | + +`2025_Kantha Bopha II`'s updated FBG is 99.8% of 951 readings below 30, which +cannot be a run of severe hypoglycaemia. A single low reading in an otherwise +ordinary column is the opposite case: a genuine severe hypo and a mis-entered +unit are indistinguishable, and converting one would multiply the clinically +most important reading in the file by 18 and hide it. + +**Implemented (option B, chosen by the user):** + +- `src/a4d/clean/glucose.py`, new. `resolve_glucose_units` runs as step 5.8 of + patient cleaning -- before range validation so the limits judge corrected + values, and before the existing mg/mmol cross-derivation so it sees a column + whose unit matches its name. +- **Column-level correction.** Where a column has >=10 readings and >=90% sit + below 30 mg/dL, its values move to the mmol sibling (never overwriting a + reading already recorded there) and the mg column is rescaled by 18. Reported + as `glucose_unit_swapped` **once per column, not once per row** -- the defect + is the header, so the source fix is file-level. **29 columns across 21 files.** +- **Row-level flagging, no conversion.** A sub-30 reading in a column that was + not swapped is reported as `glucose_unit_suspect` and left exactly as + recorded. **5,673 rows across 34 files.** This is where the 50-90% and 10-50% + bands land, per the user's instruction that the ambiguous middle goes with + the flagged side. +- **Zero is not a reading.** It sits below the analytical floor of both units, + so it becomes null rather than being read as profound hypoglycaemia. +- **Analytical limits enforced.** `cut_numeric_value` now bounds all four FBG + columns at the permissive end of the advisor's ranges (mg/dL 2-800, mmol/L + 0.1-45). This replaces an inherited 0-150 mmol/L bound from R's + `script2_process_patient_data.R` -- more than three times his ceiling -- and + covers the three columns that had **no bound at all**. **832 readings + rejected**, including 383 above the mg/dL ceiling (max 2,013) that nobody had + ever looked for: the original analysis only examined the low end. + +### Numbers (real 254-tracker run, comparison `2026-08-16T231826Z`) + +Every FBG value now sits inside the advisor's limits: mg max 800.0 (was 2,013), +mmol max 44.9 (was 600). Medians moved as expected -- baseline mg 190 -> 216 +(the mmol readings that were dragging it down have left), baseline mmol +19.3 -> 15.7 (it gained ~1,700 lower readings from the swap). Both remain +consistent with the presentation hyperglycaemia the advisor confirmed. + +### Rejected + +- **Converting every sub-30 reading** (~12,000). The advisor's limits do not + make them impossible, and it would silently rewrite real severe hypos. +- **Converting nothing but the analytically impossible.** Defensible, but + knowingly leaves 29 wholly mislabelled columns wrong in the data. +- **A row-level record for a swapped column.** The same finding repeated + hundreds of times; the workbook fix is one header. +- **Stretching the new comparison classifier to cover R-null cells** -- see + below. Ticket 31's precedent: a shape-matching label is not a decision. + +### Evidence + +**Executed** throughout: the file-column distribution, the 5-30 vs 2-5 vs 0 +banding, the mg upper-limit breach, and every before/after figure come from +real pipeline and comparison runs over the 254-tracker set on the drive, not +from unit tests. The advisor's limits are **read** -- an expert statement, not +something this repo can verify -- which is what `evidence: read` records. + +**One inconsistency in the reply, noted and immaterial:** he says readings +below 0.1 mmol/L are possible while his own table puts the analytical floor at +0.1-0.3. No reading in the dataset sits in that gap. + +**Tense:** all figures describe behaviour after this session's commit. + +### Questions 3 and 4 of the ticket's own list + +- **`Baseline FBG (mmol/L or mg/dL)` stays unmapped.** The only tracker + carrying it is from 2018, and the latest-template rule settles it: a template + should never offer the reader a choice of unit, so this is a source defect to + record, not a synonym to add. +- **A general per-column unit check was not built.** The mechanism here is + specific to a column pair that exists in both units; HbA1c has no such pair. + Revisit only if a second column turns out to need it. + +### Not fixed, deliberately + +The correction is a deliberate divergence from R, which has no unit resolution, +so patient cleaned-stage mismatches rose 98,274 -> 114,509, entirely in the four +FBG columns. `python_glucose_unit_corrected` +(`PATIENT_GLUCOSE_UNIT_CLASSIFIERS`) explains 11,189 of them on the three shapes +the correction provably produces: an exact 18x ratio either way, a null against +R's literal 0, and Python's sentinel against a reading outside the analytical +range. + +It deliberately never fires on an R-null cell, which leaves **2,842 cells on +`fbg_updated_mmol` unclassified** -- cleaned-stage unclassified 6,587 -> 9,427. +Measured, **2,794 of those 2,842 (98.3%) are in files whose column was +swapped**, so the population is understood: the swap populated a column R never +had. It is not classified because the per-cell classifier has no file context, +and a blanket R-null rule would also swallow the ~193 cells on that column that +predate this change -- exactly the labelling-without-deciding this map forbids. +Spawned as [ticket 44](44-triage-cleaned-fbg-r-null-residual.md). diff --git a/docs/wayfinder/tickets/43-triage-patient-raw-residual-3.md b/docs/wayfinder/tickets/43-triage-patient-raw-residual-3.md new file mode 100644 index 0000000..4bc7d99 --- /dev/null +++ b/docs/wayfinder/tickets/43-triage-patient-raw-residual-3.md @@ -0,0 +1,196 @@ +--- +id: 43 +title: Triage the residual patient raw-stage column mismatches (round 3) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-17b +claimed_at: 2026-08-17 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 31 +--- + +## Premise + +Rests on [Triage the residual patient raw-stage column mismatches (round +2)](31-triage-patient-raw-residual-2.md), closed: the stage's dominant column +`complication_screening` is fully resolved, in two separate halves. A real +Python data loss was fixed -- `ColumnMapper.rename_columns` kept only the +first of several source columns mapping to one canonical name, discarding +2,489 recorded values across 27 trackers; it now merges them the way R's +`tidyr::unite()` and the pipeline's own `merge_duplicate_columns_data()` do. +And R's `NA` padding from that same unite (its `na.rm` defaults to `FALSE`) +became the source-verified `r_na_unite_padding` classifier. Patient raw-stage +unclassified fell 14,903 -> 1,879 (-87%) against baseline run +`output/comparison/2026-08-15T214447Z`. + +Also rests on [Triage the patient pipeline's raw-stage column-existence +divergence](30-triage-patient-raw-column-divergence.md), closed, for the +`insulin_regimen` lead below; and on the map's standing scoping rule that the +current tracker template is the golden rule, so a column that appears in one +year and is gone the next is not worth chasing. + +Overturning ticket 31's merge decision would void this ticket rather than +require rewriting it -- several of the shapes below exist only because the +merge now happens. + +## Question + +Triage the remaining **1,879 unclassified** raw-stage mismatches across +roughly 50 columns, none larger than 235. Continue the pattern tickets +20/21/22/24/27/31 established: find systematic shapes, arbitrate against the +real source Excel, add named causes to the `PATIENT_*_CLASSIFIERS` registries +in `src/a4d/migration/compare.py` -- and, per the map's standing bar, decide +whether Python is doing the right thing rather than only labelling. + +Three shapes are already visible and should be checked first: + +- **`insulin_regimen` (235), and the trap in it.** 194 of the 235 rows are in + `2021_Kantha Bopha` and are ticket 30's source-verified blank-header + recovery: R keeps those values only as a junk-named `na1` column, Python now + recovers them, so R is null where Python has a verified value. The other 41 + rows (39 in `2024_Vietnam National Children`, 2 in `2023_...`) share the + *shape* but were never verified. Ticket 31 deliberately did **not** wire this + column to `r_extraction_gap`, because that classifier fires on any + R-null/Python-present cell and would have labelled the unverified 41 without + deciding them. Verify the 41 against source before choosing a cause, or split + the classifier so it fires only where the evidence reaches. +- **`observations` (61) -- two shapes, one of them new.** R carries a trailing + space on a sub-value before the join (`"Normal ,Insulin"` where Python has + `"Normal,Insulin"`), because ticket 31's merge strips each part while R's + `unite` does not. This is interior whitespace, so the existing + `normalize_whitespace_column` (which trims whole values) does not reach it -- + decide whether to extend it or classify. The rest are `NA,NA` on R's side + against a real Python value (`"TSH"`, `"Foot Examination (Nerves)"`), which + looks like the ticket-31 recovery reaching a column R lost entirely. +- **`fbg_baseline_mg.static` (192) and `fbg_baseline_mmol.static` (110).** + Never looked at by any ticket. The `.static` suffix is the Patient List join + suffix, so `PATIENT_JOIN_SUFFIX_COLLISION_CLASSIFIERS` and the assumption in + force about which baseline-FBG copy Python keeps are both directly relevant + -- read them before measuring. + +The rest is a long tail: `hba1c_updated` (88), `last_clinic_visit_date` (88), +`fbg_updated_date` (75), `complication_screening_lipid_profile_cholesterol_value` +(72), `complication_screening_kidney_test_value` (60), `status` (49), +`observations_category` (48), `insulin_injections` (46), and ~40 more under 45 +each. Ticket 31's early sampling of some of these predates two extraction +changes (ticket 30's blank-header recovery and ticket 31's merge), so +re-measure rather than trusting the old characterizations. + +Split further rather than leaving this open-ended if it does not converge. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: explaining a +difference and naming a cause is not enough -- each needs an explicit verdict +on whether Python is right, with the evidence. Where Python is losing +information the source carried, fix the pipeline (tickets 27, 30 and 31 all +turned out to be that). Where the source workbook itself is wrong, that is a +legitimate conclusion and a finding for [ticket +40](40-source-defect-findings-report.md), not a pipeline fix. A cause +genuinely undecidable on the available evidence is recorded as an open +question, not closed with a label. + +## Resolution + +**Decision.** Two of the residual's causes are settled and fixed; the largest +remaining one is a comparison-tool alignment failure, split into [ticket +45](45-patient-row-alignment-duplicate-keys.md). Patient raw-stage +unclassified fell **1,879 -> 1,409 (-25%)**, total raw mismatches 27,980 -> +27,510, measured against the new baseline run +`output/comparison/2026-08-17T192144Z`. + +**1. Float-to-string rounding, 434 rows -- a tool gap, not a divergence.** +`fbg_baseline_mg.static` (192), `fbg_baseline_mmol.static` (110), +`complication_screening_lipid_profile_cholesterol_value` (72) and +`complication_screening_kidney_test_value` (60) were reported purely because +R writes `9.3000000000000007` where Python writes `9.3`. The existing +`normalize_numeric_column` already handles this; it never reached these four +because `PATIENT_RAW_NUMERIC_NORMALIZE_COLS` derived from the *cleaned* +schema, which has no entry at all for the Patient List join's `.static` +copies and types a screening measurement as a string (the column can also +hold "normal"). Replaced with `numeric_normalize_targets` (every raw column +except the join keys and `__` helpers), which is safe because +`normalize_numeric_column` passes non-numeric text through untouched. +**Verified rather than assumed**: across all 27,980 raw-stage mismatches +exactly 434 had both sides parsing to the same float, all four in those +columns -- so widening equality changed nothing else, and the re-run +confirmed it (all four -> 0, no other column moved by more than 6). + +**2. Phantom patient rows, a real Python data invention -- fixed.** +`read_patient_rows` accepted any row where the row number (column A) *or* the +patient_id (column B) was present. `2024_Vietnam National Children`'s `Jul24` +sheet carries a bare list of patient IDs in rows 165-188 below the real data +block -- **exactly two non-empty cells each, the ID repeated, and nothing +else across 40 columns** (read directly from the source workbook). Python +emitted all 24 as monthly records, which then picked up real-looking +demographics from the Patient List join. R never sees them: its data block is +`which(!is.na(tracker_data[, 1]))`, the row-number column alone +(`script1_helper_read_patient_data.R`). + +Adopting R's rule outright would have been wrong, and measuring stopped that. +A sweep of all 254 trackers found only **41** rows Python accepts past R's +numbered block, in 8 files: 36 are ID-only leftovers, 4 are template filler +(`patient_id = "0"`, already dropped by both pipelines -- verified in both +output parquets), and **1 is a complete patient record R loses** -- +`2024_Mahosot` `Jun24` `LA-MH088`, 24 populated cells, which simply never got +a row number. So the fix keeps the case the `or` was written for and drops +only rows carrying no value beyond a repeat of their own identifier +(`_carries_data_beyond_identifier`). Deliberately not a cell-count threshold: +the trackers repeat the identifier in a second column, so "more than n +non-empty cells" would be a guess about layout. Verified end-to-end on the +real 248-tracker data: that file's `py_unmatched` rows went **24 -> 0**. + +**3. `hba1c_updated`/`fbg_updated_mg` interior space, 86 rows -- Python is +right, R's mechanism unlocated.** All 86 are in `2017_Yangon Children's +Hospital`, shape `8.8(20.9.16)` R-side vs `8.8 (20.9.16)` Python-side. Read +the source cell directly (`Feb17`, row 62, column 12): **the workbook +contains `8.8 (20.9.16)`, with the space**. Python reproduces the source +exactly; R drops it. Searched R's raw path for the cause and did not find it +-- `sanitize_str` only touches column names and validator lookups, the +multiline-header merge only touches headers, and the wide-format splitter is +product-side. Recorded as understood-but-unlabelled rather than closed with a +guessed cause, per the map's standing bar; the classifier is left to ticket +46 along with the rest of the tail. + +**Because.** The map's bar is deciding, not labelling, and the two items +worth deciding both turned on evidence a shape-match would have got wrong. +Cause 1 looked like a real divergence and was a tool gap; cause 2 looked at +first like Python dropping a month of data for 27 patients (the ticket's own +bidirectional-null signature) and was the reverse -- Python inventing rows. + +**Rejected.** +- *Adopting R's "row-number column defines the block" rule.* Simplest and + matches R exactly, but measured to lose `LA-MH088`, a real record. Rejected + on evidence, not taste. +- *A non-empty-cell-count threshold to spot phantom rows.* Would have worked + on today's data (2 cells vs 24) but is an invented constant about layout, + the same objection ticket 42 raised against its 35 mmol/L threshold. +- *Extending `normalize_numeric_column` by hand-listing the four columns.* + Rejected under the map's never-hand-maintain-a-derivable-list rule; the + derivation was pointed at the wrong schema, so the derivation got fixed. +- *Chasing the remaining 808 fan-out rows in this session.* They need the + patient arm to gain an ordinal row key, which is ticket 17's product work + repeated with its own verification burden -- a separate piece, split rather + than sprawled. +- *Classifying the Yangon 86 now.* Python is verified right, but naming a + cause for a mechanism not located in R would cement a guess. + +**Evidence.** All **executed**: the source workbooks read cell-by-cell +(VNC `Jul24` rows 165-188, Mahosot `Jun24` row 380, Yangon `Feb17` row 62), +the 254-tracker sweep, both output parquets checked for the filler rows, a +full `a4d run patient --force` re-run against the real 248-tracker set, and a +full comparison re-run. Full suite 717 passed, 1 skipped; ruff and +`ty check src/` clean. The one **read**-only claim is R's block-bounding rule, +taken from `script1_helper_read_patient_data.R` rather than by running R -- +but its consequence (R has no `LA-MH088` row, and none of the 24 phantom +rows) was confirmed in R's own frozen output parquet. + +**Tense.** Every count above is current behaviour measured after the fixes, +except the 1,879/27,980 starting figures, which describe the prior baseline +`2026-08-16T231826Z`. + +**What is left.** 1,409 unclassified: 808 in three duplicate-key files +(ticket 45) and ~600 in the long tail (ticket 46). diff --git a/docs/wayfinder/tickets/44-triage-cleaned-fbg-r-null-residual.md b/docs/wayfinder/tickets/44-triage-cleaned-fbg-r-null-residual.md new file mode 100644 index 0000000..7819161 --- /dev/null +++ b/docs/wayfinder/tickets/44-triage-cleaned-fbg-r-null-residual.md @@ -0,0 +1,200 @@ +--- +id: 44 +title: Classify the cleaned-stage FBG cells where R has nothing and Python has a corrected reading +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: null +claimed_at: 2026-08-21 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 42 +--- + +## Premise + +Rests on [Decide how FBG unit headers are resolved, and what to do about +physiologically implausible mmol values](42-fbg-unit-headers-and-implausible-values.md), +closed: on A4D's medical advisor's answer of 2026-08-17, the pipeline now +resolves glucose readings recorded under the wrong unit's header +(`src/a4d/clean/glucose.py`). Where a whole column labelled mg/dL holds mmol/L +readings, its values move to the mmol sibling and the mg column is rescaled by +18 -- 29 columns across 21 files. R has no unit resolution at all, so every +correction is a deliberate divergence. + +`python_glucose_unit_corrected` (`PATIENT_GLUCOSE_UNIT_CLASSIFIERS`, +`src/a4d/migration/compare.py`) explains 11,189 of them, on the three shapes the +correction provably produces. It deliberately does **not** fire where R is null, +because that rule would also swallow a pre-existing population on the same +column that has nothing to do with this change. + +Also rests on the map's assumption in force about which baseline-FBG copy Python +keeps (from [ticket 29](29-triage-patient-cleaned-residual.md)) -- R's Patient +List join suffixes both sides and loses the unsuffixed column, which is the +other reason an FBG cell can be R-null. Read it before measuring; the two +populations sit on adjacent columns and are easy to conflate. + +If ticket 42's correction were reverted, this ticket would be void rather than +in need of rewriting. + +## Question + +Classify or decide the **2,842 cleaned-stage `fbg_updated_mmol` cells** where R +is null and Python has a value (baseline run +`output/comparison/2026-08-16T231826Z`; cleaned-stage unclassified is 9,427, +of which this is the dominant share). + +What is already measured, and should not be re-derived: + +- **2,794 of the 2,842 (98.3%) are in files whose column was swapped** by + ticket 42's correction, so the mechanism is understood: Python populated a + column R never had. The largest are `2025_Kantha Bopha II` (953), + `2022_Kantha Bopha` (648), `2019_Vietnam National Children` (430), + `2021_Kantha Bopha` (419). +- **48 sit in files with no swap at all**, spread thin (`2018_CDA` 26, + `2024_Vietnam National Children` 18, then singletons). These are a different + question and have never been looked at. +- The column carried only 193 mismatches in total before ticket 42, so the + population that predates the correction is small and bounded. + +The real question is how to record the 2,794 without labelling the rest. The +obstacle is structural: `classify` sees one cell at a time and has no file +context, so it cannot ask "was this file's column swapped?". Options worth +weighing rather than assuming: + +1. Give the classifier the file-level context it needs -- e.g. have the + comparison read the run's own `glucose_unit_swapped` error records, or have + `compare_cells` pass the file through. Most faithful, most invasive. +2. Narrow the cell-level rule enough to be safe on its own terms (mmol column, + R null, Python present and within the analytical range) and accept a stated + error bound of at most ~193 cells. +3. Decide the 48 no-swap cells individually against source, then judge whether + option 2's bound is acceptable once the genuinely unrelated population is + known. + +Option 3 first looks cheapest -- 48 cells is a morning, and it turns the error +bound from an estimate into a measured number. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: a classifier +that fires on a shape it has not established is not a decision. Ticket 42 +declined to stretch this exact rule for this exact reason, and [ticket +31](31-triage-patient-raw-residual-2.md) declined the same shape on +`insulin_regimen`; both are worth reading before choosing option 2. + +## Resolution + +**Decision.** Option 1, as the user chose -- but measurement moved where it +pays off, and the population turned out to be one mechanism rather than the +two the ticket describes. + +The cleaned-stage `fbg_updated_mmol` residual is **2,935 cells, 99.3% of the +whole cleaned-stage unclassified population** (the ticket's 2,842 was measured +on 2026-08-16 and had drifted). Every one of them -- **not one exception** -- +sits beside an `fbg_updated_mg` cell on the same row that is *itself* a +mismatch with an already-named cause. Neither pipeline reads the mmol column +from the workbook: both derive it from the mg cell (`convert_glucose_units`, +clean/patient.py; `fix_fbg`'s `fbg/18` in R). So the mmol divergence is the +arithmetic shadow of a decision already taken next door, in three shapes: + +| n | shape | mg sibling's cause | +|---|---|---| +| 2,797 | R null, Python has the reading | `python_glucose_unit_corrected` / `r_fbg_text_category_invention` | +| 110 | **Python null, R has a value** | `r_fbg_text_category_invention` (80), `python_glucose_unit_corrected` (30) | +| 28 | R null, Python has the reading, no swap in the file | `python_recovers_glucose_r_rejected` | + +The 110 is not in the ticket at all -- it is the *opposite* direction, and it +is where R does the damage. Every one is exactly `mg_r / 18`: R divides a +number `fix_fbg` manufactured out of text (140 from "Lost follow up") or one +far beyond the analytical ceiling (2013 mg/dL) and publishes the quotient as a +measurement. `2023_CDA` gets **111.8 mmol/L** that way -- above the level A4D's +medical advisor called outright impossible. Python is the correct side in all +three shapes. + +Two changes, both following the existing precedent that `compare_cells` +precomputes a fact onto `CellMismatch` and the classifier judges it +(`row_order_candidate`, `row_has_bare_year_date`): + +1. **`mg_sibling`** carries the row's `(r, py)` mg pair to the mmol cell. + `mmol_derived_from_mg_sibling` (`PATIENT_GLUCOSE_CASCADE_CLASSIFIERS`) + fires only when the mg sibling itself disagrees **and** each side's mmol + value is either absent or exactly its own mg cell over 18. That arithmetic + identity is the bound: an mmol cell carrying a reading its mg sibling + cannot account for is left alone. +2. **`column_unit_swapped`** carries the file-level fact -- read back from the + run's own `glucose_unit_swapped` error records via + `load_glucose_unit_swaps`, keyed through each frame's `file_name` column + rather than by string surgery on parquet names, so a pipeline threshold + change reaches the comparison without a second edit. + `_is_python_glucose_unit_corrected` now picks its analytical bound on that + evidence instead of on whether the column's name ends `_mg`. + +**Measured against the real 254-tracker drive data** (`output/comparison/ +2026-08-21T215325Z` vs the `...T212441Z` baseline): + +- `fbg_updated_mmol` unclassified **2,935 -> 2** +- `fbg_baseline_mg` unclassified **6 -> 0** -- change 2 closes the standing fog + patch *"Whether a column-level finding can be classified at all"*, the six + 2020 Kantha Bopha cells round 10 could only write out in prose +- **cleaned-stage unclassified overall: 2,955 -> 16** +- `per_column` counts are **byte-identical**, and the patient raw, product raw + and product cleaned snapshots are identical -- nothing was suppressed, only + named +- Full suite 905 passed / 1 skipped; ruff and `ty check src/` clean + +**The remaining 16 are all already owned by fog, and none is a glucose +question**: 11 `insulin_subtype` + 2 `remote_followup` are the standing +null-vs-`Undefined` patch (round 10 counted these 13 exactly), 1 `bmi_date` is +ticket 39's slash-separated-month patch, and the last 2 `fbg_updated_mmol` are +R-null against Python's `999999` -- **neither side published a reading**, so +this is the numeric-absence fog patch (`999999` making the same false claim +`9999-09-09` made on dates), not a cascade. The classifier declines them +deliberately. + +**Because.** The ticket framed the obstacle as file-level and the answer as a +choice about error bounds. Scanning the whole population first -- per the map's +own standing preference -- showed the decisive fact is row-level and stronger +than any file-level rule: the mg sibling's value *is* the evidence, so no error +bound has to be accepted at all. Option 1's file-level context still earned its +place, just on the six cells the fog patch named rather than on the 2,794. + +**Rejected.** + +- **Option 2** (narrow the cell rule, accept a stated error bound): the exact + shape ticket 42, [ticket 31](31-triage-patient-raw-residual-2.md) and round + 10 each declined. Measurement priced it honestly -- the bound is 28 cells, + not the ticket's estimated ~193 -- and those 28 turn out to be a *real, + separate* mechanism (2018 CDA's `148 mg/dl (Mar-18)` cells), so option 2 + would have mislabelled the one population that had never been looked at. +- **Option 3** (triage the 28 first, then judge option 2): its measuring half + was done and is what produced the answer; its conclusion -- that a measured + bound licenses option 2 -- does not follow, since a bound prices a rule it + does not make correct. +- **Reordering the registry** so `PATIENT_FBG_TEXT_CLASSIFIERS` precedes the + glucose ones, to stop the widened mmol bound claiming five + `r_fbg_text_category_invention` cells (140 exceeds 45): rejected because + `r_unit_suffix_not_stripped` rides in the same registry and is broad enough + to steal the 28 cells' mg sibling from `python_recovers_glucose_r_rejected`. + Guarding the range test against R's manufactured constants instead is local + and says the true thing -- a number `fix_fbg` invented is not a reading, so + no range has anything to say about it. Caught by re-running: the first pass + moved those 5 cells and the snapshot diff showed it. +- **A classifier keyed on "the mg sibling is a mismatch" alone**, without the + ÷18 identity: it would swallow any genuine mmol divergence that happened to + sit beside a diverging mg cell, which is precisely the over-claiming the + standing bar forbids. + +**Evidence.** **Executed** throughout: the population split, the sibling +pairing and the "no exceptions" claim are duckdb queries over the run's own +`cell_mismatches` sheet and `table_errors.parquet`; the before/after numbers +are two full `just compare-outputs` runs against the real drive data, compared +snapshot-to-snapshot. R's `fix_fbg` behaviour and the analytical limits are +**read** -- inherited from rounds 8/10 and ticket 42, not re-derived here. + +**Tense.** All numbers describe current behaviour after this session's changes, +except the baseline column of each before/after pair. + +**Spawned nothing.** The residual is fully absorbed by fog patches that already +exist. diff --git a/docs/wayfinder/tickets/45-patient-row-alignment-duplicate-keys.md b/docs/wayfinder/tickets/45-patient-row-alignment-duplicate-keys.md new file mode 100644 index 0000000..3c3bc90 --- /dev/null +++ b/docs/wayfinder/tickets/45-patient-row-alignment-duplicate-keys.md @@ -0,0 +1,224 @@ +--- +id: 45 +title: Give the patient comparison an ordinal row key, so duplicated patient IDs stop faking mismatches +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-17c +claimed_at: 2026-08-17 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 43 +--- + +## Premise + +Rests on [Triage the residual patient raw-stage column mismatches (round +3)](43-triage-patient-raw-residual-3.md), closed, which measured this: **808 +of the 1,409 remaining unclassified patient raw-stage mismatches (57%) sit in +just three files**, and all three are files where the row-alignment key +`(patient_id, sheet_name)` is not unique. + +Ticket 43 fixed a genuine Python defect in the worst of them -- 24 invented +rows in `2024_Vietnam National Children`'s `Jul24` -- and after that fix both +sides hold the *same* row set: `row_key_overlap` for that file reads +`matched: 903, r_unmatched: 0, py_unmatched: 0`. Yet 770 cell mismatches +remain in it. They are join fan-out: the `Jul24` sheet lists 27 patient IDs +twice (verified in the source workbook -- two lists spliced together, the +numbered column running 1..78 while the IDs repeat), so an equi-join on the +key pairs each duplicate every way round, and half those pairings compare +unrelated physical rows. + +Also rests on [Fix the product comparison's row-alignment key](17-fix-product-row-alignment-and-triage.md), +closed, which solved exactly this for the product arm with `add_row_ordinal()` +-- ordinal position within a group, computed at comparison time and never +stored, because R's frozen baseline cannot be re-run to pick up a new column. +That ticket took product's cleaned-stage row-key match from near-0% to 97.2% +and is the precedent to follow rather than re-derive. + +Overturning ticket 43's phantom-row fix would void this ticket rather than +require rewriting it: the fan-out measurement above was taken after that fix, +and before it the same file's row sets did not even match. + +## Question + +Extend the ordinal row-alignment strategy to the patient arm so duplicated +`(patient_id, sheet_name)` keys align positionally instead of fanning out. + +Concretely: + +- The three affected files are `2024_Vietnam National Children` (770 + unclassified), `2023_Vietnam National Children` (22) and `2018_Penang + General Hospital` (16). Confirm that list by measurement rather than + trusting it -- it was derived from one run. +- Decide whether patient should adopt `add_row_ordinal` as its key + unconditionally (as product did) or only where the natural key is + ambiguous. Product had no usable natural key at all; patient's is unique in + all but three files, so switching everything to positional alignment trades + a real identity check for a positional one across 250+ files that do not + need it. Ticket 15 established patient's key *is* sound in general. +- Whatever is chosen, validate it the way ticket 17 did: by + `RowKeyOverlap.matched` and by per-column counts moving in the direction the + mechanism predicts, against the real drive data -- not by unit tests alone. + +Note the duplicated IDs are themselves a **source defect**, not something the +pipeline should silently reconcile: `Jul24` genuinely lists patients twice +with different data. That belongs in [ticket +40](40-source-defect-findings-report.md) as a finding, and this ticket should +hand it over rather than absorb it. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: a +realignment that merely makes counts fall is not a result. Show that the rows +now being compared are the rows that should be compared, and say explicitly +what the residual after realignment is. + +## Resolution + +**Decision.** Patient keeps `patient_id` + `sheet_name` as its row-alignment +key and gains an ordinal *within* that key as a tie-breaker -- not product's +unconditional positional key. The tie is broken by **content, not position**, +because within one patient-and-sheet group the row order carries no meaning +and the cleaned stage demonstrably reorders. Applied to both patient stages, +raw and cleaned. Patient raw-stage unclassified fell **1,409 -> 601 (-57%)** +and cleaned-stage **9,392 -> 8,383 (-11%)**, measured against baseline run +`output/comparison/2026-08-17T192144Z`; the new baseline is +`output/comparison/2026-08-17T202809Z`. + +**1. How widely: identity everywhere, position never.** The user set the rule +directly -- `patient_id` is the main key, and if it does not match something is +very wrong; `sheet_name` stays because monthly is the granularity of the data. +So `add_row_ordinal` is grouped by the identity key itself rather than by a +whole sheet: in the 250+ files whose key is unique every ordinal is 0 and +behaviour is unchanged, and the ordinal only does work where a sheet lists a +patient twice. Product's `(clinic_id, product_sheet_name)` grouping is +untouched. + +The two strategies are now named rather than inferred: `RowAlignment.IDENTITY` +vs `RowAlignment.POSITIONAL` on a new `Stage` dataclass +(`scripts/compare_outputs.py`), replacing a 9-field positional tuple in which +the alignment key was one unlabelled slot. `detect_row_order_divergence` -- +ticket 21's `order_group_cols` diagnostic, which asks whether a mismatched +value appears elsewhere in its group -- is derived from the alignment rather +than set per stage, and is off for patient: under IDENTITY a group is one +patient on one sheet, where that test would label noise as understood. + +**2. The three-file list was right for raw and incomplete for cleaned.** +Measured per file across both output directories rather than taken on trust +(the ticket's list came from one run): at the **raw** stage exactly the three +named files have a two-sided duplicate key -- `2024_Vietnam National Children` +(21 keys, 42 excess pairings), `2023_Vietnam National Children's` (1) and +`2018_Penang General Hospital_DC` (1). At the **cleaned** stage there are +**five**: those three plus `2023_NPH` (one key repeated 4 times) and +`2026_Surat Thani`. Two further files (`2026_Preah Kossamak`, 98 rows; +`2026_Quirino`, 5) have R-side rows keyed `patient_id = "#REF!"` with no +Python counterpart -- unmatched, never fanned out, and unaffected by this +change. + +**3. Position is the wrong tie-break at the cleaned stage -- measured, not +assumed.** Before wiring anything, both pairings were brute-forced over every +two-sided duplicate group in the real data. At raw, occurrence order is already +the mismatch-minimising pairing in all 23 groups (256 differing cells either +way) -- both pipelines emit the rows in source order. At cleaned it is not: in +9 groups R and Python emit the copies the other way round, and `2024_Vietnam +National Children` alone scores 340 differing cells under positional pairing +against 27 under the best pairing. `align_duplicate_rows` +(`src/a4d/migration/compare.py`) therefore pairs a duplicate group by minimum +differing cells, brute-forced (groups are at most 4 rows; bounded at 6), and +keeps positional order unless another pairing is *strictly* better. + +**Why this is not just minimising the count.** Raw is the control: there, +positional order is verifiable ground truth, and content matching reproduced it +exactly, changing nothing (26,697 mismatches before and after). The most the +tie-break can be hiding is the cleaned stage's 351-cell difference between the +two pairings, inside groups that are the same patient on the same sheet by +construction. + +**4. The rows now compared are the rows that should be compared.** +`row_key_overlap` is byte-identical before and after on both stages (raw +82,653 matched / 118 R-unmatched / 12 Python-unmatched; cleaned 81,738 / 121 / +15). No pairing was gained or lost -- only *which* row pairs with which inside +a duplicate group changed. Per file, raw unclassified went 770 -> 0, 22 -> 0 +and 16 -> 0 in the three files; cleaned went 789 -> 7 (VNC 2024), 146 -> 2 +(NPH), 73 -> 32 (Surat Thani), 39 -> 17 and 148 -> 128. Product is a +regression check and is unchanged to the cell (cleaned 22,716 mismatches, raw +112, row-key divergence 0). + +**5. Residual after realignment.** Raw: **601 unclassified**, the long tail +[ticket 46](46-triage-patient-raw-residual-4.md) owns; nothing left in the +three duplicate-key files. Cleaned: **8,383 unclassified**, no ticket's scope +yet -- the remaining 7 in VNC 2024 are `hba1c_updated_date` (5) and +`fbg_updated_date` (2). + +**6. Handed over, not absorbed.** The duplicated IDs are a source defect: 21 +patients listed twice on `2024_Vietnam National Children`'s `Jul24`, one on +`2023_Vietnam National Children's` `Jun23`, one on `2018_Penang General +Hospital_DC` `Oct18` -- derived from Python's raw output, not from the ticket's +prose. Recorded on [ticket 40](40-source-defect-findings-report.md). + +Chasing why the cleaned stage has *two extra* duplicate files surfaced +something else: in `2023_NPH`, four distinct raw identities (`KH_NPH026`, +`KH_NPH027`, `KH_NPH028`, `KH_NPH029`) arrive at the cleaned stage as one +`KH_NPH02`. Across all 254 trackers, 4 files lose 9 identities this way. R does +the same thing on the same file, so it is not a Python regression and not an +R/Python divergence -- which is exactly why no comparison-based ticket would +ever have found it. Split into [ticket +47](47-patient-ids-merged-at-cleaning.md) rather than chased here. + +**Because.** The ticket's open question was how widely to adopt positional +alignment, and the answer is nowhere: product needed a positional key because +it had no usable natural key, whereas patient's key is unique in 249 of 254 +files, so the only thing missing was a tie-break. Grouping the existing +`add_row_ordinal` by the identity key gives that tie-break using the exact +mechanism ticket 17 already validated, without trading an identity check for a +positional one anywhere. + +**Rejected.** +- *Product's unconditional positional key for patient.* Would discard + `patient_id` from the alignment in 249 files that do not need it, and import + product's cascade failure mode (ticket 21's `row_order_divergence`), where a + single row-membership difference shifts every later row in the group. +- *Switching strategy per file where the key is ambiguous.* Needs a file list + -- derivable, but still machinery -- and buys nothing: grouping by the + identity key already degenerates to today's behaviour wherever the key is + unique. +- *Occurrence-order tie-breaking (the ticket's own proposal).* Correct at raw, + measurably wrong at cleaned: 9 groups pair the wrong two copies, leaving 351 + false mismatches. Kept as the fallback for uneven or oversized groups. +- *Applying this to raw only, as the ticket's title scopes it.* The defect and + the key are shared by both stages, and the cleaned stage held the larger + share. It does move counts on a stage other tickets closed against -- flagged + below. +- *Chasing the `KH_NPH02` identity merge here.* A production data-correctness + bug in cleaning, not a comparison-alignment question, and not an R/Python + divergence at all. + +**Evidence.** All **executed**. Duplicate-key incidence and fan-out counted per +file with duckdb directly against `output_r`/`output_python` on the USB drive +(245 R files, 254 Python); both pairings brute-forced over every two-sided +duplicate group in the real parquets; a full `just compare-outputs`-equivalent +run before and after each of the two designs, with per-file and per-column +counts read from the resulting workbooks; `row_key_overlap` compared before and +after; the `KH_NPH02` collapse confirmed in both R's and Python's own cleaned +output and traced to 4 files / 9 identities across the whole set. TDD: 8 new +tests written before the implementation (`TestPatientOccurrenceOrdinalKey`, +`TestStageWiring`). Full suite 727 passed, 1 skipped; ruff check, ruff format +and `ty check src/` clean. One **read**-only claim: `clean/patient.py`'s +`patient_id` normalization regex was read but *not* confirmed to be the cause +of the `KH_NPH02` merge -- it does not obviously produce it, which is why +ticket 47 exists rather than a fix. + +**Tense.** Every count describes current behaviour after the change, except the +1,409 / 9,392 / 27,510 / 114,473 starting figures, which describe baseline run +`2026-08-17T192144Z`. + +**Note for the tickets that measured the cleaned stage.** Tickets 28, 29, 37 +and 38 recorded cleaned-stage counts taken under the old key. Their causes are +unaffected (nothing they diagnosed lives in a duplicate-key group), but their +numbers no longer reproduce; [ticket +44](44-triage-cleaned-fbg-r-null-residual.md) should re-measure against +`2026-08-17T202809Z` rather than the figures in its premise. + +Commit: see branch `migration`. diff --git a/docs/wayfinder/tickets/46-triage-patient-raw-residual-4.md b/docs/wayfinder/tickets/46-triage-patient-raw-residual-4.md new file mode 100644 index 0000000..4d01981 --- /dev/null +++ b/docs/wayfinder/tickets/46-triage-patient-raw-residual-4.md @@ -0,0 +1,193 @@ +--- +id: 46 +title: Triage the residual patient raw-stage column mismatches (round 4) +labels: [wayfinder:task] +status: closed +blocked_by: [45] +assignee: session-2026-08-17d +claimed_at: 2026-08-17T22:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 43 +--- + +## Premise + +Rests on [Triage the residual patient raw-stage column mismatches (round +3)](43-triage-patient-raw-residual-3.md), closed, which took patient +raw-stage unclassified from 1,879 to **1,409** by settling two causes: a +float-to-string rounding artifact reaching four columns the normalize list +could not name (434 rows, comparison-tool fix), and 24 invented rows in +`2024_Vietnam National Children`'s `Jul24` that Python was reading from a +bare list of patient IDs below the data block (pipeline fix). + +Blocked on [Give the patient comparison an ordinal row key](45-patient-row-alignment-duplicate-keys.md) +deliberately: 808 of the 1,409 are join fan-out in three duplicate-key files, +so triaging the tail before that lands would mean characterizing noise. What +this ticket inherits is the **~600 rows outside those three files**. + +**Ticket 45 is now closed, and the inherited residual is exactly 601** -- +measured on baseline run `output/comparison/2026-08-17T202809Z`, which is the +run to triage against (not `2026-08-17T192144Z`, whose numbers this ticket's +question section quotes). All three duplicate-key files now hold **zero** +unclassified raw mismatches. Patient rows are now aligned on `patient_id` + +`sheet_name` with a content-matched tie-break inside duplicate groups, so a +remaining mismatch is between two rows that genuinely describe the same patient +on the same monthly sheet. + +Also rests on the map's standing scoping rule that the current tracker +template is the golden rule, and on ticket 43's own warning that early +sampling of these columns predates three extraction changes (ticket 30's +blank-header recovery, ticket 31's duplicate-source-column merge, ticket 43's +phantom-row fix) -- so re-measure rather than trusting any characterization +written before baseline run `output/comparison/2026-08-17T192144Z`. + +## Question + +Triage what remains once ticket 45 has realigned the duplicate-key files. +Shapes ticket 43 measured but did not close, all against the pre-45 baseline: + +- **`hba1c_updated` / `fbg_updated_mg` interior space, 86 rows -- Python + already verified right, cause not yet named.** All 86 are in `2017_Yangon + Children's Hospital`, R holding `8.8(20.9.16)` where Python holds `8.8 + (20.9.16)`. The source cell (`Feb17`, row 62, column 12) was read directly + and **contains the space**, so Python reproduces the workbook exactly and R + drops it. Ticket 43 searched R's raw path for the mechanism and did not + find it (`sanitize_str` touches only column names and validator lookups; + the multiline-header merge touches only headers; the wide-format splitter + is product-side). Either locate it or wire a classifier that says what was + verified -- that Python matches source -- without claiming a mechanism. +- **`insulin_regimen` (234).** Ticket 43 did not reach this. 194 rows are + ticket 30's source-verified blank-header recovery in `2021_Kantha Bopha`; + the other ~40 share the shape but were never verified, which is exactly why + ticket 31 refused to wire the column to `r_extraction_gap` wholesale. + Verify the unverified ones against source, or split the classifier so it + fires only where the evidence reaches. +- **`observations` (55), `observations_category` (42), `status` (43), + `last_clinic_visit_date` (82), `fbg_updated_date` (74), `hba1c_updated` + (87)** and ~40 more columns under 45 rows each. Expect a large share of the + date-column entries to be the duplicate-key fan-out and to disappear with + ticket 45 -- re-measure first, then triage what survives. +- **Cheap representation shapes worth one decision each**: R `FALSE` vs + Python `False` (20 rows, `clinic_visit` and `remote_followup`); R `null` vs + Python `""` (25 rows, mostly `insulin_injections`); a trailing space inside + a merged sub-value (`"Normal ,Insulin"` vs `"Normal,Insulin"`), which + ticket 31's merge strips and R's `unite` does not -- decide whether to + extend `normalize_whitespace_column` to interior whitespace or classify. + +Split further rather than leaving this open-ended if it does not converge. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: explaining +a difference and naming a cause is not enough -- each needs an explicit +verdict on whether Python is right, with the evidence. Where Python is losing +information the source carried, fix the pipeline. Where the source workbook +itself is wrong, that is a legitimate conclusion and a finding for [ticket +40](40-source-defect-findings-report.md). A cause genuinely undecidable on +the available evidence is recorded as an open question, not closed with a +label. + +## Resolution + +**Decision.** Four of the residual's causes are settled; the rest split into +[round 5](49-triage-patient-raw-residual-5.md) and, separately, into a +newly-found Python data loss, [ticket +48](48-putrajaya-screening-columns-lost.md). Patient raw-stage unclassified +fell **601 -> 278 (-54%)**, cleaned **8,383 -> 8,148**. The new baseline run +is `output/comparison/2026-08-17T212151Z`. + +1. **A number typed into a date-formatted cell -- a Python bug, fixed.** + `read_patient_rows` (`src/a4d/extract/patient.py`) now converts a datetime + before 1903 back to its Excel serial. Excel's epoch is 1899-12-30, so such + a "date" is a small number that inherited a date format from a neighbouring + cell; readxl reads the number because it guesses the column's type, while + openpyxl honours each cell's own format and hands back a datetime, which + the numeric conversion then rejected into the 999999 sentinel. 38 raw cells + across 5 files; **in production this was destroying 24 real systolic + readings** (`2025_Hat Yai` TH_HY035 and `2025_YGH` MM_YC023_YG, one per + monthly sheet). Verdict: Python was wrong, R was right by accident. +2. **R drops a whitespace-only rich-text run -- classifier + `r_drops_richtext_space` (89 rows).** Ticket 43 left this "verified but + unexplained"; the mechanism was found in the workbooks' own XML. Where a + cell carries mixed formatting, xlsx stores it as a sequence of runs, and a + space between two differently-formatted fragments becomes a run holding + only that space (` `). openpyxl concatenates + every run; readxl drops the whitespace-only one. Verified in two unrelated + workbooks -- `2017_Yangon` (`8.8 (20.9.16)`, three runs) and `2022_Mahosot` + (`Unable to contact`, the middle run recoloured). The same text also exists + as a plain shared string elsewhere in the same column, which is why only + some cells diverge. Python reproduces the workbook; R loses a character. +3. **Python trims each sub-value before a merge -- classifier + `python_trims_merged_subvalue` (2 rows).** The opposite direction of the + same shape: ticket 31's merge strips each fragment, `tidyr::unite` does + not, so a source cell with trailing padding reaches R as `Normal ,Insulin`. + Direction-scoped so the two classifiers cannot shadow each other. +4. **`insulin_regimen` in `2021_Kantha Bopha` -- `r_extraction_gap` (194 + rows).** Re-verified at source rather than taken from ticket 30: `Mar21` + column Q has *both* header rows empty and holds `Self-mixed BD`. R has no + header, so it drops the column; Python recovers the name from the sibling + sheets that label it. All 194 of the column's residual sit in that one + verified file, which is what ticket 31's refusal was waiting for. + +**Because.** Each cause was taken to a verdict on which pipeline is right, not +to a label. The one that turned out to be a Python defect was fixed in the +pipeline; the three where Python reproduces the source became classifiers. + +**Rejected.** + +- *Recovering every date-typed cell in a numeric column.* This would also + convert `2023_Chiang Mai`'s `t1d_diagnosis_age` (Excel serial 20668, + 1956-08-01, against a D.O.B. of 2009) into a bogus age of 20668 -- exactly + what R does. Only the impossible range is recovered; a date that could + plausibly have been typed stays a date, which is the cause the product arm + already settled as `openpyxl_date_typed_stray_cell`. +- *A cell-count or magnitude threshold* ("recover if the number looks like a + blood pressure"). A guess about the column's semantics; the Excel-epoch + argument is a fact about the file format. +- *Extending `normalize_whitespace_column` to interior whitespace.* It would + have silenced both space causes with one rule and hidden the fact that they + run in opposite directions and have different mechanisms -- and it would + have been applied before the mechanism was known. +- *Wiring `r_extraction_gap` across every R-null column at once.* ~125 of the + remaining 278 share that shape, but only `insulin_regimen`'s file has been + source-verified. That is ticket 31's precedent, kept. +- *Fixing the Putrajaya screening-column loss in the same session.* It is a + second, unrelated header-handling bug needing its own sweep across all 254 + trackers; chasing it here would have been the sprawl the map's rule forbids. + +**Evidence: executed.** The four causes were measured against the real +248-tracker drive data and verified at source: the Hat Yai systolic cell read +directly from `Annual!H` (`datetime(1900, 4, 29)`, number format +`dd-mmm-yyyy`, diastolic 74 beside it); the rich-text runs read out of +`xl/sharedStrings.xml` in both workbooks; the Kantha Bopha blank header read +from `Mar21` rows 85-87. The pipeline was re-run over all 254 trackers +(`a4d run patient --force`) and the comparison re-run twice, so every count +here is a measured before/after, not a projection. R's `sanitize_str` was +re-read to confirm independently of ticket 43 that it touches only column +names and validator lookups. Full suite 735 passed, ruff and `ty check src/` +clean. + +**Tense.** Every count above describes current behaviour after this session's +changes. The 999999 systolic readings describe production output *before* the +fix; the drive's `output_python` and BigQuery's contents now differ until the +next production run. + +## Addendum: what this leaves open + +- The 278 remaining raw-stage cells: [ticket + 49](49-triage-patient-raw-residual-5.md). +- The Putrajaya merged-header data loss: [ticket + 48](48-putrajaya-screening-columns-lost.md). +- **Product's own sub-1903 cells were measured but deliberately not touched**: + 18 `product_units_received` and 2 `product_entry_date` cells carry the same + impossible-date shape (e.g. `1900-03-15`, i.e. 74 units). They currently + land on `openpyxl_date_typed_stray_cell`, whose verdict is "Python is + faithful" -- which this session's evidence says is wrong for that sub-range. + Product raw is separate extraction code and separate, already-closed triage + scope, so it belongs to [ticket + 32](32-audit-classifiers-against-decision-bar.md)'s re-audit rather than + here. +- Three new source-defect findings were added to [ticket + 40](40-source-defect-findings-report.md). diff --git a/docs/wayfinder/tickets/47-patient-ids-merged-at-cleaning.md b/docs/wayfinder/tickets/47-patient-ids-merged-at-cleaning.md new file mode 100644 index 0000000..a5faaa5 --- /dev/null +++ b/docs/wayfinder/tickets/47-patient-ids-merged-at-cleaning.md @@ -0,0 +1,250 @@ +--- +id: 47 +title: Four trackers where cleaning merges several patients into one patient ID +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-20b +claimed_at: 2026-08-20T12:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 45 +--- + +## Premise + +Rests on [Give the patient comparison an ordinal row key](45-patient-row-alignment-duplicate-keys.md), +closed, which established that `patient_id` + `sheet_name` is patient's real +identity key and made duplicate-key groups visible per file and per stage. Two +of the five cleaned-stage duplicate groups it found have no raw-stage +counterpart, which is how this surfaced: the duplication is created *by +cleaning*, not read from the source. + +Measured directly against the real 254-tracker output on the USB drive +(executed, not inferred): + +- `2023_NPH A4D Tracker`, sheet `Sep23`: four distinct raw identities -- + `KH_NPH026`, `KH_NPH027`, `KH_NPH028`, `KH_NPH029` -- arrive at the cleaned + stage as a single `KH_NPH02` with four rows. Every other patient in that file + is keyed `KH_NP0xx`, so the four carry an odd prefix in the source. +- Across all 254 trackers, **4 files lose 9 identities** between raw and + cleaned (distinct `patient_id` count drops). Row counts are preserved -- + nothing is dropped or invented, the identities are merged. + +**R does exactly the same thing on the same file**, so this is not a Python +regression and not an R/Python divergence. That is why no comparison-based +ticket could have found it: both sides agree, and agreement is what the +comparison tool is built to stay quiet about. It also means this ticket does +not need R alive to answer -- it is a question about the Python pipeline and +the source workbooks. + +`clean/patient.py`'s `_apply_preprocessing` normalizes `patient_id` by +converting hyphens to underscores and keeping `^([A-Z]+_[^_]+)` -- documented +as removing a transfer-clinic suffix (`MY_SM003_SB` -> `MY_SM003`). Ticket 45 +**read** this and could not see how it produces `KH_NPH026` -> `KH_NPH02`, +since that value has only one underscore. Treat the mechanism as unlocated, not +as diagnosed. + +## Question + +Find out what merges these identities, and decide what the pipeline should do. + +1. **Locate the mechanism.** Not the `patient_id` regex on its face -- read the + actual value at each step (raw parquet, preprocessing, the Patient List + join) for one of the four `2023_NPH` rows rather than reasoning from the + code. The Patient List join is the prime suspect, since the cleaned + identity may be coming from the Patient List rather than the monthly sheet. +2. **Decide whether the source is wrong or the pipeline is.** Open + `2023_NPH A4D Tracker.xlsx` and see what the `Sep23` sheet and the Patient + List actually record for these four patients. If the workbook itself + conflates them, that is a finding for [ticket + 40](40-source-defect-findings-report.md), not a pipeline fix. If the + pipeline is collapsing identities the workbook keeps distinct, that is + patient data being attributed to the wrong person, and the pipeline gets + fixed. +3. **Name the other three files.** Ticket 45 counted them (4 files, 9 + identities) without listing them; derive the list rather than assuming they + share `2023_NPH`'s shape. + +## Standing bar + +Per the map's **triage means deciding, not labelling**: a named cause is not +enough -- say explicitly whether the merged identity is correct, with what was +read in the source workbook. This one carries more than a mismatch count: if +cleaning is merging distinct patients, the production `patient_data_*` tables +attribute one patient's monthly records to another. + +## Resolution + +**Decision.** `fix_patient_id` (`src/a4d/clean/validators.py`) no longer +truncates. An ID that fails the `XX_YY###` format is now **recovered** against +the well-formed IDs the same tracker carries elsewhere -- accepted only at edit +distance 1 and only when exactly one candidate fits -- and **sentinelled to +`Undefined`** when no unambiguous candidate exists. Every malformed ID is +reported to the `ErrorCollector` whether recovered or sentinelled, since either +way the source workbook needs correcting (user's explicit instruction). + +The user chose this shape: "C as first rule and B as fall back" -- recover +where the tracker's own Patient List settles the intended identity, sentinel +otherwise, because "otherwise we publish patient IDs that are not in any +tracker." + +**1. The mechanism, located by execution.** Not the `patient_id` regex in +`_apply_preprocessing` (step 2), which the ticket's premise correctly refused +to accept as diagnosed. Bisecting all eleven cleaning steps over the real +`Sep23` slice put the change in `validate_all_columns` (step 7), which calls +`fix_patient_id` last. Its rule: non-conforming and longer than 8 characters +-> truncate to the first 8; 8 or shorter -> `Undefined`. `KH_NPH026` is 9 +characters, so all four collapsed onto `KH_NPH02`. + +This was a faithful port. R's `fix_id` +(`r-archive/R/script2_helper_patient_data_fix.R:681-728`) has the identical +asymmetry -- `str_sub(id, 1, 8)` for long, `ERROR_VAL_CHARACTER` for short. +The premise's hypothesis of a hyphenated source spelling (`KH_NPH02-6`) is +**dead**: the raw parquet holds `KH_NPH026` with no hyphen. + +**2. The source is wrong, and the pipeline made it worse.** Read directly from +`Cambodia/NPH/2023_NPH A4D Tracker.xlsx`, rows 128-131 of each sheet: the +`Patient List`, `Oct23`, `Nov23` and `Dec23` all write `KH_NP026`-`KH_NP029`; +only `Sep23` writes `KH_NPH026`-`KH_NPH029`. One sheet, one stray `H`, at the +patients' enrolment month. + +So the workbook is defective -- a finding for [ticket +40](40-source-defect-findings-report.md). But truncation did not merely fail to +repair it: it manufactured `KH_NPH02`, an identifier present in no source +workbook anywhere, and filed four different people's September records under +it. The function was also self-inconsistent -- a 7-character bad ID was +honestly sentinelled while a 9-character bad ID was silently reshaped into +something that looks real. + +**3. The other three files are three different mechanisms, and two are the +pipeline working correctly.** Ticket 45's count of "4 files, 9 identities" is +right; its implied uniform shape is not. + +| File | What happens | Verdict | +|---|---|---| +| `2023_NPH A4D Tracker` | 4 IDs truncate onto `KH_NPH02` (-3) | **Wrong** -- source typo amplified into a false merge | +| `2021_Mahosot Hospital A4D Tracker_DC` | `LA-MH056/057/058` (Jun-Nov) merge into `LA_MH056/057/058` (Dec) (-3) | **Correct** -- same patients, hyphen vs underscore across months | +| `2026_Surat Thani Hospital A4D Tracker_Jun_26` | `TH-ST029` merges into `TH_ST029` (-1) | **Correct** -- same patient, two spellings | +| `2026_NOGH T1D Tracker_June_26` | `MM_NO97`, `MM_NO98`, `MM_NO99` (7 chars) all -> `Undefined` (-2) | **Source defect** -- 18 rows of 3 distinct patients share one sentinel | + +**Recovery is deliberately narrow, and the NOGH case is why.** `MM_NO97/98/99` +look like a missing leading zero, but that workbook's **own Patient List** +writes `MM_NO97/98/99` too, and it has `MM_NO090`-`MM_NO096` with no +`MM_NO097`. Nothing in the tracker licenses inventing one, so they stay +`Undefined` -- unchanged by this fix, and reported. Same for +`2026_YGH T1D Tracker_June_26`'s `MM_NO55_MW_YG` (3 rows, one identity, no +merge). + +**Rejected.** +- *Change nothing / stay bit-faithful to R.* Rejected by the user: it keeps + publishing an identity no tracker contains. +- *Sentinel only, never recover (option B alone).* Rejected as the primary + rule, kept as the fallback. It removes the misattribution but throws away + four recoverable rows when the tracker itself says what was meant. +- *Recover by fuzzy-matching the Patient List with a looser rule.* Not + attempted. Edit distance 1 plus a uniqueness requirement is what makes the + recovery a reading rather than a guess; anything looser risks attaching a + mistyped ID to a different real patient, which is the exact failure this + ticket exists to remove. +- *Fix the extraction-side Patient List join in the same session.* Split out -- + see the spawned ticket below. + +**Evidence: executed.** The mechanism was found by bisecting the cleaning +steps over real data, not by reading the code (reading is what left the premise +with the wrong suspect). The source verdict was read from the workbook's own +cells. The blast radius was measured by re-cleaning **all 254 raw parquets** on +the USB drive with the new code and diffing every file's `patient_id` +distribution against the existing cleaned output: exactly one file changes. + +- `2023_NPH A4D Tracker`: `KH_NPH02` (4 rows) disappears; `KH_NP026`, + `KH_NP027`, `KH_NP028`, `KH_NP029` each go 3 rows -> 4. +- Every other tracker is byte-identical, including NOGH, YGH, Mahosot DC and + Surat Thani. +- Corpus-wide, the cleaned output now contains **zero** patient IDs that fail + the `XX_YY###` format (previously one: `KH_NPH02`). +- The recovered rows attach to the right people: `KH_NP026`'s September row + joins a patient whose own `t1d_diagnosis_date` is 2023-09-04. + +Full suite 865 passed / 1 skipped, `ruff check`, `ruff format --check` and +`ty check src/` all clean. Six new tests in +`tests/test_clean/test_validators.py`; three pre-existing tests that asserted +R's truncation were rewritten to the decided behaviour rather than deleted. + +**Tense.** Everything above describes current behaviour on this branch after +the fix, except the R descriptions, which are of `r-archive/` as frozen. + +## Spawned + +- **The recovered rows have no demographics, because extraction joins the + Patient List on the *unfixed* ID.** `KH_NP026`'s September row now carries + the right identity but null `dob`, `sex`, `province` and + `t1d_diagnosis_date`, where its October-December rows have all four. The + Patient List join happens in `extract/patient.py` (~line 1107), long before + `fix_patient_id` runs in cleaning, so a misspelled monthly ID silently misses + its own demographics. This is a second, independent defect of the same source + typo, and its blast radius is not the 4 NPH rows alone -- it plausibly also + covers the hyphen-spelled Mahosot and Surat Thani rows, whose Patient List + entries use underscores. Split out rather than fixed here: changing the join + key changes matching for every tracker, which is a different measurement. + +## Reported to ticket 40 + +- `2023_NPH A4D Tracker`, `Sep23`, rows 128-131: `KH_NPH026`-`KH_NPH029` + should be `KH_NP026`-`KH_NP029` (stray `H`; the same workbook's Patient List + and its Oct/Nov/Dec sheets spell them correctly). +- `2026_NOGH T1D Tracker_June_26`: `MM_NO97`, `MM_NO98`, `MM_NO99` are + 7-character IDs where the template wants 8, in the Patient List and every + month sheet. Three distinct patients, currently all published as + `Undefined` (18 rows). Cannot be repaired in the pipeline -- the workbook + has to say what was meant. +- `2026_YGH T1D Tracker_June_26`: `MM_NO55_MW_YG` reduces to `MM_NO55`, again + 7 characters. One patient, 3 rows, published as `Undefined`. +- `2021_Mahosot Hospital A4D Tracker_DC`: `LA-MH056/057/058` in Jun-Nov vs + `LA_MH056/057/058` in Dec -- the same patients spelled two ways within one + workbook. The pipeline merges them correctly; the workbook should still be + made consistent. +- `2026_Surat Thani Hospital A4D Tracker_Jun_26`: `TH-ST029` and `TH_ST029` + both appear in `May26`, so that patient has two rows for one month after + normalization -- likely a duplicated entry in the source. + +## Addendum (same session, after closure): the comparison was re-run + +The resolution above was recorded before the R/Python comparison had been run +against the changed output. The user asked for it immediately after, so it is +recorded here rather than left as an open thread. + +The patient arm was regenerated with `a4d run patient` against the real +254-tracker set on the USB drive (85,300 monthly rows), replacing +`output_python/`'s patient stages; product was untouched and its report +confirms "no change from the previous run". `just compare-outputs` was then run +against the frozen `output_r/`. + +**The divergence this fix creates is row-key non-overlap, not cell mismatches** +-- which is worth stating plainly, because a future session scanning the +`cell_mismatches` sheet will not find it there. Patient's row-alignment key is +`patient_id` + `sheet_name`, so R's four `KH_NPH02` rows and Python's four +recovered `KH_NP026`-`KH_NP029` `Sep23` rows no longer pair with each other: + +- Row-key divergence, whole patient cleaned stage: **6 files / 121 R-unmatched + / 15 Py-unmatched -> 7 files / 125 / 19**. The entire delta is + `2023_NPH A4D Tracker` gaining 4 R-unmatched and 4 Py-unmatched. No other + file moved. +- Cell mismatches, whole patient cleaned stage: **114,373 -> 114,371**. Both + removed cells are FBG (`fbg_updated_mg|python_glucose_unit_corrected` 5,341 + -> 5,340 and `fbg_updated_mmol|unclassified` 2,936 -> 2,935), from two of + those same four rows, which previously paired with R and contributed a + mismatch each. +- **No new `unclassified` cells**, and the per-column table shows no other + column moving at all. +- Patient raw stage: **zero delta**, as expected -- the fix is in cleaning. + +No cause classifier was added. The cause registry classifies *cell* mismatches +and this divergence never reaches a cell comparison, so a classifier could not +fire on it; the standing entry in the map's Decisions-so-far is where it is +documented instead. This is the same shape as the ticket-42 unit swap, which +the map's fog already records as a finding no per-cell classifier can see. + +**Tense:** these are current measured numbers on this branch, against +`output_r/` as frozen. diff --git a/docs/wayfinder/tickets/48-putrajaya-screening-columns-lost.md b/docs/wayfinder/tickets/48-putrajaya-screening-columns-lost.md new file mode 100644 index 0000000..77042ac --- /dev/null +++ b/docs/wayfinder/tickets/48-putrajaya-screening-columns-lost.md @@ -0,0 +1,197 @@ +--- +id: 48 +title: Python drops complication-screening results and dates where a merged header spans the block +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-17e +claimed_at: 2026-08-17T23:30:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 46 +--- + +## Premise + +Rests on [Triage the residual patient raw-stage column mismatches (round +4)](46-triage-patient-raw-residual-4.md), closed, which found this while +triaging and split it out rather than chasing it in the same session. It is a +Python data loss, not a triage question, which is why it is its own ticket. + +Rests also on [Triage the patient pipeline's raw-stage column-existence +divergence](30-triage-patient-raw-column-divergence.md) (blank-header recovery +from sibling sheets) and [Triage the residual patient raw-stage column +mismatches (round 2)](31-triage-patient-raw-residual-2.md) (merging several +source columns that share one canonical name), because the defect sits in the +same header-handling code both changed and any fix has to leave both working. + +## Question + +In `2021_Putrajaya Hospital A4D Tracker_DC`, the monthly sheets lay the +complication-screening block out as (verified against the source, `Dec21` header +rows 76/77): + +| col | header row 1 | header row 2 | +|-----|--------------|--------------| +| 31 | `Complication Screening \n(Current Month Testing)` | `Select for Drop Down` | +| 32-34 | *(blank -- the row-1 header is merged across the block)* | *(blank)* | +| 35 | *(blank)* | `Results` | +| 36 | *(blank)* | `Date (mmm-yy)` | +| 37 | `Other Patient\nObservations` | | + +Python's raw output for `MY_PJ001`/`Dec21` carries: + +- `complication_screening` = the col-31 selection (correct) +- `observations` = `Foot Examination (Nerves)` -- a *screening selection*, not + an observation +- a literal, unmapped column named `Results` holding + `Urine Albumin:normal Foot:normal TG:4.2 ...` +- no `complication_screening_results`, no `complication_screening_date` + +R gets all three right. Measured on comparison run +`output/comparison/2026-08-17T212151Z`: 11 `complication_screening_results` +cells where R has the recorded results and Python has null, plus +`complication_screening_date` and the mis-filed `observations` rows -- all in +this one file, ~25 cells in total. + +To settle: + +1. **Which mechanism actually does it** -- the merged row-1 header not being + propagated across columns 32-36, blank-header recovery filling column 32 + from a sibling sheet that labels it differently, or the duplicate-column + merge. Instrument rather than infer; three changes have landed in this code + since it was written. +2. **How many other files it touches.** One file is what the comparison + surfaces, but the comparison only sees columns R also has. Sweep all 254 + trackers for the same layout (a merged header spanning a block whose + sub-headers live in row 2 only) before deciding the fix's shape. +3. **Whether the fix belongs in header merging or in the synonym map.** + Mapping the bare `Results`/`Date (mmm-yy)` sub-headers would fix this file + and nothing else; propagating a merged header across its span is the + general rule, and is what R does. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: the +verdict here is already clear (Python loses data the workbook records and R +reads), so this ticket owes a pipeline fix, a regression test, and a +before/after count against the real drive data -- not a classifier. + +## Resolution + +**Decision.** A merged upper header now names every column its span covers, but +only where that column has a sub-header of its own to qualify. +`merged_header_spans()` (`src/a4d/extract/patient.py`) reads the sheet's +`` ranges straight from the XML in the archive openpyxl already has +open, and `merge_headers()` takes them as `merged_spans`. Separately, +`recover_blank_headers()` now abstains for any position inside such a span. +Both were needed: they are two independent mechanisms that happened to damage +the same block. + +**Because.** Question 1 asked which mechanism does it, and instrumenting rather +than inferring found *both* of the candidates it listed, doing different harm: + +1. *The merged title never propagates.* `merge_headers`' forward-fill carries + the upper header rightward in `prev_h2` but executes `prev_h2 = None` on any + column blank in both header rows. In `2021_Putrajaya` `Dec21` the block + title is merged across columns 31-36 (`AE76:AJ76`, verified in the file) and + columns 32-34 are blank in both rows, so by column 35 the title is + forgotten and `Results`/`Date (mmm-yy)` are emitted bare. The synonym map + **already contains** `Complication Screening (Current Month Testing) + Results` and its `Date (mmm-yy)` twin -- the fill would have worked; only + the reset stopped it. +2. *Blank-header recovery then misfiles the leftovers.* Putrajaya's month + sheets do not share one layout: `Jul21` carries `Patient Observations` at + the position `Dec21` uses for a screening selection. Ticket 30's + sibling-donor rule saw exactly one donor and filed `Foot Examination + (Nerves)` under `observations`. This is why the fix could not be a synonym + entry (question 3): mapping the bare sub-headers would have left this half + untouched. + +Question 2 asked how many other files it touches, and the answer changed the +fix's shape twice. Sweeping all 254 trackers found **693 sheets across 64 +files** with a merged upper header, not one file -- and naive propagation was +actively harmful on two of them: + +- The 2022 template merges `Insulin Regimen` across two columns whose second + holds a **near-duplicate** of the first (`Basal-bolus MDI (AN/HI)` against + `Basal-bolus (AN/HI)`, 3,659 cells across 10 trackers). Naming both would + comma-join them into one worse value. R and Python currently **agree** here, + both taking the first column, so this would have been a self-inflicted + divergence on data that was never wrong. +- In the 2022 complication-screening block, a column blank in both header rows + would take the bare block title, which maps to `complication_screening` -- + a column the block's own first column already claims. Measured: **290 sheets + would gain a second column competing for one standard name.** + +Hence the rule adopted: a merged title *qualifies a sub-header*, it never names +a column outright. A column with no sub-header has no name to qualify, and +guessing the block title for it is what caused both hazards. That one rule +drops the collision count to **0** and excludes every Insulin Regimen cell, +with no measured loss -- of the 225 cells that bare-title propagation would +have "recovered", the only mapped ones were the 164 collision cases. + +**Rejected.** + +- *Stop resetting `prev_h2` on a blank column* (the one-line fix). Makes + forward-fill unbounded, so a title leaks into a block it does not cover. The + reset is load-bearing. +- *Add `Results`/`Date (mmm-yy)` to the synonym map.* Fixes this file and + nothing else, does not touch the `observations` misfiling, and bare + sub-headers like `Date` recur under several unrelated blocks. +- *Propagate the bare title into columns with no sub-header.* Rejected on + measurement, above -- 290 standard-name collisions and 3,659 comma-joined + insulin cells. +- *Load the workbook read-write for openpyxl's own merge metadata.* Gives up + ticket 10's 6.6x; `merged_header_spans` reads the same information from the + already-open archive instead, and degrades to today's behaviour if the two + private attributes it uses ever move. + +**What this gives up.** Putrajaya's columns 32-34 hold three further screening +selections (`Foot Examination (Nerves)`, `Lipid profile`, `TSH`) that Python +still drops, because they have no sub-header. R keeps them, but only in +unmapped suffixed columns (`...selectfordropdown1/2/3`), so neither pipeline +carries them into mapped output and the comparison stays quiet about it. Both +pipelines keep only the first selection of a multi-select field. Recorded as +fog rather than fixed here -- it is a shared limitation, not a divergence. + +**Evidence: executed.** Every number below was measured, not read. + +- Source layout read directly from `2021_Putrajaya Hospital A4D Tracker_DC` + `Dec21`: header rows 76/77, merged ranges `AE76:AJ76` and `AE77:AH77`. +- Full-run before/after against the real 248-tracker drive data + (`a4d run patient --force`, 254/254 processed, then `just compare-outputs`), + baseline run `output/comparison/2026-08-17T212151Z` -> current + `2026-08-17T222225Z`: + - `complication_screening_results`: **11 -> 0** + - `complication_screening_date`: **31 -> 0** + - `observations`: 8 unclassified -> **1**, the other 7 now matching the + existing `r_na_unite_padding` classifier because Python's value is + correctly null + - patient raw unclassified **278 -> 229**, cleaned **8,148 -> 8,141** + - no other column moved in either stage; product byte-identical +- Sweeps over all 254 trackers produced the 693/64, 3,659/10 and 290-sheet + figures above; the collision count under the final rule is 0. +- `2023_Chiang Mai`'s R output carries `complicationscreeningcompletedtsh` -- + R propagates merged headers too, so the renamed columns move Python toward R, + not away. +- Full suite 742 passed / 1 skipped, `ruff format --check`, `ruff check`, + `ty check src/` all clean. Four regression tests added + (`TestMergeHeadersWithMergedSpans`, `TestRecoverBlankHeadersInsideMergedSpan`) + covering the recovery, both hazards, and the 2021 Kantha Bopha recovery that + must keep working. + +**Tense.** Every claim above describes behaviour after this change, measured; +the 693/64 and 3,659/10 sweep figures describe the source workbooks and are +independent of it. + +**One comparison-tool change came with it.** The fix initially made +`complication_screening_date` look *worse* (31 -> 382). It was not: R holds the +Excel serial (`44474`) where Python holds the parsed date (`2021-10-05`) -- the +same date, and the same representation artifact tickets 20/23 already fixed +elsewhere. The column is absent from the cleaned schema (which splits screening +dates per test), so `get_date_columns()` cannot derive it, exactly like +`meter_received_date` in ticket 27. Appended to +`PATIENT_RAW_DATE_NORMALIZE_COLS` with that reason recorded, taking it to 0. +Ticket 49's premise predicted this check. diff --git a/docs/wayfinder/tickets/49-triage-patient-raw-residual-5.md b/docs/wayfinder/tickets/49-triage-patient-raw-residual-5.md new file mode 100644 index 0000000..3a671f6 --- /dev/null +++ b/docs/wayfinder/tickets/49-triage-patient-raw-residual-5.md @@ -0,0 +1,204 @@ +--- +id: 49 +title: Triage the residual patient raw-stage column mismatches (round 5) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-18 +claimed_at: 2026-08-18T10:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 46 +--- + +## Premise + +Rests on [Triage the residual patient raw-stage column mismatches (round +4)](46-triage-patient-raw-residual-4.md), closed, which took patient raw-stage +unclassified from **601 to 278** by settling four causes: a number typed into a +date-formatted cell (pipeline fix, 38 rows), R's dropped rich-text space +(classifier, 89), Python's trimming of merged sub-values (classifier, 2), and +`insulin_regimen`'s blank header in `2021_Kantha Bopha` (classifier, 194). + +**Updated 2026-08-17, after [ticket +48](48-putrajaya-screening-columns-lost.md) closed.** The residual is now +**229**, measured on baseline run `output/comparison/2026-08-17T222225Z`, which +is the run to triage against. Every count measured before it -- including the +278 below and every per-shape figure in the Question -- is historical. + +Ticket 48 removed **49** of the 278, not the ~25 first estimated: its +merged-header fix took `complication_screening_results` 11 -> 0 and +`complication_screening_date` 31 -> 0, and moved 7 `observations` rows from +unclassified onto the existing `r_na_unite_padding` classifier. Three shapes +listed below are therefore already gone -- see the Question's notes. + +Rests also on the map's standing scoping rule that the current tracker template +is the golden rule, and on the standing bar that a classifier records +understanding, never a verdict by itself. + +## Question + +Triage the 229. The shapes below were counted against the *old* 278-row +baseline; three of them are already resolved by ticket 48 and are struck +through in place rather than deleted, so the arithmetic stays checkable. +Re-measure the rest against `output/comparison/2026-08-17T222225Z` before +working them -- do not trust these numbers. + +- **R null, Python has a value (~125 rows, ~12 columns)**: + `last_clinic_visit_date` (40, one file), `fbg_updated_date` (25), + `blood_pressure_sys_mmhg`/`blood_pressure_dias_mmhg` (13 each), `edu_occ` + (12), `last_remote_followup_date` (4), `hospitalisation_cause` (3), + `other_issues` (2), `status` (1). Shape-identical to `r_extraction_gap`, but + ticket 31's refusal applies: verify per column against the source before + wiring, or the classifier labels without deciding. +- **A real date typed into a numeric column, or the reverse (51)**: + ~~`complication_screening_date` (31)~~ -- **resolved by ticket 48**: it was + exactly the predicted gap, a raw-only date column `get_date_columns()` cannot + derive, now appended to `PATIENT_RAW_DATE_NORMALIZE_COLS`. 31 -> 0. + `t1d_diagnosis_age` (16, source defect -- see [ticket + 40](40-source-defect-findings-report.md)), `blood_pressure_mmhg` (3), + `testing_frequency` (1). +- **`t1d_diagnosis_date` (23, two files)**: R 41837 vs Python 41821 in + `2017_Yangon` -- two different real dates 16 days apart, so neither side is + null and neither is obviously right. Unexplained. +- **Booleans (25)**: R `FALSE` vs Python `False` on `clinic_visit` and + `remote_followup`, one and two files. A representation difference; decide + normalization vs classifier. +- **R null vs Python empty string (25)**: `insulin_injections` (22), + `hba1c_updated` (3). Python emitting `""` where the source cell is blank + looks like a Python cleanliness defect worth fixing rather than classifying. +- **`dm_complications` (15, two files)**: `Kidney \nDamage` on both sides, + visually identical in the report. Inspect the bytes before assuming a cause. +- **`observations` (8 -> 1)**: mostly **resolved by ticket 48** -- 7 of the 8 + were Putrajaya rows where Python had a screening selection misfiled into + `observations`; with Python now correctly null they match the existing + `r_na_unite_padding` classifier. **1 row remains**: R's `... January ,NA` + against Python's `... January`, where the classifier still does not fire (the + trailing space before the comma is the likely reason). +- **`complication_screening` (2)**: Python holds two selections where R holds + one. Ticket 48 established that both pipelines keep only the *first* + selection of a multi-select block, so re-measure this before assuming it + still says what it says. +- **`hospitalisation_date` (5)**: Python's raw value is `2958352`, the Excel + serial for 9999-12-31 -- the date sentinel appearing at the *raw* stage, + which raw is not supposed to carry. + +Split further rather than leaving this open-ended if it does not converge. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: explaining a +difference and naming a cause is not enough -- each needs an explicit verdict on +whether Python is right, with the evidence. Where Python is losing information +the source carried, fix the pipeline. Where the source workbook itself is wrong, +that is a legitimate conclusion and a finding for [ticket +40](40-source-defect-findings-report.md). A cause genuinely undecidable on the +available evidence is recorded as an open question, not closed with a label. + +## Resolution + +**Decision.** Patient raw-stage unclassified goes **229 -> 84** (63%), settled +by four causes: two comparison-tool defects fixed, and two R limitations +root-caused to a named mechanism and classified. Python is right in every one +of the four; no pipeline change was needed, which is itself the finding -- +unlike ticket 46 and ticket 48, this round's residual held no Python defect. +The remaining 84 did not converge and are split into [ticket +50](50-triage-patient-raw-residual-6.md). + +**1. The whitespace target list was derived from the wrong schema (40 rows).** +`PATIENT_WHITESPACE_NORMALIZE_COLS` resolved through `get_string_columns()`, +i.e. the *cleaned* patient schema, but the raw stage stores every value as +text. That list cannot see a raw-only column (`dm_complications`, 15 rows, no +cleaned equivalent) and excludes as non-string the columns the cleaned schema +types `Float64` (`insulin_injections` 22, `hba1c_updated` 3), so readxl's +`trim_ws=TRUE` and its `\r\n` line endings kept surfacing on exactly those +three. This is the identical defect ticket 43 fixed for the *numeric* list, +never applied to the whitespace one. Added `whitespace_normalize_targets()` +(`src/a4d/migration/compare.py`) -- every string column, minus `__` join +helpers -- reusing the existing `ALL_RAW_COLUMNS` sentinel, wired to the +Patient (raw) stage only. The cleaned stage keeps the schema-derived list, +where the dtypes are real. + +`dm_complications` is why this went unfound for five rounds: the Excel report +shows both sides as byte-identical `Kidney \nDamage`, because DuckDB's xlsx +reader normalizes `\r\n` away on read. Unzipping the workbook and dumping +`xl/worksheets/sheet10.xml` shows R's `Kidney \r\nDamage` against Python's +`Kidney \nDamage`. **A shape that looks identical in the report is not +evidence the values are equal** -- inspect the bytes at the source. + +**2. R and Python spell an Excel boolean differently (20 rows).** readxl reads +a logical and R writes `FALSE`; openpyxl reads a Python `bool` and Python +writes `False`, on `clinic_visit` and `remote_followup`. Only a genuine Excel +boolean produces both spellings at once, so neither side is wrong. Checked +whether it reaches cleaned output: it does not -- **zero** mismatches on either +column at the cleaned stage, because cleaning canonicalizes both. That makes it +a raw-stage representation artifact, handled like `\r\n` and the date serials +rather than labelled: `normalize_boolean_literal_column()` folds only the exact +literals, so free text containing "false" is untouched. + +**3. R's header sanitizer keeps non-Latin script (44 rows).** `2022_Mukdahan` +appends Thai translations to two headers -- `Mar22!F44` is +`'Last Clinic \nVisit\nไปโรงพยาบาล'`, `G44` is `'Last Remote \nFollow +Up\nสายเข้า'`. R's `sanitize_str` strips `[^[:alnum:]]`, whose character class +is Unicode-aware, so the Thai survives: the header sanitizes to +`lastclinicvisitไปโรงพยาบาล`, `harmonize_patient_data_columns`' `match()` is +exact equality, no synonym matches, and R drops the column -- every row null. +Python's `sanitize_str` strips to `[^a-z0-9]`, reduces the same header to +`lastclinicvisit`, and matches. Executed both implementations on the real +header string rather than reading the two regexes; verified against the source +(`Mar22!F46` = 2022-02-17 = Python's serial 44609). Python recovers 44 real +dates R discards. New classifier `r_non_latin_header_miss` +(`PATIENT_NON_LATIN_HEADER_CLASSIFIERS`), registered for +`last_clinic_visit_date` and `last_remote_followup_date`. + +Incidental, not fixed: Python's `sanitize_str` docstring claims it "matches the +R implementation". It does not, and here the divergence favours Python. + +**4. The 2026 Annual sheet reaches five more columns (41 rows).** Ticket 37 +already verified that the 2026 template moved a block to a new `Annual` sheet +that R reads no values from; `r_extraction_gap` simply was not registered for +`blood_pressure_sys_mmhg`, `blood_pressure_dias_mmhg`, `edu_occ`, +`other_issues`, `status`. Verified per column against the source rather than +wired on shape (ticket 31's refusal): `2026 ISDFI, Annual!E32/H32/I32/AD32` for +`PH_IS022` reads `"college graduate"`, `80`, `50`, `"for cataract surgery"` -- +exactly Python's four values. Confirmed the monthly sheet's patient header row +(`May26!162`) carries no blood-pressure or education column at all, so the +Annual sheet is the only source, and that both pipelines *do* read the Annual +sheet -- R's gap is column-level, not sheet-level. + +**Because.** Two of the four were the comparison tool mis-measuring, not the +pipeline diverging, and the map's precedent is unambiguous that a representation +artifact gets normalized rather than classified (tickets 20, 22, 43). The other +two are R limitations with a mechanism traced to R's own source, which is the +bar ticket 31 set when it refused to wire `r_extraction_gap` on shape alone. + +**Rejected.** +- *Classifying the whitespace and boolean shapes instead of normalizing.* A + classifier would have recorded 60 rows as "understood" while leaving the tool + mis-measuring every future run, and would have hidden the `dm_complications` + `\r\n` from every later round exactly as it had been hidden for five. +- *Wiring `r_extraction_gap` across all ~125 R-null rows at once.* Shape- + identical, and it would have closed the ticket in one line -- but ticket 49 + found the R-null rows have **at least three** distinct mechanisms (Mukdahan's + header, 2026's Annual sheet, and a still-unexplained `2019_Preah Kossamak` + which has no non-Latin header divergence at all). A single classifier would + have cemented all three as one understood cause. +- *Extending `normalize_whitespace_column` to interior whitespace.* Ticket 46 + rejected this and nothing here reopens it; widening the column *scope* is a + different change from widening what it strips. +- *Chasing the remaining 84 this session.* Splitting is the map's own rule and + each remaining shape needs its own source-workbook check. + +**Evidence.** `executed` throughout. Both comparison runs were real: 229 -> 169 +after the two normalizations, 169 -> 84 after the two classifiers, against the +full 248-tracker drive data (`output/comparison/2026-08-18T134503Z` and +`2026-08-18T135258Z`). Source-Excel claims were read out of the real workbooks +on the drive; the R/Python sanitizer divergence was produced by running both +implementations, not by reading them. Full suite 752 passed, 1 skipped (+10 +new tests; the drive being mounted also un-skipped 65 drive-dependent tests +that had been skipped earlier in the session), ruff and `ty check src/` clean. The one judgement call is that +`2026 Surat Thani` and `2026 YGH`'s `edu_occ` rows share ISDFI's mechanism: +verified same template and same shape, mechanism traced directly on ISDFI only. + +**Tense.** Every count above is current behaviour, measured after the change. diff --git a/docs/wayfinder/tickets/50-triage-patient-raw-residual-6.md b/docs/wayfinder/tickets/50-triage-patient-raw-residual-6.md new file mode 100644 index 0000000..f9918cc --- /dev/null +++ b/docs/wayfinder/tickets/50-triage-patient-raw-residual-6.md @@ -0,0 +1,188 @@ +--- +id: 50 +title: Triage the residual patient raw-stage column mismatches (round 6) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-18b +claimed_at: 2026-08-18 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 49 +--- + +## Premise + +Rests on [Triage the residual patient raw-stage column mismatches (round +5)](49-triage-patient-raw-residual-5.md), closed, which took patient raw-stage +unclassified from **229 to 84** by settling three causes: the whitespace target +list being derived from the cleaned schema rather than the raw stage (comparison +fix, 40 rows), R's and Python's spellings of an Excel boolean (comparison fix, +20), R's Unicode-aware header sanitizer missing a header carrying a local- +language translation (`r_non_latin_header_miss`, 44), and the 2026 template's +Annual sheet reaching columns ticket 37's `r_extraction_gap` had not been +registered for (41). + +The residual is **84**, measured on baseline run +`output/comparison/2026-08-18T135258Z`, which is the run to triage against. + +Rests also on the map's standing scoping rule that the current tracker template +is the golden rule, and on the standing bar that a classifier records +understanding, never a verdict by itself. + +## Question + +Triage the 84. Re-measure against the baseline run before working any shape -- +ticket 49 found that two of its predecessor's stated shapes had already moved. + +- **`fbg_updated_date` (25, one file)**: `2019_Preah Kossamak`, all R-null with + Python holding a value. Shape-identical to `r_extraction_gap`, but ticket 31's + refusal applies and ticket 49 honoured it: verify the mechanism against the + source workbook before wiring, since ticket 49 found this file has *no* + non-Latin header divergence, so it is not the Mukdahan cause. +- **`t1d_diagnosis_date` (23, two files)**: R 41838 vs Python 41821 in + `2017_Yangon` -- two different real dates 16 days apart, so neither side is + null and neither is obviously right. Unexplained, and carried unexplained + through rounds 5 and 6. +- **`t1d_diagnosis_age` (16, two files)**: a real date typed into a numeric + column, or the reverse. Ticket 49's predecessor recorded this as a source + defect -- if that holds, it is a finding for [ticket + 40](40-source-defect-findings-report.md) rather than a pipeline change. +- **`hospitalisation_date` (5) and `hospitalisation_cause` (3)**: both + `2022_Kantha Bopha`, both R-null. `hospitalisation_date`'s Python values + include `2958352`, the Excel serial for 9999-12-31 -- the date sentinel + appearing at the *raw* stage, which raw is not supposed to carry. Ticket 49 + did not establish whether the sentinel is in the source workbook or is Python + applying a cleaning-stage rule too early; that is the load-bearing question. +- **`clinic_visit` (5, one file)**: `2025_LWCH`, R-null against Python `Y`. + Distinct from the 10 boolean-spelling rows ticket 49 resolved in the same + column -- do not assume the boolean fix covers it. +- **`blood_pressure_mmhg` (3)**, **`complication_screening` (2)**, + **`testing_frequency` (1)**, **`observations` (1)**: long tail, neither side + null except where noted. `observations`'s single row is R's `... January ,NA` + against Python's `... January`, where `r_na_unite_padding` does not fire (the + trailing space before the comma is the likely reason). + +Split further rather than leaving this open-ended if it does not converge. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: explaining a +difference and naming a cause is not enough -- each needs an explicit verdict on +whether Python is right, with the evidence. Where Python is losing information +the source carried, fix the pipeline. Where the source workbook itself is wrong, +that is a legitimate conclusion and a finding for [ticket +40](40-source-defect-findings-report.md). A cause genuinely undecidable on the +available evidence is recorded as an open question, not closed with a label. + +## Resolution + +**Decision.** All 84 triaged, and the patient raw stage now has **zero** +unclassified mismatches. Six mechanisms, none of them a Python defect: + +1. **R never mapped the column at all** -- 38 rows, three separate header + defects, all `r_extraction_gap` (the docstring now records each). 2022 + Kantha Bopha's hospitalisation header opens with thirteen spaces, so R's + sanitizer emits `xcurrentmonthhospitalisationdkahypootherdropdown` -- the + same leading-space mechanism ticket 37 found on `edu_occ_updated`, and R's + own raw parquet carries the junk column to prove it (5 `hospitalisation_date` + + 3 `hospitalisation_cause`). 2019 Preah Kossamak's `Jul19`/`Aug19` sheets + have lost every header merge the other ten still have, so "Updated FBG" no + longer spans its "Date" sub-header; R is left with a bare `date` column and + reads **0** values for those two sheets against Python's 13 and 12 (25 + `fbg_updated_date`). 2025 LWCH leaves the clinic-visit column unheaded in + both header rows, so R drops it and ticket 30's `recover_blank_headers` + names it from the sibling sheets (5 `clinic_visit`). +2. **A date typed into a numeric column** -- 20 rows, wired to the existing + `openpyxl_date_typed_stray_cell` ticket 24 root-caused on the product arm. + R's readxl coerces the column to its majority numeric type and carries the + Excel serial; Python's openpyxl honours the cell's own date format. Each of + the five serials decodes to exactly Python's value (20668 -> 1956-08-01, + 42859 -> 2017-05-04, 22190 -> 1960-10-01, 25873 -> 1970-11-01, 44228 -> + 2021-02-01), and every source cell is genuinely date-formatted. A source + defect, reported to [ticket 40](40-source-defect-findings-report.md). +3. **R drops a whitespace-only rich-text run, and the harness then dated the + damage** -- 11 rows. 2017 Yangon Children's `Feb17!I66` is the rich-text + string `July` / ` ` / `2014` (read out of the workbook's own + `sharedStrings.xml`); readxl loses the middle run, so R's raw holds + `July2014` where Python's holds `July 2014`. Both are the same cell, but + `MONTH_NAME_PATTERN` required a word boundary after the month name -- which a + digit does not provide -- so the month-year branch never saw `July2014` and + dateutil filled the day **from today**, producing 2014-07-18 against Python's + 2014-07-01. The pattern now ends in a negative lookahead. This was a live + non-determinism: the same comparison run on a different day produced a + different number. +4. **A Buddhist-Era serial too large for the parser's ceiling** -- 12 rows. + 2022 Hat Yai `Patient List!G25` holds 2560-01-01 (serial 241062); + `parse_date_flexible` only treated a number below 100000 as an Excel serial, + so 241062 fell through to dateutil, which read the digits positionally as + 24/10/62. The ceiling is now `MAX_EXCEL_DATE_SERIAL = 256_000` (~year 2600), + deliberately below the numeric error sentinel 999999 and the largest observed + non-date value (1,141,523). Both sides now decode to the same date, so the + mismatch disappears rather than being labelled -- and so do **363** rows that + `buddhist_era_typo` had been absorbing (369 -> 6) because R's side had been + sentinelling an unparseable serial. +5. **R keeps only the first selection of a multi-select block** -- 2 rows, the + new `r_duplicate_header_selection_dropped`. 2021 NPH `Dec21!AD84`/`AE84` are + two "(Select)" sub-columns of one merged screening header; R's + `make.names(unique = TRUE)` suffixes the duplicate and its mapping keeps only + the first -- proven in R's own raw parquet, which parks "Foot Examination + (Nerves)" in `complicationscreeningselect1`. Python unites both. Fires only + where R's value is exactly Python's first selection. +6. **Two verified causes stacked in one cell** -- 1 row. + `r_na_unite_padding` compared the surviving parts untrimmed, so a source + value ending in a space (`... January ,NA` against `... January`) matched + neither it nor ticket 46's `python_trims_merged_subvalue`. Parts are now + compared trimmed, which cannot absorb an interior whitespace difference. + +**Because.** Every one of the six was checked against the source workbook or +against R's own output before a classifier was written, per ticket 31's refusal +to wire on shape alone -- and that refusal earned its keep twice. The +`fbg_updated_date` population is shape-identical to `r_extraction_gap` but its +mechanism (lost header merges) is unrelated to any previously recorded one, and +the ticket's own load-bearing question -- whether Python was stamping the date +sentinel at the raw stage -- turned out to have a third answer: Python's raw +holds the verbatim text `on stamlor 5mg`, and the `9999-09-09` in the report was +the **comparison harness's** own date normalization of it. No raw-stage +sentinel, no pipeline bug. + +**Rejected.** +- *Naming the `fbg_updated_date` population `r_extraction_gap` on its shape.* + Refused until the mechanism was traced; it turned out to be a fourth, + genuinely different mechanism, now documented rather than merged silently + into the other three. +- *Giving the Buddhist-Era serial its own classifier.* A classifier would have + recorded a difference that does not exist: both sides hold the same cell, and + only the parser's ceiling made them disagree. Fixing the parse removes 375 + rows from the report instead of labelling 12. +- *Leaving `MAX_EXCEL_DATE_SERIAL` alone and special-casing the harness.* The + harness reuses `parse_date_flexible` on purpose. Measured before changing it: + 95 values across all 254 R raw files sit above the old ceiling, 93 of them in + the 241k-245k BE band and two far outside it, so the new ceiling takes exactly + the BE band. Production output is unchanged -- Python's own raw never carries + these as text (openpyxl returns datetimes), and the cleaned stage's + future-date guard sentinels the decoded date exactly as it sentinelled the + unparseable one. +- *Fixing the source workbooks in the pipeline.* Six findings are workbook + defects (three header defects, two dates in numeric columns, one clinical + note in a date column, one BE year); all went to + [ticket 40](40-source-defect-findings-report.md) per the map's standing rule. +- *A round 7.* Nothing is left to split: the raw stage is at zero. + +**Evidence.** `executed` throughout. The 84 and all 13 column/file shapes were +re-measured off the stated baseline before any work, and every shape matched the +ticket. Source claims were read out of the real workbooks on the drive +(including `sharedStrings.xml` for the rich-text run) and cross-checked against +R's own raw parquet. Final comparison run +`output/comparison/2026-08-18T214952Z` against the full 254-tracker drive data: +patient raw total mismatches 26,557 -> 26,171, **unclassified 84 -> 0**; patient +cleaned unclassified 8,008 -> 7,967 and `r_extraction_gap` +40, from the same +wiring reaching the cleaned stage; product raw and cleaned unchanged (0 and 20 +unclassified). Per-cause deltas match the six mechanisms exactly: +`r_extraction_gap` +38, `openpyxl_date_typed_stray_cell` +20, +`r_duplicate_header_selection_dropped` +2, `r_na_unite_padding` +1, +`buddhist_era_typo` -363. Full suite 759 passed, 1 skipped (+6 new tests), ruff +clean. + +**Tense.** Every count is current behaviour, measured after the change. diff --git a/docs/wayfinder/tickets/51-triage-patient-cleaned-residual-4.md b/docs/wayfinder/tickets/51-triage-patient-cleaned-residual-4.md new file mode 100644 index 0000000..8dca28c --- /dev/null +++ b/docs/wayfinder/tickets/51-triage-patient-cleaned-residual-4.md @@ -0,0 +1,211 @@ +--- +id: 51 +title: Triage the residual patient cleaned-stage mismatches (round 4) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-19 +claimed_at: 2026-08-19T10:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 50 +--- + +## Premise + +Rests on [Triage the residual patient raw-stage column mismatches (round +6)](50-triage-patient-raw-residual-6.md), closed, which took the patient **raw** +stage to zero unclassified mismatches. Six rounds of raw-stage triage are +finished; the cleaned stage is what remains of the patient arm. + +Rests on the three cleaned-stage rounds already closed -- [round +1](29-triage-patient-cleaned-residual.md) (55,670 -> 16,698), [round +2](37-triage-patient-cleaned-residual-2.md) (16,698 -> 8,259) and [the date +family](38-triage-patient-cleaned-date-family.md) (8,259 -> 6,652) -- and on +their standing conclusions, in particular that `r_numeric_error_sentinel` was +exhaustively verified in round 1 and that the date-absence marker list is now +declared once and shared. + +Rests also on the map's standing scoping rule that the current tracker template +is the golden rule, and on the standing bar that a classifier records +understanding, never a verdict by itself. + +The residual is **7,967 across 129 files and 27 columns**, measured on run +`output/comparison/2026-08-18T214952Z`, which is the run to triage against. +Note that this is *higher* than round 3 left it: later rounds moved the number +in both directions as classifiers were re-scoped and the tracker set was +refreshed to 254 files, so re-measure rather than reasoning from the history. + +## Question + +Triage the 7,967, largest shapes first. Re-measure against the baseline run +before working any shape -- every round so far has found at least one stated +shape had already moved. + +- **`fbg_updated_mmol` (2,936)** is **not** this ticket's work: it is [ticket + 44](44-triage-cleaned-fbg-r-null-residual.md), still open, and the two must + not be triaged twice. Exclude it and confirm the exclusion by column before + starting. +- **The date family (3,278 across seven columns)**: `fbg_updated_date` (940), + `hba1c_updated_date` (904), `t1d_diagnosis_date` (661), `hospitalisation_date` + (638), `bmi_date` (135), `recruitment_date` (80), `last_clinic_visit_date` + (60), `lost_date` (18), `last_remote_followup_date` (9). Round 3 worked this + family and left it at 1,994 sentinel-stamped cells; whether this is the same + population re-grown by later changes or a different one is the first thing to + establish. +- **`t1d_diagnosis_age` (558)** and **`bmi` (22)**: both formula-derived in the + source, so `excel_formula_error` is the raw-stage cause; what survives + cleaning is a different question. +- **Blood pressure (491)**: `blood_pressure_dias_mmhg` (289), + `blood_pressure_sys_mmhg` (202). +- **The long tail (~470)**: `height` (114), `fbg_updated_mg` (113), the two + screening dates (132), `insulin_subtype` (67), `province` (57), and eleven + columns in single digits. `province` is worth taking early despite its size: + the map's **Not yet specified** section already suspects Python's + `sanitize_str` strips accents from province names (`Kratié` -> `krati`), and + 57 cleaned-stage province mismatches is exactly where that would show. + +Split further rather than leaving this open-ended if it does not converge -- +three rounds have already split rather than sprawled. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: explaining a +difference and naming a cause is not enough -- each needs an explicit verdict on +whether Python is right, with the evidence. Where Python is losing information +the source carried, fix the pipeline. Where the source workbook itself is wrong, +that is a legitimate conclusion and a finding for [ticket +40](40-source-defect-findings-report.md). A cause genuinely undecidable on the +available evidence is recorded as an open question, not closed with a label. + + +## Resolution + +**Decision.** Three of the four largest shapes in the cleaned-stage residual are +explained, decided and landed as named causes; the rest is split into [round +5](52-triage-patient-cleaned-residual-5.md). Measured on the drive's real +254-tracker set, the in-scope residual (excluding `fbg_updated_mmol`, which is +[ticket 44](44-triage-cleaned-fbg-r-null-residual.md)'s) goes **5,031 -> 2,538**, +and the whole cleaned-stage unclassified count **7,967 -> 5,474**. Final run: +`output/comparison/2026-08-18T222716Z`. + +### 1. `r_ymd_first_misparse` -- 1,714 cells, 11 files. Python is right. + +R's `parse_date_string` (`r-archive/R/script2_helper_dates.R`) calls +`lubridate::parse_date_time` with `orders = c("ymd", "dmy", "my")`. lubridate +takes the **first** order that parses, and a source string written `D.M.YY` +parses perfectly well as ymd -- `30.1.18` becomes year 30 -> 2030 -- so the +`dmy` order is never reached. R swaps the day and the year on every such cell. + +Verified at source, not inferred: 2018 Yangon Children's Hospital writes these +dates as text inside the value cell -- `Jan18!L92` is `10.3 (22.8.17)` and +`Jan18!N92` is `223(30.1.18)` for MM_YC001 -- day-first, matching Python's +2017-08-22 / 2018-01-30 against R's 2022-08-17 / 2030-01-18. + +Python is right on three independent grounds: the source strings above; R's +reading putting **1,236 of the 1,714** cells in the *future* relative to the +tracker's own year while Python's puts none there and lands 1,713 within three +years of it; and source days above 12 (`31.5.16` -> R's 2031-05-16) having no +well-formed year-first reading at all. + +Columns: `fbg_updated_date` 801, `hba1c_updated_date` 781, `t1d_diagnosis_date` +107, `hospitalisation_date` 23, `bmi_date` 2. Dominated by 2017/2018 Yangon +(1,467) and 2018 Vietnam National Children's Hospital (211). + +### 2. `python_rejects_beyond_tracker_year` -- 722 cells, 25 files. Python is right. + +Python's `_validate_dates` (`clean/patient.py`) replaces any date past +31 December of `tracker_year` with the 9999-09-09 sentinel and logs an +`invalid_value` error per patient. R has **no** equivalent guard -- grep over +`r-archive/R` finds no future-date or tracker-year bound on any date column -- +so R carries the impossible value through unchanged. + +The values are genuinely impossible, not merely surprising. 2022 Vietnam +National Children's Hospital writes **every** `Date of T1D Diagnosis` cell in +its Patient List as a 2023 date: `Patient List!G13` = 2023-07-16 for VN_VC001, +born 2013-03, recruited 2017-07. The workbook has patients diagnosed a year +after the tracker was filled in and five years after they were recruited for +having the disease. That is a source defect -- a finding for [ticket +40](40-source-defect-findings-report.md) -- and sentinelling it is the pipeline +recognizing an unusable value, the same verdict `buddhist_era_typo` reached. + +**A false claim was found and corrected while doing this**: `_validate_dates`'s +docstring said the guard "matches R pipeline behavior". It does not; it is a +deliberate divergence, and the docstring now says so and says why. + +### 3. `r_unicode_sanitizer_rejects_accent` -- 57 cells, 5 files. Python is right. + +The two `sanitize_str` implementations were meant to be the same function and +are not. R's (`r-archive/R/script2_sanitize_str.R`) strips `[^[:alnum:]]`, +which under ICU is Unicode-aware, so accented letters survive; Python's +(`clean/validators.py`) strips `[^a-z0-9]` and folds them away. Both sides +sanitize the value *and* the allowed list before matching, so the difference +only shows where a source spelling differs from its canonical form in the +accents alone. + +The entire cause is one Vietnamese province across five VNCH trackers: the +source writes `Thái Nguyễn` (tilde on the second e) where +`reference_data/provinces/allowed_provinces.yaml` line 148 has `Thái Nguyên`. +R sanitizes those to `tháinguyễn` vs `tháinguyên`, misses, and stamps the +`"Undefined"` character sentinel; Python sanitizes both to `thinguyn` and +recovers the canonical name. Python is right -- the province is real and the +accent is a typo -- and this is the value-side twin of ticket 49's +`r_non_latin_header_miss`. + +**This resolves the standing fog patch** about `sanitize_str` stripping accents +(`Kratié` -> `krati`). Measured, executed: the folding cannot silently merge two +different provinces, because `validate_allowed_values` *raises* on any two +allowed values that sanitize alike, and the 209-entry province list has zero +such collisions. The suspicion was also backwards -- the folding makes Python +more permissive than R, not less, and it recovers data R discards. + +**Because.** All three clear the map's two-bar standard: each has a mechanism +traced to the responsible line of R's or Python's own code, and each has an +explicit verdict on which pipeline is right, backed by the source workbook +rather than by the shape of the diff. + +**Rejected.** + +- *One `python_rejects_unusable_date` cause covering all 1,167 cells where + Python sentinels and R keeps a date.* Killed because it merges two different + mechanisms with two different verdicts. Joining those cells back to Python's + own raw parquet splits them cleanly: **634** hold a genuine Excel date the + beyond-tracker-year guard then rejected, **533** hold free text Python's + parser could not read at all (`9-Dce-20`, `25-Ma4-2025`, `4-Okt-2023`, + `26/102022`, `DKA - Feb-2020`, `admitted to Yangon General Hosp...`), and 33 + did not match a raw row. Lumping them would have been exactly the + "labelled, not decided" failure ticket 27 had to be reopened for. The + classifier that shipped is deliberately narrower and keys on the tracker year + in the row's own sheet name. +- *Teaching Python to recover the typo'd dates (`9-Dce-20` -> 9 Dec 2020).* + Killed by the map's standing scoping rule: where a divergence traces to a + human error in the source workbook, the fix is the workbook. R's guess here + is actively worse than refusing -- it reads `9-Dce-20` as **2020-09-01**, + dropping the misspelled month and promoting the day into the month slot. +- *Deciding the free-text cases (`DKA - Feb-2020`, `26 Jun (ceton urine high)`) + in this session.* Killed because that is precisely [ticket + 39](39-recover-dates-embedded-in-free-text.md)'s open question, and answering + it here would decide a HITL grilling ticket without the human. +- *Working all four named shapes shallowly rather than the date family deeply.* + Killed at the start of the session, with the user's agreement. + +**What this gives up.** 2,538 in-scope cells remain, and `t1d_diagnosis_age` +(558), blood pressure (491), `height` (114) and `fbg_updated_mg` (113) were not +touched at all. The 533 parse-failure cells stay unclassified on purpose, +because part of that population belongs to ticket 39. + +**Evidence.** `executed` throughout: `duckdb` over the exported +`cell_mismatches` sheet for every count; `openpyxl` reads of the real source +workbooks (2018 Yangon, 2022 VNCH) for the source strings; a `polars` join of +the sentinelled cells back to `patient_data_raw` for the 634/533/33 split; a +run of `load_canonical_provinces` + `sanitize_str` for the zero-collision +check; `rg` over `r-archive/R` for R's missing future-date guard; and a full +re-run of `scripts/compare_outputs.py` against the drive for the before/after. +Test suite: 776 passed, 1 skipped; ruff clean. + +**Tense.** Every count above is current behaviour of the code as committed in +this session, measured on run `2026-08-18T222716Z`. The 634/533 split is a +measurement of the *previous* run's population, made to justify scoping the new +classifier -- the shipped classifier's own count is 722, which is that 634 plus +the cells in columns the join could not reach. diff --git a/docs/wayfinder/tickets/52-triage-patient-cleaned-residual-5.md b/docs/wayfinder/tickets/52-triage-patient-cleaned-residual-5.md new file mode 100644 index 0000000..48c4d71 --- /dev/null +++ b/docs/wayfinder/tickets/52-triage-patient-cleaned-residual-5.md @@ -0,0 +1,215 @@ +--- +id: 52 +title: Triage the residual patient cleaned-stage mismatches (round 5) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-19b +claimed_at: 2026-08-19T14:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 51 +--- + +## Premise + +Rests on [round 4](51-triage-patient-cleaned-residual-4.md), closed, which took +the cleaned-stage in-scope residual from 5,031 to **2,538** by landing three +causes: `r_ymd_first_misparse` (1,714), `python_rejects_beyond_tracker_year` +(722) and `r_unicode_sanitizer_rejects_accent` (57). Read its resolution before +starting -- it decides three things this ticket must not re-open: + +- **R's date parsing is year-first and wrong**, so any further date divergence + should be checked against `r_ymd_first_misparse` before being called new. +- **Python's beyond-tracker-year guard is a deliberate divergence from R**, not + a parity gap, and `_validate_dates`'s docstring now says so. +- **Python's ASCII-folding `sanitize_str` is safe on allowed-value columns**, + because `validate_allowed_values` raises on any two allowed values that + sanitize alike. Do not re-argue the accent question. + +Rests on the four cleaned-stage rounds before it ([1](29-triage-patient-cleaned-residual.md), +[2](37-triage-patient-cleaned-residual-2.md), [the date +family](38-triage-patient-cleaned-date-family.md), [round +4](51-triage-patient-cleaned-residual-4.md)) and on the map's standing scoping +rule that the current tracker template is the golden rule, and on the standing +bar that a classifier records understanding, never a verdict by itself. + +Would be void, not merely rewritten, if the destination stopped requiring every +R/Python difference to be explained before R is retired. + +## Question + +Triage the **2,538** in-scope cells left on run +`output/comparison/2026-08-18T222716Z`. Re-measure before working any shape; +every round so far has found at least one stated shape had already moved. + +`fbg_updated_mmol` (2,936) is **not** this ticket's work -- it is [ticket +44](44-triage-cleaned-fbg-r-null-residual.md). Exclude it and confirm the +exclusion by column before starting. + +- **`t1d_diagnosis_age` (558)** and **`height` (114)**, **`bmi` (22)** -- + untouched by round 4. `t1d_diagnosis_age` is formula-derived in the source, + so `excel_formula_error` is the raw-stage cause; what survives cleaning is a + different question. Largest single shape in scope: take it first. +- **The parse-failure date cells (~533 measured in round 4, spread across + `hospitalisation_date` 546, `hba1c_updated_date` 108, `fbg_updated_date` 103, + `bmi_date` 94, `recruitment_date` 72, `last_clinic_visit_date` 41, and a + small tail).** Round 4 measured this population by joining the sentinelled + cells back to `patient_data_raw`: Python stamps the date sentinel on a cell + it could not parse at all, and R guesses -- badly, reading `9-Dce-20` as + 2020-09-01. It splits into two kinds, and **only the first is takeable here**: + - *A mangled date token and nothing else* (`9-Dce-20` 81 cells, + `25-Ma4-2025` 18, `4-Okt-2023` 3, `26/102022` 5, `19-Jan_2023` 5). Round 4's + verdict, if it holds on re-measurement, is that Python is right to refuse + and the workbook is what needs correcting -- a [ticket + 40](40-source-defect-findings-report.md) finding. + - *A date buried in a clinical note* (`DKA - Feb-2020` 20, + `26 Jun (ceton urine high)` 30, `DKA; Jul 2018` 12, `admitted to Yangon + General Hosp...`). This is [ticket + 39](39-recover-dates-embedded-in-free-text.md)'s open HITL question. Do not + decide it here; measure it and leave it. +- **Blood pressure (491)**: `blood_pressure_dias_mmhg` (289), + `blood_pressure_sys_mmhg` (202). Note these columns already carry + `r_extraction_gap` for the 2026 template's Annual sheet, so the residual is + something else. +- **The long tail (~370)**: `fbg_updated_mg` (113), + `complication_screening_lipid_profile_cholesterol_value` (72), + `insulin_subtype` (67), `complication_screening_kidney_test_value` (60), + `lost_date` (18), and nine columns in single digits. + +Split further rather than leaving this open-ended if it does not converge -- +four rounds have already split rather than sprawled. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: explaining a +difference and naming a cause is not enough -- each needs an explicit verdict on +whether Python is right, with the evidence. Where Python is losing information +the source carried, fix the pipeline. Where the source workbook itself is wrong, +that is a legitimate conclusion and a finding for [ticket +40](40-source-defect-findings-report.md). A cause genuinely undecidable on the +available evidence is recorded as an open question, not closed with a label. + + +## Resolution + +**Decision.** The largest in-scope shape, `t1d_diagnosis_age`, is explained and +decided, and it turned out to be two real Python bugs rather than a labelling +job. The in-scope cleaned residual goes **2,538 -> 2,278**; the rest is split +into [round 6](53-triage-patient-cleaned-residual-6.md). Final run: +`output/comparison/2026-08-19T174047Z`. + +### 1. A bare four-digit year in a date cell became a 1905 date. Python was wrong; fixed. + +Nine trackers type a year alone where the template's own subheading asks for +`(dd-mmm-yyyy)`. `parse_date_flexible` read `2011` as an Excel serial, which +lands on 1905-07-03, and the damage showed downstream: `t1d_diagnosis_age` +came out **-95, -97, -101** (1905 minus the birth year) on 239 cells, and +`age` hit the 999999 sentinel for every patient with a bare birth year. + +Two code paths had to change, because the misreading happens at two different +places depending on one cell format: + +- `BARE_YEAR_MIN`/`BARE_YEAR_MAX` (`clean/date_parser.py`): a whole number in + 1900-2100 is now read as that year, resolved to 1 January. As a serial the + same number means 1905, which no tracker records anything from, so the two + readings cannot collide. +- `_IMPOSSIBLE_DATE_BEFORE` raised 1903 -> 1906 (`extract/patient.py`), + extending ticket 46's rule. Where the cell is *date*-formatted, openpyxl + resolves the number before cleaning ever sees it; those serials land in + March-September 1905 and so escaped the old bound. 2019 Yangon Children's + is the only file in the set that formats the column this way -- its 2017, + 2018 and 2020 siblings leave it General -- which is why this half stayed + invisible until the parser fix made the two sides disagree. + +**Python is right, and the source says so rather than the shape of the diff.** +Sarawak General Hospital's **2024** workbook writes the same patients' +diagnoses as real 1-January dates (`Patient List!G10` is 2011-01-01 for +MY_SW001, read with openpyxl) where its 2025 and 2026 workbooks write the bare +year, so the first of January is the clinic's own convention for a year with +no day, and the recovered ages agree across the three years (MY_SW001: 10 in +both; MY_SW003: 1 in both). R has every such patient born or diagnosed in +1905. + +Scale: 590 `dob` cells across four Yangon Children's trackers and 425 +`t1d_diagnosis_date` cells across Sarawak, Uni Med Center, VNCH, Heart of +Jesus and MMMHMC. After the fix, the whole cleaned output holds **zero** +birth dates before 1930, down from 590. + +### 2. `t1d_diagnosis_age` went negative where the source contradicts itself. Python was wrong; fixed. + +2023 Likas records MY_LW004 as born 2016-08-17 and diagnosed 2015-05-29 -- +diagnosed a year before birth -- and the workbook's *own* age formula resolves +to `#NUM!`. `_fix_t1d_diagnosis_age` (`clean/patient.py`) derived anyway and +emitted -2. It now returns null when the derivation goes negative: the two +dates contradict each other, so the arithmetic has nothing to say about the +row. 25 cells across 2023 Likas (12), 2023 Yangon General (10) and 2024 +Putrajaya (3). The source workbooks are what need correcting -- a [ticket +40](40-source-defect-findings-report.md) finding. The whole cleaned output now +holds zero negative diagnosis ages. + +### 3. Two classifiers, both keyed on mechanism rather than shape. + +- `python_reads_bare_year` (1,166 cells) requires R's date to be *exactly* the + Excel serial of the year Python read, which no coincidentally-1-January date + can satisfy. +- `python_age_from_bare_year` (992 cells) covers `age` and + `t1d_diagnosis_age` derived from such a date. It needed a new row-level + flag, `row_has_bare_year_date`, set by `compare_cells` across the whole row + -- the same opt-in-diagnostic pattern as `row_order_candidate` (ticket 21). + The derived values' shape alone (R at the 999999 sentinel, Python plausible) + cannot be told apart from a genuine age disagreement, and 590 of 597 `age` + mismatches sit on a row that itself carries a bare-year date, measured + rather than assumed. + +**Because.** Both fixes clear the map's two-bar standard: each has a mechanism +traced to the responsible line of Python's own code, and each has a verdict +backed by the source workbook -- the cross-year Sarawak comparison for the +bare year, Excel's own `#NUM!` for the negative ages. + +**Rejected.** + +- *Treating the bare year as a source defect to report rather than recover* + (the map's standing "fix the workbook, not the pipeline" rule). Killed by + the cross-year evidence: the same clinic writes 2011-01-01 explicitly in one + year and `2011` in the next, so recovering the year reproduces what the + clinic itself wrote rather than guessing at it -- and the parser already + accepts reduced precision one notch finer (`Jun 2006` -> 2006-06-01, ticket + 37). Refusing would have left 590 birth years and 425 diagnosis dates as + 1905 in production BigQuery, which is not a neutral outcome. +- *Fixing only `parse_date_flexible`.* Killed on measurement: it left 2019 + Yangon Children's unfixed and produced a **163-cell regression at the raw + stage** where six earlier rounds had reached zero. That regression was the + signal that found the extraction half; it is resolved, and the raw stage is + back to 0 unclassified. +- *Sentinelling the negative ages with 999999 rather than nulling them.* Would + match R, but 999999 claims "a value was recorded and is invalid"; nothing + was recorded here, and ticket 29 already settled that null is the right + claim for that case. +- *Adding a `t1d_diagnosis_age` range bound to `validation_rules.yaml`.* + Killed as the wrong layer: the bound would fire on a value the derivation + should never have produced, and `patient.py`'s ranges are a golden file the + YAML only mirrors. +- *Working the other named shapes shallowly.* `hospitalisation_date` (546), + blood pressure (491), `height` (114) and `fbg_updated_mg` (113) are + untouched, deliberately -- see round 6. + +**What this gives up.** 2,278 in-scope cells remain and most of the ticket's +named shapes were not reached. The parse-failure date cells stay unclassified +on purpose, because part of that population belongs to [ticket +39](39-recover-dates-embedded-in-free-text.md). + +**Evidence.** `executed` throughout: `duckdb` over the exported +`cell_mismatches` sheets for every count; `openpyxl` reads of the real source +workbooks (2024/2025 Sarawak, 2019 Yangon, 2023 Likas) for the cell values and +number formats; `polars` joins of the flagged cells back to +`patient_data_raw`; a row-level join proving 590/597 `age` mismatches sit on a +bare-year row; and two full re-runs of the patient pipeline plus +`scripts/compare_outputs.py` against the drive's 254-tracker set for the +before/after. Test suite: 785 passed, 1 skipped; ruff and `ty check src/` +clean. + +**Tense.** Every count is current behaviour of the code as committed in this +session, measured on run `2026-08-19T174047Z`. The 2,538 starting figure is +round 4's closing measurement, re-verified at the start of this session. diff --git a/docs/wayfinder/tickets/53-triage-patient-cleaned-residual-6.md b/docs/wayfinder/tickets/53-triage-patient-cleaned-residual-6.md new file mode 100644 index 0000000..efa375d --- /dev/null +++ b/docs/wayfinder/tickets/53-triage-patient-cleaned-residual-6.md @@ -0,0 +1,279 @@ +--- +id: 53 +title: Triage the residual patient cleaned-stage mismatches (round 6) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-19c +claimed_at: 2026-08-19T18:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 52 +--- + +## Premise + +Rests on [round 5](52-triage-patient-cleaned-residual-5.md), closed, which took +the cleaned-stage in-scope residual from 2,538 to **2,278** by settling +`t1d_diagnosis_age` -- and found two real Python bugs doing it, not a labelling +job. Read its resolution before starting; it decides three things this ticket +must not re-open: + +- **A bare four-digit year in a date cell is recovered as 1 January of that + year**, in `parse_date_flexible` *and* in `read_patient_rows` (the second is + needed only where the cell is date-formatted). The Sarawak cross-year + evidence settles that Python is right. Do not re-argue it as a source defect. +- **A derived `t1d_diagnosis_age` that goes negative is nulled**, because the + source's two dates contradict each other. The workbooks are the thing to + correct -- a [ticket 40](40-source-defect-findings-report.md) finding. +- **`row_has_bare_year_date`** now exists on `CellMismatch`, set across the + whole row by `compare_cells`. It is the precedent for keying a *derived* + column's cause on the mechanism rather than on the derived values' shape. + +Rests on the five cleaned-stage rounds before it +([1](29-triage-patient-cleaned-residual.md), +[2](37-triage-patient-cleaned-residual-2.md), [the date +family](38-triage-patient-cleaned-date-family.md), [round +4](51-triage-patient-cleaned-residual-4.md), [round +5](52-triage-patient-cleaned-residual-5.md)), on the map's standing scoping +rule that the current tracker template is the golden rule, and on the standing +bar that a classifier records understanding, never a verdict by itself. + +Would be void, not merely rewritten, if the destination stopped requiring every +R/Python difference to be explained before R is retired. + +## Question + +Triage the **2,278** in-scope cells left on run +`output/comparison/2026-08-19T174047Z`. Re-measure before working any shape -- +every round so far has found at least one stated shape had already moved, and +round 5 moved two of them in the *upward* direction by making Python more +correct. + +`fbg_updated_mmol` (2,936) is **not** this ticket's work -- it is [ticket +44](44-triage-cleaned-fbg-r-null-residual.md). Exclude it and confirm the +exclusion by column before starting. + +- **`hospitalisation_date` (546)** -- now the largest single shape in scope. + Round 4 measured this population by joining the sentinelled cells back to + `patient_data_raw`: Python stamps the date sentinel on a cell it could not + parse at all, and R guesses -- badly, reading `9-Dce-20` as 2020-09-01. It + splits into two kinds, and **only the first is takeable here**: + - *A mangled date token and nothing else* (`9-Dce-20`, `25-Ma4-2025`, + `4-Okt-2023`, `26/102022`, `19-Jan_2023`). Round 4's verdict, if it holds + on re-measurement, is that Python is right to refuse and the workbook is + what needs correcting -- a [ticket 40](40-source-defect-findings-report.md) + finding. + - *A date buried in a clinical note* (`DKA - Feb-2020`, `26 Jun (ceton urine + high)`, `DKA; Jul 2018`). This is [ticket + 39](39-recover-dates-embedded-in-free-text.md)'s open HITL question. Do not + decide it here; measure it and leave it. + The same split governs the smaller date columns: `hba1c_updated_date` (108), + `fbg_updated_date` (103), `bmi_date` (94), `recruitment_date` (72), + `last_clinic_visit_date` (41), `lost_date` (18). +- **Blood pressure (491)**: `blood_pressure_dias_mmhg` (289), + `blood_pressure_sys_mmhg` (202). These columns already carry + `r_extraction_gap` for the 2026 template's Annual sheet, so the residual is + something else. +- **`t1d_diagnosis_age` (298 left)** -- the share round 5 did *not* explain. + 590 of 597 `age` mismatches proved to sit on a bare-year row, but only 239 of + 537 diagnosis-age ones did, so the remainder is a different mechanism. + Round 5 found two of its shapes without deciding them: R stamps the 999999 + sentinel where the source records an age with a unit (`11yr`, `4mth`, 2017 + Mandalay) that Python parses, and R carries a raw Excel serial (20668) into + the age column in 2023 Chiang Mai where Python has null. +- **`height` (114)**, **`fbg_updated_mg` (113)**, **`bmi` (22)** -- untouched + by rounds 4 and 5. `bmi` is formula-derived in the source, so + `excel_formula_error` is the raw-stage cause; what survives cleaning is a + different question. +- **The long tail (~300)**: + `complication_screening_lipid_profile_cholesterol_value` (72), + `insulin_subtype` (67), `complication_screening_kidney_test_value` (60), and + the columns in single digits -- including the four + `clinic_visit`/`remote_followup` rows the map's **Not yet specified** section + has been carrying since ticket 49, which belong with whatever round works the + tail. + +Split further rather than leaving this open-ended if it does not converge -- +five rounds have already split rather than sprawled. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: explaining a +difference and naming a cause is not enough -- each needs an explicit verdict on +whether Python is right, with the evidence. Where Python is losing information +the source carried, fix the pipeline. Where the source workbook itself is wrong, +that is a legitimate conclusion and a finding for [ticket +40](40-source-defect-findings-report.md). A cause genuinely undecidable on the +available evidence is recorded as an open question, not closed with a label. + +## Resolution + +**Decision.** Two of the three shapes this round set out to explain turned out +to be **Python defects, both fixed**; the third -- `hospitalisation_date`, the +largest single column in scope -- turned out to be **entirely** [ticket +39](39-recover-dates-embedded-in-free-text.md)'s open question and is handed +there rather than triaged here. The in-scope residual went **2,278 -> 1,689** +(26%) and the patient raw stage held at 0 unclassified. What is left became +[round 7](54-triage-patient-cleaned-residual-7.md). + +### 1. dateutil completed an absent day from today (fixed) + +`parse_date_flexible` (`src/a4d/clean/date_parser.py`) has an alphabetic +month-year branch precisely so a month-and-year cell resolves to the first of +the month rather than being handed to `dateutil.parser.parse`, which fills any +absent field from `datetime.now()`. The branch's pattern only ever matched an +alphabetic month with a hyphen or space, so three spellings still reached +dateutil: the **numeric** form (`10/2019`), the **comma-separated** form +(`Mar, 2017`, `August,2015`, `June ,2016`), and the **apostrophe** form +(`Jun'09`, `Apr'21`). Ticket 50 hit this same fill through a rich-text-damaged +`July2014` and closed that one route into it. + +Verified by execution, not by reading: `parse_date_flexible("10/2019")` +returned **2019-10-19** on 19 August, and `"06/2020"` returned 2020-06-19 -- +the day is the run date's day of month. 42 raw cells across five date columns +(`hospitalisation_date` 24, `fbg_updated_date` 9, `hba1c_updated_date` 4, +`last_remote_followup_date` 3, `bmi_date` 2) carry a source in this shape, so +that many production values changed meaning from one run to the next with no +input change. + +The fix is deliberately **not** another spelling added to the pattern. +`_parse_with_dateutil` parses the string twice against two disjoint defaults +(1111-01-01 and 2222-02-02); a component that differs between the two parses is +one dateutil invented rather than read. An invented **day** resolves to the 1st +-- the same convention the alphabetic branch already uses -- and an invented +**month or year** makes the cell unparseable, because neither is recoverable +from the string and today's date is not an answer. Two supporting corrections +came out of running it over all 5,185 distinct raw date strings: the +apostrophe was admitted to the month-year separator class (`May'21` had been +read as the 21st of May *this* year, now 2021-05-01), and +`_parse_longest_parseable_prefix` now skips a bare number below the bare-year +floor (truncating `26 Jun (ceton urine high)` down to `26` had made it Excel +serial 26, i.e. 1900-01-25). + +Of the 5,185 distinct sources, **36 parse differently**; every one was checked +by hand. 23 move from a today-filled day to the 1st of the correct month; the +rest move from a confidently wrong date to the error sentinel (`Sep` was +2026-09-19, `3 month come back meet Doctor` was 1900-01-02, three Thai-script +dates were 1900-01-0x). None moves from a right answer to a wrong one. + +**Verdict: Python was wrong and is now right.** R is not the reference here -- +R sentinels most of these -- but a value that depends on the calendar date of +the run is indefensible on its own terms. + +### 2. A blood pressure with a space beside the slash was discarded (fixed) + +All 491 in-scope `blood_pressure_sys_mmhg` / `blood_pressure_dias_mmhg` cells +had the identical shape: R holds a plausible reading (100, 60, 70, 40) and +Python holds the 999999 numeric error sentinel. Joining them back to Python's +own raw stage showed **every single affected source value has whitespace +around the slash** -- `70 / 40`, `103/ 69`, `116 / 66`, `105/ 62` -- and +nothing else wrong with it. + +`split_bp_in_sys_and_dias` (`src/a4d/clean/transformers.py`) splits on `/` and +leaves the padding on each fragment. R's `separate_wider_delim` +(`script2_helper_patient_data_fix.R:645`) leaves it on too, but R's +`as.numeric` ignores surrounding whitespace where Polars' cast fails on it -- +so the identical intermediate value survives in R and becomes the error +sentinel in Python. The fragments are now trimmed. + +Measured end-to-end against the real 248-tracker drive data: +`blood_pressure_dias_mmhg` **302 -> 13** total mismatches, +`blood_pressure_sys_mmhg` **215 -> 13**; both columns leave the unclassified +residual entirely. 465 cells across 7 trackers (2019/2020/2021 Mandalay, +2019/2020/2021 CDA, 2021 Sunprasitthiprasong) had a recorded blood pressure +restored. + +**Verdict: Python was losing information the source carried, and R was right.** +This is the map's ticket-27 precedent again -- a classifier here would have +cemented a data-loss bug as "explained". + +### 3. `hospitalisation_date` is ticket 39's population, not this round's + +The ticket expected this column to split into *a mangled date token and +nothing else* (takeable here) and *a date buried in a clinical note* (ticket +39's). **On re-measurement that split does not exist for this column.** All +546 cells were joined back to the raw stage and grouped by source value; every +distinct source is a clinical note. The three directions are: + +- **Python sentinels, R has a date (302)**: `DKA - Feb-2020`, `DKA; Jul 2018`, + `Passed away 28/10/2019 due to DKA`, `DKA 2020: June, Aug, Nov`. +- **R sentinels, Python has a date (179)**: `April 19 (due to very high + Hba1c)`, `19th-29th Aug 2019`, `7/2020 DKA Admit Ratchaburi Hospital 5 + Days`. Python already recovers dates from free text in these; R does not. +- **Both have a date and they differ (65)**: `May 2019, Dec 2019 DKA, August + 2020 DKA` -- the cell lists *several* admissions and the two pipelines pick + different ones. + +That third direction is the substantive new finding: the question is not only +"recover or discard" but **which** date, when the cell records more than one. +It is recorded on ticket 39 rather than decided here. + +Deliberately **not** classified. Naming a cause would move 489 cells out of +`unclassified` while the decision that governs them is still open, which is +exactly what the map's standing bar forbids. They stay visible. + +### Because + +The two fixes were found the same way the last four rounds found theirs: join +the flagged cells back to Python's own raw stage and group by the *source* +value, rather than by the shape of the two outputs. Both populations were +100% homogeneous at the source once looked at that way -- every BP cell had a +space by the slash, every hospitalisation cell was prose -- which is what +distinguishes a mechanism from a coincidence. + +### Rejected + +- **Adding `10/2019` (and then `Mar, 2017`, and then `Jun'09`) to the + month-year pattern.** This is what ticket 50 did for `July2014`, and this + round is the second time the same fill has resurfaced through a spelling + nobody enumerated. Probing dateutil's defaults closes the class instead of + the instance. What it gives up: three sources that previously produced a + confidently wrong date now produce the sentinel (`26-05- 2007`, a real date + broken by a stray space, is the one genuine loss) -- accepted, because the + old answers were wrong, not merely imprecise. +- **Normalizing whitespace around date separators** so `26-05- 2007` parses. + Correct-looking and untested across the other 5,184 sources; a third + normalization in one session is how a triage round turns into a regression. + Left as a [ticket 40](40-source-defect-findings-report.md) source-defect + finding. +- **Trimming inside `safe_convert_column` instead of in the BP splitter.** + Would have silenced the symptom for every numeric column at once and hidden + that the splitter is what introduced the padding. It also risks + re-classifying cells ticket 29 verified exhaustively under + `r_numeric_error_sentinel`. +- **Classifying `hospitalisation_date` as `date_inside_clinical_note`.** See + above -- a label there would read as "understood and settled" when the + decision is open. +- **Working `t1d_diagnosis_age` (298).** Measured, not decided: it is three + shapes -- R null where Python derives an age from a `dob`/`t1d_diagnosis_date` + pair R failed to parse (dominant, mostly Sarawak's bare-year dates from + ticket 52); R's 999999 where the source records an age in words (`4 months`, + `At birth`, `4mth`) and Python derives 0 from the dates; and 2023 Chiang + Mai, where a *date* (`1956-08-01`) is typed into the age column and R carries + Excel serial 20668 into it while Python nulls it. All three look decidable, + none is decided here -- the session had already changed the pipeline twice + and re-run it twice. Carried to round 7 with these measurements. + +### Evidence + +**Executed:** every count in this resolution. The parser probes were run +against the installed module; the 36-source diff was produced by running the +old and new parser over all 5,185 distinct raw date strings; the before/after +mismatch counts come from two full pipeline runs against the real 248-tracker +set on the USB drive and the comparison tool's own run-over-run delta +(baseline `2026-08-19T174047Z`, intermediate `2026-08-19T184342Z`, final +`2026-08-19T185253Z`). Full suite 788 passed / 1 skipped, ruff and +`ty check src/` clean. Pushed as `f16101e`. + +**Read:** R's `split_bp_in_sys_and_dias` +(`r-archive/R/script2_helper_patient_data_fix.R:623-653`) was read, not run -- +the claim that R's `as.numeric` tolerates surrounding whitespace is R language +behaviour taken on knowledge, and the *observable* fact executed instead is +that R's frozen output holds the value where Python's held the sentinel. That +is the load-bearing half, so no assumption is recorded for it. + +**Tense:** every count describes current behaviour after the fixes, except the +"was 2019-10-19" and "302 -> 13" readings, which describe behaviour before +them. diff --git a/docs/wayfinder/tickets/54-triage-patient-cleaned-residual-7.md b/docs/wayfinder/tickets/54-triage-patient-cleaned-residual-7.md new file mode 100644 index 0000000..3c57ee2 --- /dev/null +++ b/docs/wayfinder/tickets/54-triage-patient-cleaned-residual-7.md @@ -0,0 +1,213 @@ +--- +id: 54 +title: Triage the residual patient cleaned-stage mismatches (round 7) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-19d +claimed_at: 2026-08-19T21:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 53 +--- + +## Premise + +Rests on [round 6](53-triage-patient-cleaned-residual-6.md), closed, which took +the cleaned-stage in-scope residual from 2,278 to **1,689** by fixing two Python +defects rather than labelling them. Read its resolution before starting; it +settles three things this ticket must not re-open: + +- **`parse_date_flexible` no longer lets dateutil invent a component from + today.** An absent day resolves to the 1st; an absent month or year makes the + cell unparseable. Do not re-add a month-year spelling to the pattern -- the + probe-the-defaults mechanism closes the class. +- **`split_bp_in_sys_and_dias` trims each fragment**, so a source written + `70 / 40` keeps its reading. Blood pressure has left the residual; do not + expect it back. +- **`hospitalisation_date` (489) is entirely [ticket + 39](39-recover-dates-embedded-in-free-text.md)'s**, in all three of its + directions, including the newly measured one where *both* pipelines find a + date and disagree about which of several admissions the cell means. It is + deliberately left unclassified. Do not triage it here and do not label it. + +Rests on the six cleaned-stage rounds before it +([1](29-triage-patient-cleaned-residual.md), +[2](37-triage-patient-cleaned-residual-2.md), [the date +family](38-triage-patient-cleaned-date-family.md), [round +4](51-triage-patient-cleaned-residual-4.md), [round +5](52-triage-patient-cleaned-residual-5.md), [round +6](53-triage-patient-cleaned-residual-6.md)), on the map's standing scoping rule +that the current tracker template is the golden rule, and on the standing bar +that a classifier records understanding, never a verdict by itself. + +Would be void, not merely rewritten, if the destination stopped requiring every +R/Python difference to be explained before R is retired. + +## Question + +Triage the **1,689** in-scope cells left on run +`output/comparison/2026-08-19T185253Z`, excluding `fbg_updated_mmol` ([ticket +44](44-triage-cleaned-fbg-r-null-residual.md)) and `hospitalisation_date` +(ticket 39). Re-measure before working any shape -- every round so far has found +at least one stated shape had already moved. + +The method that found both of round 6's defects, and round 5's: **join the +flagged cells back to Python's own raw stage and group by the *source* value**, +not by the shape of the two outputs. A population that is homogeneous at the +source is a mechanism; one that is only homogeneous in the report may be a +coincidence. + +- **`t1d_diagnosis_age` (298)** -- now the largest takeable shape, measured by + round 6 but deliberately not decided. Three sub-shapes, all of which look + decidable: + - *R null, Python derives an age* (dominant; 2024/2025/2026 Sarawak, 2021 + Children's Hospital 2). Both `dob` and `t1d_diagnosis_date` are present on + Python's side and R failed to parse one of them -- for Sarawak these are the + bare-year dates [round 5](52-triage-patient-cleaned-residual-5.md) + recovered, so this is likely a downstream face of `python_reads_bare_year` + that `python_age_from_bare_year` does not catch, because extraction now + resolves the bare year before the classifier can see it. + - *R 999999, Python 0*: the source records an age in words -- `4 months`, + `4mth`, `At birth` (2017/2018 Mandalay, 2017/2018 Yangon) -- which R cannot + parse, while Python derives 0 from the two dates. + - *R 20668, Python null*: 2023 Chiang Mai has a **date** (`1956-08-01`) typed + into the age column; R carries the Excel serial into the age, Python nulls + it. A [ticket 40](40-source-defect-findings-report.md) source-defect + finding if it holds. +- **The smaller date columns** -- `hba1c_updated_date` (108), + `fbg_updated_date` (101), `bmi_date` (88), `recruitment_date` (58), + `last_clinic_visit_date` (41), `lost_date` (18), `t1d_diagnosis_date` (12), + `dob` (7), `last_remote_followup_date` (3). Round 6 established that + `hospitalisation_date` is *all* clinical notes; whether the same holds for + these is unmeasured. Split each by direction (Python sentinels / R sentinels / + both real and differing) before deciding, and hand any clinical-note share to + ticket 39 rather than triaging it. +- **`height` (114)**, **`fbg_updated_mg` (113)**, **`bmi` (22)** -- untouched by + rounds 4, 5 and 6. `bmi` is formula-derived in the source, so + `excel_formula_error` is the raw-stage cause; what survives cleaning is a + different question. +- **The long tail (~200)**: + `complication_screening_lipid_profile_cholesterol_value` (72), + `insulin_subtype` (67), `complication_screening_kidney_test_value` (60), + `hba1c_updated` (8), `fbg_baseline_mg` (6), and the columns in single digits + -- including the four `clinic_visit`/`remote_followup` rows the map's **Not + yet specified** section has been carrying since ticket 49, which belong with + whatever round works the tail. + +Split further rather than leaving this open-ended if it does not converge -- six +rounds have already split rather than sprawled. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: explaining a +difference and naming a cause is not enough -- each needs an explicit verdict on +whether Python is right, with the evidence. Where Python is losing information +the source carried, fix the pipeline. Where the source workbook itself is wrong, +that is a legitimate conclusion and a finding for [ticket +40](40-source-defect-findings-report.md). A cause genuinely undecidable on the +available evidence is recorded as an open question, not closed with a label. + +## Resolution + +**Decision.** `t1d_diagnosis_age` (298) and the two string-typed screening +measurement columns (132) are fully explained and decided; the cleaned-stage +in-scope residual falls **1,200 -> 770** (36%), the raw stage holds at **0**, +and total cleaned-stage cell mismatches fall 114,712 -> 114,580. No pipeline +behaviour changed: both findings are R limitations or comparison-harness gaps, +not Python defects. Everything else measured this session is handed to [round +8](55-triage-patient-cleaned-residual-8.md) with its mechanism already named. + +A note on the count: this ticket's own premise said 1,689 in-scope. The +measured figure is 1,200 -- 4,625 unclassified less `fbg_updated_mmol` (2,936) +and `hospitalisation_date` (489). 1,689 is 1,200 plus the 489 this ticket +excludes from work; the frontier was the same either way. + +**`t1d_diagnosis_age` (298 -> 0), two causes, Python right in both.** + +`r_never_derives_diagnosis_age` (282). R's `fix_t1d_diagnosis_age` exists in +`script2_helper_patient_data_fix.R` and is unit-tested against exactly the +strings the trackers carry -- "At birth", "4 months", "5y", "10y10m" -- but its +**call site is commented out** (`script2_process_patient_data.R:251`, read +directly, not inferred). So R's `t1d_diagnosis_age` is only ever the source +column through `as.numeric`: a blank cell stays NA, a cell written in words +becomes R's 999999 "recorded but invalid" sentinel. Python's +`_fix_t1d_diagnosis_age` fills from `dob` and `t1d_diagnosis_date` in exactly +those two cases and otherwise keeps what the clinic recorded. + +Python is right on the source's own evidence, not on the shape of the diff. +Where the source wrote the age in words, Python's derived figure agrees with +the words: 2017 Mandalay's MM_MD010 reads `11yr` against a D.O.B. of +2004-03-01 and a diagnosis of 2016-01-01, and Python derives 11; MM_MD011 +reads `4mth` against 2013-10-23 and 2014-02-01, and Python derives 0. Where +the cell is blank, both pipelines hold the *same* two dates and only Python +uses them -- 2024 Sarawak's MY_SW001 is (2000-06-30, 2011-01-01) on both sides, +null in R and 10 in Python. R keeps nothing a clinic recorded; Python recovers +information R discards. + +`source_date_in_diagnosis_age` (16). A date typed into the `Age at Diagnosis` +column, confirmed in both source workbooks. 2023 Chiang Mai's Patient List row +for TH_CP005 holds 1956-08-01 (serial 20668) in a workbook whose `Date of T1D +Diagnosis` column is empty for **every** patient, so its formula-derived age +column reads `#NUM!` throughout; 2023 Yangon General's row for MM_YC043_YG +holds 2017-05-04 (serial 42859) against a D.O.B. of 2008-01-01 and a diagnosis +date of 2007-06-01 -- a diagnosis a year before the birth. R's readxl reads +the column as numeric and carries the raw serial into the age; Python's cast +fails and the cell is null. Python is right (20,668 is not an age), and both +are source defects for [ticket 40](40-source-defect-findings-report.md). + +Why these 298 were not already `python_age_from_bare_year` (ticket 52): that +classifier fires on `row_has_bare_year_date`, which requires the row's own date +cells to *disagree*. Sarawak's 2024 workbook writes real 1-January dates, so R +and Python agree on the dates and no bare-year mismatch exists on the row -- +the ages differ purely because only Python derives one. The ticket's stated +hypothesis (a downstream face of `python_reads_bare_year`) was therefore +wrong, and the real mechanism is more general. + +**The screening measurement columns (132 -> 0), a harness gap.** +`complication_screening_lipid_profile_cholesterol_value` (72) and +`complication_screening_kidney_test_value` (60) were entirely +`4.8600000000000003` against `4.86` -- the float-to-string rounding difference +`normalize_numeric_column` has handled since ticket 22, which was scoped to the +raw stage on the argument that cleaning casts its numeric columns. That +argument has an exception nobody had measured: the cleaned schema types these +columns as **String** (verified in both pipelines' own parquet schemas) because +the same cell can read "normal", so cleaning never casts them and the artifact +survives. Added `string_numeric_normalize_targets` (derived from each frame's +own dtypes, not a named list) and wired the `Patient (cleaned)` stage to it via +a new `ALL_STRING_COLUMNS` sentinel. The 132 cells stop being mismatches at +all rather than being labelled -- total mismatches fell by exactly 132, so +nothing already classified was masked. + +**Rejected.** +- *Labelling the 298 as a bare-year effect*, per the ticket's own hypothesis. + Killed by measurement: R's and Python's dates are identical on the dominant + Sarawak population, so there is no bare-year mismatch to be downstream of. +- *One classifier for all 298.* The 16 date-in-age cells have the opposite + direction (Python nulls, R carries) and a different verdict (the source is + broken, not R), so folding them in would have hidden a ticket 40 finding + behind an R-limitation label. +- *Naming the two screening columns explicitly.* Rejected per the map's + never-hand-maintain rule -- which columns the schema types as strings is the + schema's decision, and a copy of it drifts. +- *Chasing `height`, `fbg_updated_mg`, `bmi`, `insulin_subtype` here.* Each was + measured far enough to name its mechanism and two of them look like real + Python defects; that is more than one session and is round 8's work. + +**Evidence.** Executed. R's dead call site read directly from +`r-archive/R/script2_process_patient_data.R`; both source workbooks opened with +openpyxl and the offending cells printed; the mismatch population joined back +to Python's own raw stage per the ticket's prescribed method; verdicts checked +against the real 248-tracker `output_r`/`output_python` pair on the USB drive. +Two full comparison runs (`2026-08-19T195136Z`, `2026-08-19T195625Z`) measured +the before/after. Full suite 799 passed / 1 skipped, ruff and `ty check src/` +clean. + +**Tense.** Every count above is current behaviour of the code as committed, +measured on run `2026-08-19T195625Z`, not a consequence of a proposed design. + +**Also corrected in passing.** `clean/patient.py`'s step 5.5b comment still +read "Replaces any existing value (including Excel errors like #NUM!)", which +ticket 52 made false -- the function now prefers the tracker's own recorded +age. Same class of correction as round 4's `_validate_dates` docstring. diff --git a/docs/wayfinder/tickets/55-triage-patient-cleaned-residual-8.md b/docs/wayfinder/tickets/55-triage-patient-cleaned-residual-8.md new file mode 100644 index 0000000..54228cc --- /dev/null +++ b/docs/wayfinder/tickets/55-triage-patient-cleaned-residual-8.md @@ -0,0 +1,242 @@ +--- +id: 55 +title: Triage the residual patient cleaned-stage mismatches (round 8) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-19e +claimed_at: 2026-08-19T23:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 54 +--- + +## Premise + +Rests on [round 7](54-triage-patient-cleaned-residual-7.md), closed, which took +the cleaned-stage in-scope residual from 1,200 to **770** and settles two things +this ticket must not re-open: + +- **R never derives a diagnosis age at all** -- `fix_t1d_diagnosis_age`'s call + site is commented out (`script2_process_patient_data.R:251`). The whole + `t1d_diagnosis_age` column is decided; do not re-triage it. +- **The cleaned stage now numeric-normalizes its string-typed columns** + (`string_numeric_normalize_targets`, `ALL_STRING_COLUMNS`). A float-rounding + difference on a string-typed measurement column is no longer a mismatch. If + one reappears, the harness broke -- do not classify it. + +Rests on the seven cleaned-stage rounds before it +([1](29-triage-patient-cleaned-residual.md), +[2](37-triage-patient-cleaned-residual-2.md), [the date +family](38-triage-patient-cleaned-date-family.md), [round +4](51-triage-patient-cleaned-residual-4.md), [round +5](52-triage-patient-cleaned-residual-5.md), [round +6](53-triage-patient-cleaned-residual-6.md), [round +7](54-triage-patient-cleaned-residual-7.md)); on [ticket +42](42-fbg-unit-headers-and-implausible-values.md), which set the analytical +bounds on all four FBG columns and owns the medical-limits question; on the +map's rule that the current tracker template is the golden rule; and on the +standing bar that a classifier records understanding, never a verdict by +itself. + +Would be void, not merely rewritten, if the destination stopped requiring every +R/Python difference to be explained before R is retired. + +## Question + +Triage the **770** in-scope cells left on run +`output/comparison/2026-08-19T195625Z`, excluding `fbg_updated_mmol` ([ticket +44](44-triage-cleaned-fbg-r-null-residual.md)) and `hospitalisation_date` +([ticket 39](39-recover-dates-embedded-in-free-text.md)). Re-measure before +working any shape. + +Round 7 measured every shape below far enough to name a mechanism but did not +decide any of them. **Two look like real Python defects and should be taken +first** -- per the map's bar, where Python is losing or corrupting information +the pipeline gets fixed, not labelled. + +- **`height` (114) -- likely a real Python gap.** The source cells hold `6.9`, + `2.52`, `2.43`, `2.72`, `13.0` (2025/2026 Señor Sto. Niño, 2021-2023 Likas, + 2019 Mahosot). R sentinels every one of them (999999); Python divides by 100 + and emits `0.069`, `0.0252` metres into production output. Whatever the + source values mean, Python's are not heights. Find R's own height range + check, decide whether Python needs the equivalent guard, and settle whether + the source values are a unit the pipeline should recognize or a source defect + for [ticket 40](40-source-defect-findings-report.md). Check whether these + rows also feed `bmi` (see below) before deciding either. + +- **`fbg_updated_mg` (113) -- two opposite directions, one of them alarming.** + 85 cells where R produces a plain number and Python sentinels: the source + reads `Lost follow up` (41 cells, R produces **140**), `SMBG 50-HI`, + `129-HI`, `CBG 57-High`, bare `HI` (R produces **200**). R's `fix_fbg` is + manufacturing readings from text -- read it and decide whether 200-for-HI is + a documented clinical convention worth adopting or an invention, and whether + 140-from-"Lost follow up" is anything but garbage. The other direction (~28 + cells) is the reverse and looks like Python winning: the source reads + `148 mg/dl (Mar-18)` and Python extracts 148 where R sentinels. + +- **`bmi` (22).** R sentinels, Python computes 10-12. Formula-derived in the + source, so `excel_formula_error` is the raw-stage cause; what survives + cleaning is a different question. Decide together with `height` -- a BMI + computed from an implausible height is not a BMI. + +- **`insulin_subtype` (67).** R null, Python `Undefined` on every one -- 56 in + 2024 Sarawak, the rest in 2025/2026 Calamba Doctors and 2025 Pahol. The + source rows carry real brands (`Novorapid` with `Glargine`, `Toujeo`, + `Ryzodeg`), so Python's derivation runs and its allowed-values validator then + rejects the result. Decide whether the allowed-value config is missing + entries -- which would make this a real Python data loss -- or whether the + subtype genuinely is undefined for these combinations. Note both pipelines + lose here; R just loses silently. + +- **The smaller date columns** -- `hba1c_updated_date` (108), + `fbg_updated_date` (101), `bmi_date` (88), `recruitment_date` (58), + `last_clinic_visit_date` (41), `lost_date` (18), `t1d_diagnosis_date` (12), + `dob` (7), `last_remote_followup_date` (3). Still unmeasured. Round 6 + established that `hospitalisation_date` is *all* clinical notes; whether the + same holds here is unknown. Split each by direction (Python sentinels / R + sentinels / both real and differing) before deciding, and hand any + clinical-note share to ticket 39 rather than triaging it. + +- **The long tail (~30):** `hba1c_updated` (8), `fbg_baseline_mg` (6), + `remote_followup` (2), `testing_frequency` (1), `fbg_baseline_mmol` (1) and + the columns in single digits -- including the four + `clinic_visit`/`remote_followup` rows the map's **Not yet specified** section + has carried since ticket 49, which belong with whatever round works the tail. + +Split further rather than leaving this open-ended if it does not converge -- +seven rounds have already split rather than sprawled. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: explaining a +difference and naming a cause is not enough -- each needs an explicit verdict on +whether Python is right, with the evidence. Where Python is losing information +the source carried, fix the pipeline. Where the source workbook itself is wrong, +that is a legitimate conclusion and a finding for [ticket +40](40-source-defect-findings-report.md). A cause genuinely undecidable on the +available evidence is recorded as an open question, not closed with a label. + +## Resolution + +**Decision.** The in-scope cleaned-stage residual went **770 -> 448** (42%) and +the raw stage held at 0 unclassified. Both suspected Python defects at the head +of the queue were real and are fixed; a third, unsuspected one was found in the +date family; the alarming-looking FBG column turned out to be R manufacturing +readings, with no Python defect at all. + +**`height` (114 -> 0) and `bmi` (22 -> 0): one Python defect with two halves, +both live in production output.** `_apply_range_validation` (clean/patient.py) +converted a height to metres whenever it exceeded **2.3**, where R's +`transform_cm_to_m` converts only above **50**. A source value between the two +is neither unit -- the affected cells hold `2.31`, `2.37`, `2.43`, `2.52`, +`2.56`, `2.70`, `2.72`, `2.82`, `6.9`, `10.4`, `13.0` (120 cells, measured over +the whole raw output) -- so R divides nothing, the value fails the [0, 2.3] +bound and becomes the sentinel, while Python divided by 100 and published +`0.069`, `0.0243`, `0.13` metres. The threshold is now 50, which makes an +unusable cell sentinel instead of turning it into a plausible-looking metre +reading. The 2025 Señor Sto. Niño tracker records metres throughout +(1.52-1.73), so these are source defects for [ticket +40](40-source-defect-findings-report.md), not a third unit. + +The second half is ordering. R computes `fix_bmi` *after* cutting height, so an +out-of-bounds height propagates its sentinel into the BMI; Python computed BMI +at step 5.7, before range validation, from the raw height -- `60 / 2.43^2` = +10.16, which then passed the [10, 80] BMI bound and reached production as a +real-looking BMI derived from an impossible height. The BMI derivation now runs +inside `_apply_range_validation`, after the height and weight cuts and before +the BMI cut, exactly as R sequences it. + +**`fbg_updated_mg` (113 -> 0): no Python defect -- R manufactures glucose +readings from text.** All 85 cells where R holds a number and Python sentinels +were joined back to their raw source. `fix_fbg` +(r-archive/R/script2_helper_patient_data_fix.R:551) runs its CDC category +patterns through `grepl` on the whole string with no word boundary: + +- 41 cells read **`Lost follow up`** and become **140**, because "fol-low" + contains "low". A patient lost to follow-up has no reading at all, and R + publishes a fasting glucose for them. +- 41 cells record a meter range or an out-of-range marker -- `SMBG 50-HI`, + `DSMP 250-HI`, `129-HI`, `CBG 57-High`, `112/ High`, bare `HI` -- and all + become **200**, discarding the numeric endpoint the clinic wrote. On a + glucometer "HI" means a reading above the analytical ceiling, far above 200, + so the substitution is wrong in direction as well as invented. +- 3 are a genuine `low`/`Low`, R's rule working as designed. + +Python diverges here **by design and was already correct**: `_fix_fbg_column` +implements the same CDC mapping but anchors each pattern to the full string +(`^(low|good|okay)$`), so only a cell that says exactly "low" is a category and +everything else sentinels. The opposite direction (28 cells) is Python winning +outright: the 2018 trackers write `148 mg/dl (Mar-18)`, and Python strips the +unit and lifts the date into `fbg_updated_date` while R's `as.numeric` fails on +the whole string. Two classifiers, `r_fbg_text_category_invention` and +`r_unit_suffix_not_stripped`, both scoped to the two mg/dL columns `fix_fbg` +runs over. + +**`insulin_subtype` (67 -> 11): a real Python data loss, recovered.** 2024 +Sarawak General ticks the template's five insulin tick boxes by writing the +drug's name -- `Novorapid` in the rapid-acting column, `Glargine`, `Toujeo` or +`Ryzodeg` in the long-acting one. Both pipelines tested for `Y` exactly, so +both discarded the subtype, and Python's empty derivation was then published as +`Undefined` for 56 rows. `_insulin_ticked` (clean/patient.py) now reads any +value that is not a negative marker as a tick, and those rows carry +`Rapid-acting,Long-acting`. The negative set is enumerated rather than inferred +because the vocabulary is closed and tiny: across all 248 trackers these five +columns hold only `Y` (39,775), `-` (68,001), `0` (4), the four drug names +(116) and null. Classifier `r_drops_drug_name_tick`. + +**A third Python defect, found in the date family and fixed: +`parse_date_flexible` accepted a year with a digit missing.** 2024 Vietnam +National Children's writes `1/16/224` and `5/16/223`, 2023 Yangon General +writes `13-Mar-0202`, and other cells read `31 oct 222` and `1-Oct-205`. +dateutil reads each literally, so Python published `0224-01-16`, `0202-03-13`, +`0205-10-01` into production output -- 24 cells across six date columns. The +cleaned stage's future-date guard only bounds the calendar at one end; +`_MIN_PLAUSIBLE_YEAR = 1900` now bounds the other, in the parser so every route +into it is covered, and such a cell sentinels as unreadable. + +**What is left (448 in scope) and why it is not this session's.** The nine +smaller date columns still hold 412 cells, and measuring them by direction +showed the shape the ticket expected is not there: only 2 of 414 are a +day/month swap, none is a year-only difference, and the residual is a mix of +per-tracker populations (2021 and 2020 Mahosot DC alone account for ~150, with +R and Python each sentinelling where the other holds a real date). That is a +round of its own, and became [round 9](56-triage-patient-cleaned-residual-9.md) +together with the 11 remaining `insulin_subtype` cells and the ~25-cell tail. + +**Rejected.** +- *Adopting R's `HI` -> 200 convention.* It is not a clinical convention but a + CDC page about diagnostic thresholds; a meter reporting "HI" is above its + measuring range, so 200 understates a critical hyperglycaemia. Python's + sentinel is the honest reading and the source cell is a ticket 40 finding. +- *Extracting the numeric endpoint from `SMBG 50-HI` or `129-HI`.* These are + ranges over a month, not single readings; choosing either endpoint is a + guess, and the map's bar treats a guess as worse than a refusal. +- *Publishing null instead of `Undefined` for an insulin row with no tick.* + Tried and reverted after measuring: R publishes `Undefined` on 17,418 such + rows and Python already agreed with it there, so the change created 17,418 + new divergences while resolving none. Whether `Undefined` should claim a + subtype the clinic never recorded is a real question -- recorded as fog on + the map, in the same family as the numeric absence-as-word patch, not + smuggled into a triage round. +- *Treating the between-units heights as a third unit (feet, say).* 2.43 feet + is 74cm and 6.9 feet is 2.10m; no single interpretation fits the set, and the + tracker that holds most of them records metres correctly on every other row. +- *A tick rule of "anything that is not `Y` or `-`".* Would have read `0` as a + tick. The negative markers are enumerated instead. + +**Evidence.** Executed. R's `fix_fbg`, `transform_cm_to_m`, `cut_numeric_value` +and the `script2_process_patient_data.R` mutate that sequences them read +directly from `r-archive/`; every mismatch population joined back to Python's +own raw stage to recover the source strings; the parser's behaviour on the +truncated-year strings reproduced by calling `parse_date_flexible` directly. +Measured across three full pipeline runs and four comparison runs over the real +248-tracker drive data (`2026-08-19T195625Z` before, `2026-08-19T204438Z` the +reverted insulin attempt, `2026-08-19T204927Z` after the fixes, +`2026-08-19T205332Z` after the classifiers). Full suite 816 passed / 1 skipped, +ruff and `ty check src/` clean. + +**Tense.** Every count is current behaviour of the code as committed, measured +on run `2026-08-19T205332Z`. The `0.069`-metre and `0224-01-16` values +described as reaching production were doing so before this session's fixes. diff --git a/docs/wayfinder/tickets/56-triage-patient-cleaned-residual-9.md b/docs/wayfinder/tickets/56-triage-patient-cleaned-residual-9.md new file mode 100644 index 0000000..9cf6d18 --- /dev/null +++ b/docs/wayfinder/tickets/56-triage-patient-cleaned-residual-9.md @@ -0,0 +1,192 @@ +--- +id: 56 +title: Triage the residual patient cleaned-stage mismatches (round 9) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-19f +claimed_at: 2026-08-19 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 55 +--- + +## Premise + +Rests on [round 8](55-triage-patient-cleaned-residual-8.md), closed, which took +the in-scope cleaned residual from 770 to **448** and settles four things this +ticket must not re-open: + +- **`height`, `bmi` and `fbg_updated_mg` are decided.** Python's cm-to-m + threshold now matches R's 50, BMI is derived after the height cut, and R's + `fix_fbg` is confirmed to manufacture readings from text. Do not re-triage + them; if one reappears, a fix regressed. +- **`parse_date_flexible` now rejects any year below 1900.** A date in + antiquity is no longer reachable from a source year typed short. The date + columns below are what survives that fix, not what preceded it. +- **`insulin_subtype`'s drug-name rows are recovered** and only the 11 + `Undefined`-against-null cells remain. +- **An unticked insulin row keeping `Undefined` is deliberate**, not an + oversight: R publishes the same on 17,418 rows where the two pipelines agree. + It is fog on the map, not this ticket's to change. + +Rests on the eight cleaned-stage rounds before it, on [ticket +42](42-fbg-unit-headers-and-implausible-values.md) for the FBG analytical +bounds, on the map's rule that the current tracker template is the golden rule, +and on the standing bar that a classifier records understanding, never a +verdict by itself. + +Would be void, not merely rewritten, if the destination stopped requiring every +R/Python difference to be explained before R is retired. + +## Question + +Triage the **448** in-scope cells left on run +`output/comparison/2026-08-19T205332Z`, excluding `fbg_updated_mmol` ([ticket +44](44-triage-cleaned-fbg-r-null-residual.md)) and `hospitalisation_date` +([ticket 39](39-recover-dates-embedded-in-free-text.md)). Re-measure before +working any shape. + +- **The date family (412), which is now almost the whole residual.** + `hba1c_updated_date` (103), `fbg_updated_date` (91), `bmi_date` (87), + `recruitment_date` (58), `last_clinic_visit_date` (40), `lost_date` (18), + `t1d_diagnosis_date` (12), `dob` (7), `last_remote_followup_date` (3). Round + 8 measured the direction split rather than the causes, and killed the obvious + framings: only **2** of 414 are a day/month swap and **none** is a year-only + difference, so this is not a systematic parse-order divergence. It is + concentrated by tracker instead -- 2021 Mahosot DC and 2020 Mahosot DC + together account for roughly 150, and within them R and Python each sentinel + where the other holds a real date, on the same rows across all three of + `hba1c_updated_date`, `fbg_updated_date` and `bmi_date`. That the three + columns move together points at + `_extract_date_from_measurement` / R's `extract_date_from_measurement` (the + legacy path that lifts a date out of the measurement cell's parentheses) + rather than at nine separate causes. Start there, per tracker, and hand any + clinical-note share to [ticket 39](39-recover-dates-embedded-in-free-text.md) + rather than triaging it. +- **`insulin_subtype` (11).** R holds null, Python `Undefined`, in 2025/2026 + Calamba Doctors and 2025 Pahol. These rows tick nothing (`-` in all five + columns) -- but R publishes `Undefined` for the 17,418 rows whose columns are + all *null*, so R distinguishes the two cases and Python does not. Find what + R's derivation does with an all-`-` row and decide whether Python should + follow. +- **The long tail (~25):** `hba1c_updated` (8), `dob` (7), `fbg_baseline_mg` + (6), `remote_followup` (2), `testing_frequency` (1), `fbg_baseline_mmol` (1) + -- including the four `clinic_visit`/`remote_followup` rows the map's **Not + yet specified** section has carried since ticket 49, which belong with + whatever round works the tail. `fbg_baseline_mg` is worth a direct look + first: round 8 wired the FBG text classifiers to that column too, so what is + left there is a different mechanism. + +Split further rather than leaving this open-ended if it does not converge -- +eight rounds have already split rather than sprawled. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: explaining a +difference and naming a cause is not enough -- each needs an explicit verdict on +whether Python is right, with the evidence. Where Python is losing information +the source carried, fix the pipeline. Where the source workbook itself is +wrong, that is a legitimate conclusion and a finding for [ticket +40](40-source-defect-findings-report.md). A cause genuinely undecidable on the +available evidence is recorded as an open question, not closed with a label. + +## Resolution + +**Decision.** The nine-column date family is one R mechanism, not nine causes, +and Python is the correct side of all of it. Three classifiers now carry it, +and three Python gaps the same investigation exposed are fixed. The in-scope +cleaned residual goes **448 -> 96 (79%)**; the patient raw stage is +byte-identical at 0 unclassified and product is unchanged. + +**The mechanism, executed rather than read.** R's `parse_dates` +(`r-archive/R/script2_helper_patient_data_fix.R`) shortens any word of four or +more letters with `sub("([[:alpha:]]{3})[[:alpha:]]", "\\1", date)`, which +deletes the *fourth* letter and leaves the rest of the word standing, then +hands the wreckage to `lubridate::parse_date_time` with the fixed order list +`c("dmy","dmY","dbY","by","bY","mY","my","y")`. Nothing in that chain can fail +loudly. Installing lubridate and running it over the source strings taken from +the raw parquet reproduced R's frozen output exactly in every case: + +- `April-17` becomes `Aprl-17`, no order matches a month, the list falls to + `y`, and R publishes **2017-01-01**. Same for `August,2015` -> `Augst,2015` + -> 2015-01-01. That is `r_month_name_truncated_to_year` (88 cells). +- `9-Dce-20` has an unreadable month, so `my` reads the remaining digits as + month and year and R publishes **2020-09-01** -- the *day* promoted to a + month, the day itself invented as the 1st. Same for `11-Mach-20` -> + 2020-11-01, `4-Okt-2023` -> 2023-04-01, `10-MAC-2026` -> 2026-10-01. That is + `r_reads_day_as_month` (143 cells). +- `25-Mach-20` needs month 25, so the whole list misses and R sentinels; so do + `05/15/2026` and `Dec-22-2025`, month-first spellings no order in the list can + express. That is `r_parse_order_cannot_read_cell` (107 cells), deliberately + not applied to `hospitalisation_date`, whose residual round 6 measured as + 100% clinical notes -- R sentinels those because it cannot read a sentence, + which is [ticket 39](39-recover-dates-embedded-in-free-text.md)'s question + wearing the same shape. + +**Three Python gaps, all real data loss, all fixed.** Python sentinelled 32 +distinct source strings a human reads without effort. Verified the way round 6 +verified its parser change -- old parser against new over all **5,187** distinct +raw date strings on the 248-tracker set: **32 changed, every one from the +sentinel to a real date, and no already-parsing value changed its reading.** + +1. *Month names Python did not know.* `TYPO_REPLACEMENTS` gains `DCE` (a + transposition) and `UG` (a dropped letter), plus the Bahasa Malaysia + abbreviations the Malaysian clinics write in a column everyone else writes in + English -- `Mac`, `Mei`, `Okt` observed, `Ogos` and `Dis` added to complete + the set. A new `_THAI_MONTH_REPLACEMENTS` covers all twelve Thai + abbreviations with or without their full stops, for Nakornping's 2025 and + 2026 trackers. These are locale spellings, not typos, and the code says so. +2. *Separator runs damaged by a stray keystroke.* `26-05- 2007`, `19-Jan_2023`, + `02-Apr=-2026`, `23/05//2025`, `16-July-/2025`, `7_May-21` -- R recovers all + of these because lubridate splits on any non-alphanumeric run where dateutil + needs a well-formed separator. `_DAMAGED_SEPARATOR` collapses a run of two or + more separator characters, or a lone `_`/`=`, to a single `-` on a second + attempt only. Deliberately narrow: a single `/` or `.` is left alone so no + already-parsing value changes reading, and a whitespace-only run is left + alone so the trailing-free-text path still sees its word boundaries. +3. *Invisible characters.* A zero-width space pasted in from another + application (`11 Mar 2026`) is stripped. + +**The guard that measurement found, not reasoning.** The first version of the +separator repair turned `11-15 /01/2019` -- a *range* of two visit days -- into +2001-11-15, a date the cell does not hold. The repair now runs only when the +result leaves at most three numbers. + +**Rejected.** A general "any run of non-alphanumerics is a separator" rule, +matching lubridate: it is what produced the fabricated 2001-11-15, and it would +also have taken `26/102022` and `10/1023`, where the missing separator's +position is a guess. Fuzzy month matching by edit distance instead of an +explicit spelling list: it would have swallowed `Ma4` silently, which is exactly +the cell nobody can resolve. Classifying the R-sentinel shape on `day > 12` +across *all* date columns: measured first, and it would have mislabelled 79 +`hospitalisation_date` clinical notes as this cause -- the reason that column is +excluded by name. What the chosen route gives up: the three classifiers are +shape tests, so they cannot see the source string that proves the mechanism; +the proof lives in this comment and in the executed R runs behind it, not in +the code. + +**Deliberately not decided.** `insulin_subtype`'s 11 cells were traced to their +mechanism and left unclassified. R's derivation +(`script2_process_patient_data.R:101`) is a chain of `ifelse(x == "Y", ...)` +pasted together: an all-`-` row yields `""`, which `check_allowed_values` turns +into NA, while an all-null row yields `NA`, which pastes to a string and becomes +`Undefined`. So **R's null-vs-`Undefined` split is NA propagation, not a +designed distinction**, and R's null on these rows is not evidence of intent. +That makes these cells the map's existing fog entry on unticked insulin rows, +reached from the `-` side -- a change to shared behaviour on 17,418 rows, which +round 8 already measured and reverted. Left visible rather than labelled. + +**Evidence.** Executed: R's truncation and order list run over the real source +strings under R 4.5 with lubridate installed; the old-vs-new parser sweep over +5,187 distinct raw strings; a full patient pipeline run over the 248-tracker +drive set followed by a full four-stage comparison +(`output/comparison/2026-08-19T212139Z`); 843 tests, ruff, `ty check src/`. +Read: R's `extract_date_from_measurement` and `insulin_subtype` derivation, +whose consequences are described but not executed -- both are round 10's to +confirm. + +**Tense.** Every count above is current behaviour measured on the new run, not +a prediction. The 96 remaining are described in [ticket +57](57-triage-patient-cleaned-residual-10.md). diff --git a/docs/wayfinder/tickets/57-triage-patient-cleaned-residual-10.md b/docs/wayfinder/tickets/57-triage-patient-cleaned-residual-10.md new file mode 100644 index 0000000..90e77de --- /dev/null +++ b/docs/wayfinder/tickets/57-triage-patient-cleaned-residual-10.md @@ -0,0 +1,218 @@ +--- +id: 57 +title: Triage the residual patient cleaned-stage mismatches (round 10) +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-20 +claimed_at: 2026-08-20T10:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 56 +--- + +## Premise + +Rests on [round 9](56-triage-patient-cleaned-residual-9.md), closed, which took +the in-scope cleaned residual from 448 to **96** and settles three things this +ticket must not re-open: + +- **R's `parse_dates` is understood end to end**, and it is one mechanism, not + nine. It deletes the fourth letter of any 4+ letter word, then walks + `c("dmy","dmY","dbY","by","bY","mY","my","y")` over whatever digits remain -- + so a spelled-out month becomes 1 January of its year, an unreadable month + makes R read the *day* as the month, and a day past 12 leaves R with no + reading at all. Executed against R 4.5, not read. The three classifiers + `r_month_name_truncated_to_year`, `r_reads_day_as_month` and + `r_parse_order_cannot_read_cell` carry it, and Python is the correct side of + all three. +- **Malay and Thai month names, "Dce" and "ug" now parse.** `TYPO_REPLACEMENTS` + and `_THAI_MONTH_REPLACEMENTS` (clean/date_parser.py) map them before + parsing. If one reappears, a fix regressed. +- **A separator run damaged by a stray keystroke is repaired** -- but only when + the repair leaves at most three numbers, so a date range is never collapsed + into one date. + +Rests on the nine cleaned-stage rounds before it, on the map's rule that the +current tracker template is the golden rule, on the standing bar that a +classifier records understanding and never a verdict by itself, and on +[ticket 42](42-fbg-unit-headers-and-implausible-values.md) for the FBG +analytical bounds. + +Would be void, not merely rewritten, if the destination stopped requiring every +R/Python difference to be explained before R is retired. + +## Question + +Triage the **96** in-scope cells left on run +`output/comparison/2026-08-19T212139Z`, excluding `fbg_updated_mmol` ([ticket +44](44-triage-cleaned-fbg-r-null-residual.md)) and `hospitalisation_date` +([ticket 39](39-recover-dates-embedded-in-free-text.md)). Re-measure before +working any shape. Five groups, and unlike earlier rounds they share no +mechanism: + +- **The 2017/2018 measurement-cell dates (~27).** `fbg_updated_date` and + `hba1c_updated_date` where R holds the first of a month and Python holds + null or the sentinel, in 2017/2018 Mandalay, Penang DC, CDA and Mahosot. + These trackers have **no date column at all** in Python's raw output: R's + `extract_date_from_measurement` (script2_helper_patient_data_fix.R) splits a + measurement cell written `8.53 (28/8/2017)` into a value and a date, and + Python never does this for those years. This is the legacy path round 9 was + sent looking for and did not find in the Mahosot DC concentration -- it lives + here instead. **Decide whether Python should extract it too**; if so, this is + a pipeline fix, not a classifier. +- **`25-Ma4-2025` (18), one cell in 2025 Taunggyi.** "Ma4" is Mar or May and + nothing in the workbook says which; R reads the digits positionally and + publishes 2025-04-25, a third answer neither spelling supports. The row's + `status` is `Active` while carrying a `lost_date` at all. Most likely a + finding for [ticket 40](40-source-defect-findings-report.md) rather than + something to infer -- say so explicitly rather than labelling it. +- **Glued or stray digit groups (~19).** `26/102022`, `8/1023`, `10/1023`, + `3/10.23`, `10-Oct-2-24`, `13-Mar-0202`. Where the missing separator goes is + a guess, and R's own answer is not stable: R's frozen output has + `10/1023` as 2010-10-23 while running R's `parse_dates` over that string in + isolation returns 2023-10-10. Decide whether any of these are recoverable at + all, or whether they are source defects. +- **The numeric tail (14).** `hba1c_updated` 240, 299 and 125, and + `fbg_baseline_mg` 47.8 and 53.8, where R publishes the number and Python + stamps 999999; plus one `fbg_baseline_mmol` and one `testing_frequency` + holding a date. The hba1c values look like a glucose reading typed into the + HbA1c column, but the FBG pair sits inside the mg/dL analytical bounds and + should not have been rejected -- **check why Python sentinels it before + assuming Python is right.** +- **`Undefined` on a row that ticks nothing (13).** `insulin_subtype` (11) and + `remote_followup` (2). Round 9 established the mechanism and deliberately did + not fix it: R's `ifelse(x == "Y", ...)` chain yields `""` for an all-`-` row, + which `check_allowed_values` turns into NA, and `NA` for an all-null row, + which becomes `Undefined`. R's null-vs-`Undefined` split is NA propagation, + not a designed distinction, so R's null here is not evidence of intent. These + cells are the **Not yet specified** fog entry on unticked insulin rows + reached from the `-` side, and they stay unclassified until that question is + answered. Confirm `remote_followup` is the same shape; do not decide the fog. + +Split further rather than leaving this open-ended if it does not converge -- +nine rounds have already split rather than sprawled. + +## Standing bar + +Per the map's **triage means deciding, not labelling** preference: explaining a +difference and naming a cause is not enough -- each needs an explicit verdict on +whether Python is right, with the evidence. Where Python is losing information +the source carried, fix the pipeline. Where the source workbook itself is +wrong, that is a legitimate conclusion and a finding for [ticket +40](40-source-defect-findings-report.md). A cause genuinely undecidable on the +available evidence is recorded as an open question, not closed with a label. + +## Resolution (session-2026-08-20) + +**Decision.** The in-scope cleaned residual falls **96 -> 19** (80%), the raw +stage is byte-identical at 26,172 and product is untouched. Every one of the 19 +carries a written verdict: 13 are the standing `Undefined` fog and 6 are a +tooling limitation named below. **There is no round 11** -- this closes the +ten-round patient cleaned-stage chain. + +One pipeline fix, four classifiers, and a verdict on each of the five groups. + +**The measurement-cell group was a real Python defect, and this ticket's +premise about it was wrong.** The premise said Python "never does this for +those years". It does: `_extract_date_from_measurement` +(`src/a4d/clean/patient.py`) is a documented port of R's function and has been +called from `_apply_legacy_fixes` all along. The defect was one character. +R's `separate_wider_regex` makes the closing parenthesis **optional** +(`[)]?`, `script2_helper_patient_data_fix.R:124`) and R's own test suite +covers an unclosed cell (`"10.75 (01-Sept-2022"`, +`test-helper_patient_data_fix.R:229`). Python's `\(([^)]+)\)` required the +closer -- and the 2017/2018 Mandalay, CDA, Mahosot and Penang DC trackers +write `180(May-2017`, `201(Jun-2017`, `8.4(Jan-2018` and never close it. +25 of the 30 cells in this group had no closing parenthesis at all. +Python now uses R's shape: greedy prefix, date after the **last** `(`, +optional closer. **24 cells recovered**, and the dates are right -- MM_MD022's +Dec17 row now publishes `fbg_updated_mg = 180`, `fbg_updated_date = +2017-05-01`, matching R and the source cell. + +**Matching R exactly would have destroyed data, and the measurement caught +it.** R's greedy prefix leaves `196(` as the *value* for `196((Dec-2017)`, and +R's own `as.numeric` then fails on it -- R loses the reading of 196. The first +version of this fix reproduced that faithfully, which made the comparison count +*drop by two* because both sides were now equally wrong. Python strips the +stray `(` and keeps 196. This is a deliberate divergence from R, in the +direction the map's standing bar requires. + +**The two FBG cells this ticket flagged as wrongly sentinelled are correctly +sentinelled.** The ticket's suspicion was that 47.8 and 53.8 "sit inside the +mg/dL analytical bounds and should not have been rejected". Measured: 2020 +Kantha Bopha's `fbg_baseline_mg` is 382 of 424 non-null readings below 30 -- +**90.1%**, just over ticket 42's 90% threshold -- so `resolve_glucose_units` +reads the whole column as mmol/L, and 47.8 mmol/L is past the analytical +ceiling. They are not mg/dL readings, so the mg/dL bounds never applied. The +verdict rests on the ticket-42 assumption in force, and 90.1% is close enough +to the threshold that both cells are reported for source correction rather than +called settled. + +**The other three groups: R invents, Python declines, and the source is the +defect.** `25-Ma4-2025` (18 cells), the glued digit groups (19) and the 2018 +Penang DC template placeholder `e.g. xxx (mth-18)` (6) are one story told +three ways. Round 9 established by running R 4.5 that `parse_dates` cannot fail +quietly, so R always publishes *something*; where the source token is damaged +past reading, that something is an invention, and R's own answer is not even +stable (`10/1023` is 2010-10-23 in R's frozen output and 2023-10-10 when R's +parser is run over that string in isolation). Python declining is correct, and +all 37 cells are workbook fixes -- appended to [ticket +40](40-source-defect-findings-report.md). + +**Because.** Every verdict here was measured rather than argued: the source +value behind all 96 cells was joined back from Python's raw parquet, the +parenthesis shapes counted (25 unclosed of 30), the swap threshold recomputed +against the real column, and the whole patient arm re-run and re-compared twice +against the real 254-tracker drive data. + +**Rejected.** + +- *Making Python match R byte-for-byte on `196((Dec-2017)`.* Rejected because R + loses the reading. Costs a permanent 2-cell divergence in `fbg_updated_mg` + and `fbg_updated_mmol`, which is the right trade. +- *Widening `_is_python_glucose_unit_corrected` to "outside either unit's + range" so it would catch the six Kantha Bopha cells.* Written, measured + against the test suite, and **backed out**: it would also claim every + genuinely-rejected mg/dL reading between 45 and 800 as a unit correction. + Whether a column was swapped is a property of the column; a per-cell + classifier cannot see it. The six stay `unclassified` with the reason + recorded, which the map's bar explicitly permits, rather than wearing a label + that would be wrong for most of the population it could match. +- *Inferring a date for `25-Ma4-2025` or the glued digit groups.* Rejected: + nothing in the workbook decides between Mar and May, and R's instability on + `10/1023` shows its number is not evidence either. +- *Deciding the `Undefined`-on-an-unticked-row question (13 cells).* Out of + scope by round 8's measurement -- changing it created 17,418 new divergences. + Confirmed this round that `remote_followup` (2) is the same shape as + `insulin_subtype` (11): both source cells are empty, R holds null, Python + holds `Undefined`. Left to the fog entry. + +**Evidence: executed.** The pipeline fix, the parenthesis counts, the 90.1% +swap threshold, the recovered values and the 96 -> 19 movement were all run +against the real drive data, not read. R's optional `[)]?` and its unit test +were **read** from `r-archive/` source, not re-executed -- but the behaviour +they predict is confirmed by R's frozen output on those same cells. 860 tests, +ruff and `ty check src/` all pass. + +**Tense.** The 96 -> 19 figure, the recovered dates and the 19-cell residual +describe current behaviour after this session's change, verified on run +`output/comparison/2026-08-20T074301Z`. Nothing here is a proposal. + +### What was added + +- `_extract_date_from_measurement` (`src/a4d/clean/patient.py`) -- R's regex + shape, minus R's stray-`(` data loss. 5 unit tests. +- `python_declines_unreconstructable_date` -- R on a real date, Python on the + sentinel. Wired to every schema-derived date column except + `hospitalisation_date` (ticket 39's, same carve-out round 9 made), and + appended last so it only ever sees what nothing narrower claimed. 37 cells. +- `python_rejects_out_of_range_hba1c` -- a fasting-glucose reading typed into + the HbA1c column, rejected against the bound read from + `validation_rules.yaml` rather than restated. 8 cells. +- `stray_date_dropped` -- the third face of ticket 24's stray-date cause, for + an integer-typed column where Python's failed cast leaves null rather than 0. + 1 cell. +- `python_recovers_glucose_r_rejected` -- the unit swap moving across a reading + R had already sentinelled. 1 cell. diff --git a/docs/wayfinder/tickets/58-patient-list-join-uses-unfixed-id.md b/docs/wayfinder/tickets/58-patient-list-join-uses-unfixed-id.md new file mode 100644 index 0000000..ba05e15 --- /dev/null +++ b/docs/wayfinder/tickets/58-patient-list-join-uses-unfixed-id.md @@ -0,0 +1,218 @@ +--- +id: 58 +title: Monthly rows with a misspelled ID silently lose their Patient List demographics +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-24d +claimed_at: 2026-08-24T20:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 47 +--- + +## Premise + +Rests on [Four trackers where cleaning merges several patients into one patient +ID](47-patient-ids-merged-at-cleaning.md), closed, which established that a +malformed `patient_id` is now recovered against the tracker's own well-formed +IDs (edit distance 1, unique candidate) or sentinelled -- and that this happens +in **cleaning**, at `fix_patient_id` (`src/a4d/clean/validators.py`), step 7. + +Extraction runs long before that. `extract/patient.py` (~line 1107) joins the +`Patient List` sheet onto the monthly rows with +`.join(patient_list_join, on="patient_id", how="left", suffix=".static")`, +keyed on the **raw, unfixed** ID. So a monthly row whose ID is misspelled +cannot match its own Patient List entry, and the row keeps its measurements but +loses every static column the join supplies. + +Measured (executed) after ticket 47's fix, on the real 254-tracker output: +`KH_NP026`'s recovered `Sep23` row has null `dob`, `sex`, `province` and +`t1d_diagnosis_date`, where the same patient's `Oct23`, `Nov23` and `Dec23` +rows carry all four. Ticket 47 fixed the identity; it did not fix this. + +Ticket 47 deliberately did not touch it: changing the join key changes matching +for every tracker, which is a different measurement from the one that ticket +made. + +**What would void this ticket rather than rewrite it:** ticket 47's recovery +being reverted, since without recovery these rows have no identity to attach +demographics to in the first place. + +## Question + +1. **Measure the real blast radius.** The 4 NPH rows are the known case, but + the hyphen-spelled IDs are the likely larger one: `2021_Mahosot Hospital A4D + Tracker_DC` writes `LA-MH056/057/058` on its monthly sheets, and + `2026_Surat Thani…_Jun_26` writes `TH-ST029` -- do those workbooks' Patient + Lists use underscores, and are those rows missing demographics today? Derive + the count across all 254 trackers rather than reasoning from these two. + Note that a raw-stage null `dob` is a weak proxy (2,063 rows across 62 files + carry one, most for unrelated reasons) -- find a measure that isolates + *join misses* specifically. +2. **Decide where the normalization belongs.** Candidates: normalize both sides + of the join key inside extraction; move `fix_patient_id` earlier; or join a + second time after cleaning. Each has a different reach -- the first changes + the raw parquet's own `patient_id`, which the comparison tool's patient + row-alignment key depends on. +3. **Check R.** R's own join ordering may or may not have the same gap; if R + matches these rows and Python does not, the comparison tool has been + reporting it as ordinary cell divergence all along. + +## Standing bar + +Per the map's **triage means deciding, not labelling**: measure before +proposing, and say explicitly whether Python is losing data a source workbook +carries. + +## Resolution + +**Decision.** Both whole-tracker joins in `extract/patient.py` -- the +`Patient List` demographics and the `Annual` sheet -- now key on a **derived +normalized ID** (`join_static_sheet`), not the raw one. The normalization rule +is the one cleaning already applied (fold `-` to `_`, drop any transfer-clinic +suffix), lifted out of `clean/patient.py::_apply_preprocessing` into +`extract/common.py::normalize_patient_id_expr` so the two layers cannot drift +into disagreeing about what one patient's identity is. + +The key is derived **for the join only**. `patient_id` in the raw parquet is +still the spelling the month sheet used, which is what the raw layer promises +and what the comparison's patient row-alignment key depends on -- so this +change creates no row-key divergence at all (raw row-key divergence held at +4 files / 115 rows, cleaned at 6 / 129). + +The user chose this over the two alternatives after seeing the measurement. + +**1. The blast radius is 100x the ticket's known case, and it is one clinic.** +Derived across all 254 real trackers by extracting each workbook's own +`Patient List` with the pipeline's own extractor and diffing its ID set +against that tracker's raw monthly IDs -- the join-miss measure the ticket +asked for, rather than the null-`dob` proxy it warned off. + +| File | Rows | What the month sheets write vs the Patient List | +|---|---|---| +| `2024_Mahosot Hospital A4D Tracker` | 397 | `LA-MH056`..`LA-MH093` (38 patients) vs `LA_MH0xx` | +| `2023_Mahosot Hospital A4D Tracker` | 275 | `LA-MH056`..`LA-MH082` (27 patients) vs `LA_MH0xx` | +| `2024_Mandalay General Hospital A4D Tracker` | 11 | `MM_YG013_MG`, absent from that Patient List entirely | +| `2024_CDA A4D Tracker` | 6 | `KH-CD016/017` vs `KH_CD0xx` | +| `2023_NPH A4D Tracker` | 4 | ticket 47's stray `H` | +| `2026_Surat Thani Hospital A4D Tracker_Jun_26` | 2 | `TH-ST029` vs `TH_ST029` | + +**695 rows in 6 files; 680 recoverable by normalization, carrying 6,863 +Patient List cells** across 12-13 columns -- `hba1c_baseline`, +`recruitment_date`, `patient_consent`, `lost_date`, `status_out` and the +diagnosis block, not just the four the ticket named. The boundary is exact: +`LA_MH055` had all 12 months of `dob`/`sex`/`province`/`t1d_diagnosis_date`, +`LA_MH056` had none of them. + +**2. A second join had the identical defect and the ticket did not know about +it.** The `Annual` sheet join (same function, ~40 lines later) also keyed on +the raw ID. Measured the same way over the 131 trackers that have an Annual +sheet: **132 miss rows, 19 recoverable, 40 cells** (8 rows carrying data; the +other 11 are `VN_VC070`, whose Annual row is blank). Small, but leaving one +join on the raw key while the other used the normalized one is exactly the +drift the shared helper exists to prevent, so both were fixed together. The +remaining 113 misses name a patient the Annual sheet does not list at all -- +a source fact, not a key mismatch. + +**3. R has the identical gap, and the comparison was structurally blind to +it.** R's own frozen cleaned `2024_Mahosot` shows the same null block starting +at `LA_MH056`. Both sides null means no cell mismatch, so this never appeared +in `cell_mismatches` -- the same shape as [ticket +59](59-triage-unmatched-row-keys.md)'s unmatched rows and [ticket +42](42-fbg-unit-headers-and-implausible-values.md)'s unit swap. It is a +divergence the tool cannot report, found only by measuring the workbooks +directly. That answers question 3: R is **not** matching these rows, so this +was never being reported as ordinary cell divergence. + +**4. The safety check that made the choice cheap.** Normalizing the key can +only hurt by fan-out -- two static entries folding to one key would silently +duplicate a patient's month rows. Measured across all 192 Patient-List-bearing +trackers: **zero** colliding keys, and zero Patient Lists with exact duplicate +IDs. The Annual sheet showed 20 files with apparent collisions, all of which +turned out to be junk rows (`'0'`, `'NEW'`) that already collide on the *exact* +key. `join_static_sheet` still dedupes and logs, as a guard against a future +workbook rather than a current one. + +**Rejected.** +- *Also run `fix_patient_id`'s edit-distance recovery at join time*, which + would have caught NPH's 4 rows too. Rejected: it pulls cleaning's fuzzy + identity recovery into the raw layer, where [ticket + 47](47-patient-ids-merged-at-cleaning.md) deliberately kept it out, to gain + 4 rows. Those 4, plus Mandalay General's 11, are **source defects** and go to + [ticket 40](40-source-defect-findings-report.md). +- *Join a second time after cleaning.* Rejected: the Patient List rows do not + survive into the cleaned stage, so this means re-reading each workbook or + persisting a second parquet -- much larger machinery for the same result. +- *Rewrite the raw `patient_id` itself instead of deriving a key.* Rejected: + the comparison aligns patient rows on it, and the raw layer's promise + (ticket 27) is that it records what the source cell actually contained. +- *Wire the new cause to every patient column and let its own gate bound it.* + Tried and **measured**, not reasoned about -- it over-claimed on the three + columns the month sheets also carry (`t1d_diagnosis_age` 1,069 rows against + a join-miss population of 679, `bmi` 144, `age` 1), where R's null is not + necessarily the join's fault. Backed out to an explicit column list. + +**5. The divergence the fix creates is named.** New cause +`r_static_join_misses_respelled_id` (`migration/compare.py`): R null, Python +present, and the row's own ID still carrying the hyphen or transfer-clinic +suffix. It fires on **5,959 raw-stage cells across 16 columns**, every +per-column count matching the measured delta exactly, and **raw-stage +`unclassified` is now zero** (was 3,577 immediately after the fix, 0 before +it). + +It is deliberately **raw-stage only**. The discriminator is the unnormalized +spelling, which is exactly what cleaning removes -- by the cleaned stage both +pipelines publish `LA_MH056` and the cause cannot tell itself apart from +`r_extraction_gap`'s column-naming failures. Cleaned-stage `unclassified` is +therefore **16 -> 3,593**, and that residue is [ticket +63](63-name-the-cleaned-stage-static-join-divergence.md). + +One trap worth carrying forward: the classifier silently never fired on its +first run, because `add_row_ordinal` renames every key column to +`__key_` and a bare `m.key.get("patient_id")` finds nothing. +`_sheet_year` documents the same trap. There is now a regression test for it. + +**Evidence: executed.** Every number above is from a run, not a reading. The +blast radius and the collision check are corpus-wide scans of the real 254 +trackers on the USB drive; the Patient List contents were read from +`Laos/MHS/2024_Mahosot Hospital A4D Tracker.xlsx` itself (`LA_MH056`: +Vientiane Capital, M, D.O.B. 2014-02-22, recruited 2021-06-22, all present); +the recovery was verified by re-running `a4d run patient` over all 254 +trackers and re-counting. R's matching gap was read from R's own frozen +cleaned parquet. Full suite 958 passed / 1 skipped, `ruff check`, +`ruff format --check` and `ty check src/` clean; 22 new tests across +`test_extract/test_common.py`, `test_extract/test_patient_helpers.py` and +`test_migration/test_compare.py`. + +Comparison totals, before this session -> after: patient raw cell divergence +**26,171 -> 33,488**, patient cleaned **114,306 -> 121,742**. Product is +byte-identical ("no change from the previous run" on both stages). Every one +of those new cells is Python publishing a value the source workbook carries +and R does not. + +**Tense.** Everything describes current behaviour on this branch after the +change, except the R descriptions, which are of `r-archive/` and `output_r/` +as frozen. + +## Spawned + +- [Name the cleaned-stage half of the static-join divergence](63-name-the-cleaned-stage-static-join-divergence.md) + -- 3,593 cleaned-stage cells with no cause, plus the three columns the + corpus-wide wiring over-claimed on. + +## Reported to ticket 40 + +- `2023_Mahosot Hospital A4D Tracker` and `2024_Mahosot Hospital A4D Tracker`: + every month sheet writes `LA-MH056` and up with a **hyphen**, while the + `Patient List` in the same workbook writes `LA_MH056` with an underscore -- + 38 patients in 2024, 27 in 2023. Both pipelines lost every demographic for + those patients; Python now recovers them, but the workbooks should be made + consistent. +- `2024_CDA A4D Tracker`: `KH-CD016` and `KH-CD017` on the month sheets + against `KH_CD016`/`KH_CD017` in the Patient List. +- `2024_Mandalay General Hospital A4D Tracker`: the month sheets carry + `MM_YG013_MG` for 11 rows, but that workbook's `Patient List` has no such + entry -- it lists `MM_MG013` and `MM_YG064_MG`. One of those is presumably + meant; the pipeline cannot tell which, so the patient has no demographics. diff --git a/docs/wayfinder/tickets/59-triage-unmatched-row-keys.md b/docs/wayfinder/tickets/59-triage-unmatched-row-keys.md new file mode 100644 index 0000000..7568a5e --- /dev/null +++ b/docs/wayfinder/tickets/59-triage-unmatched-row-keys.md @@ -0,0 +1,198 @@ +--- +id: 59 +title: Rows that pair with nothing on the other side, which no ticket has ever triaged +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-24 +claimed_at: 2026-08-24 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 47 +--- + +## Premise + +Rests on [Four trackers where cleaning merges several patients into one patient +ID](47-patient-ids-merged-at-cleaning.md), closed, whose comparison re-run +exposed this. That ticket's four recovered rows stop pairing with R, and +finding *which* rows they were required querying the parquets by hand -- so +`RowKeyOverlap` was extended to carry the unmatched keys, not just their +counts, and the report gained a `row_key_unmatched` sheet. + +The sheet's first run named a population nobody had looked at. Measured +(executed) against the real 254-tracker output on the USB drive: + +- **Patient cleaned: 144 unmatched rows across 7 files.** Only 8 of them are + ticket 47's (`2023_NPH`, 4 R-only + 4 Python-only). The rest: + `2026_Preah Kossamak…_Jun_26` 98 R-only, `2022_Children's Hospital 2` 15 + R-only, `2024_Mandalay Children's` 11 Python-only, + `2026_Quirino Memorial…_Jun_26` 5 R-only, `2026_YGH…_June_26` 3 R-only and 3 + Python-only, `2024_Mahosot Hospital` 1 Python-only. +- **Patient raw: 130 unmatched rows.** +- **Product, both stages: 0** -- resolved by + [ticket 17](17-fix-product-row-alignment-and-triage.md)'s ordinal key. + +This population is structurally invisible to every triage ticket the map has +run. A row with no partner never reaches `compare_cells`, so it appears in no +`cell_mismatches` sheet, contributes to no per-column or per-cause count, and +cannot be classified -- the ten-round patient chain worked exclusively off +those counts. It is the same shape of gap as `compare_columns` was before +[ticket 26](26-triage-product-column-divergence.md). + +**What would void this ticket rather than rewrite it:** the counts turning out +to be an artifact of the row-alignment key rather than of the data, which +[ticket 45](45-patient-row-alignment-duplicate-keys.md) settled for patient +(the key is unique in 249 of 254 files) -- so this is unlikely, but it is the +thing to check first. + +## Question + +1. **Is this a divergence at all, or a shape difference?** A whole tracker + present on one side only is already reported separately as a file-level + finding; these are *rows* inside files both sides produced. Establish + per file whether R is holding rows Python drops, Python is producing rows R + never had, or the two are the same rows keyed differently. +2. **Start with `2026_Preah Kossamak…_Jun_26`'s 98 R-only rows**, the largest + single population and 68% of the patient cleaned total. 98 rows is a + plausible whole clinic-month. +3. **Decide each, to the map's standing bar** -- for every group, whether + Python is right, R is right, or the source workbook is defective. Ticket + 47's four rows are already decided (Python correct, recorded there) and + should be excluded rather than re-argued. + +## Standing bar + +Per the map's **triage means deciding, not labelling**. Note that the usual +instrument does not work here: there is no cell mismatch to classify, so the +cause registry cannot carry the answer and the resolution comment has to. + +## Resolution (session-2026-08-24) + +**Decision.** The 144 cleaned / 130 raw unmatched rows are **five distinct +populations, not one**, and each is now decided. Two were pipeline defects -- +one Python's, one R's -- two were already-closed decisions resurfacing under a +key the comparison could not pair, and one is a source defect with a behaviour +question the user settled. Python is the correct side of all five. + +| population | rows | verdict | +|---|---|---| +| 2022 Children's Hospital 2, `Oct22` | 15 R-only | **Python defect, fixed.** Whole sheet was being dropped | +| 2026 Preah Kossamak `May26` + 2026 Quirino `Jan26` | 98 + 5 R-only | Source defect; Python's drop kept, now **reported** rather than silent | +| 2024 Mandalay Children's, `MM_MD001` | 11 Python-only | **R defect.** Python correct, no change | +| 2026 YGH, `MM_NO55_MW_YG` | 3 + 3 | Same rows keyed differently; ticket 47 already decided it. Source defect | +| 2023 NPH / 2024 Mahosot | 4+4 / 1 | Already decided by tickets 47 and 43; excluded per this ticket's own scope | + +**Because.** + +*Children's Hospital 2 (Python defect).* `find_data_start_row` scanned column A +for the first **numeric** value. In `Oct22` the first patient's row-number cell +(`A70`) holds a whitespace-only string -- Excel residue of a cleared number -- +so Python started at row 71 and read its two header rows from 70 and 69. Row 70 +*is* data, so the patient-ID header became the literal string `VN_CH001`, +harmonization found no `patient_id`, and `read_all_patient_sheets` skipped the +entire sheet. Fifteen patients' October records were lost. R's rule -- +`which(!is.na(tracker_data[, 1]))`, any non-NA cell -- lands correctly. + +The rule was chosen by scanning all 254 trackers first, per this map's standing +preference, and the scan **killed the obvious fix**: 28 month sheets have a +non-empty column-A cell above the first numeric one, and 14 of them (2026 +Gensan, 2025/2026 VNCH) hold a stray `'m'`/`'f'`/`'n'` in row 1 -- adopting R's +rule wholesale would set those sheets' data start to row 1 and destroy them. +What Python now does instead: extend the numeric block back over a +whitespace-only cell **directly abutting** it, one row only. Measured over all +254 trackers, that moves the start on **exactly one sheet** -- this one. + +*Preah Kossamak and Quirino (source defect + a behaviour decision).* Both sheets +carry `#REF!` in the patient ID *and* name columns on every row, and +`read_all_patient_sheets` dropped them via +`filter(~pl.col("patient_id").str.starts_with("#"))` with **no error record at +all** -- just an INFO line counting "invalid rows". The rows are not empty: +Preah's 98 carry 50 ages and updated FBG readings, 47 baseline HbA1c / weight / +height / BMI, 45 insulin regimens. + +The user decided (2026-08-24) to **keep dropping and start reporting**. The +reasoning is about the group key, not the row: `Undefined` is a bucket rather +than an identity, so keeping these rows would pool 98 people's measurements +under one `patient_id` and quietly corrupt any per-patient grouping downstream. +That does not contradict [ticket 47](47-patient-ids-merged-at-cleaning.md), +which *keeps* a malformed ID under `Undefined`: a misspelled ID is a real +identifier a human wrote and a clinic could reconcile against its own records, +whereas `#REF!` is not an identifier at all -- the cell's content is gone, so +there is nothing to reconcile, ever. New `excel_error_patient_id` error code, +one record per dropped row. It fires on **120 rows across 9 trackers**, not the +103 this ticket could see -- the comparison only ever showed the rows R also +kept. + +*Mandalay Children's (R defect, Python correct).* R hardcodes +`row_min <- row_min + 1` for 2022+ trackers +(`script1_helper_read_patient_data.R:67`) because "openxlsx always skips empty +rows at the start of the file" -- it assumes exactly **one** leading empty row. +Verified in the workbook: `Jan24`'s row 1 is genuinely empty, and every other +month sheet holds a single space in row 1. So openxlsx skips nothing on those +11 sheets, R's `+1` pushes the data window one row down, and `MM_MD001` -- a +real patient with a full clinical record every month -- falls out of 11 of 12 +sheets. Exactly the same whitespace residue as the Children's Hospital 2 case, +mishandled in the opposite direction. + +*YGH.* Source ID is `MM_NO55_MW_YG` (a transfer, two clinic codes spliced). R +truncates to `MM_NO55_`; Python sentinels to `Undefined` under ticket 47's rule. +Same three rows, two keys -- no data difference, and Python is right by a +decision already made. Reported to [ticket 40](40-source-defect-findings-report.md). + +**Rejected.** + +- *Adopt R's "first non-empty cell" data-start rule.* Killed by the 254-tracker + scan: 14 sheets would start at row 1 on a stray letter. This is why the fix is + the narrow abutting-cell rule and not the general one. +- *Back up over a whole run of blank-string cells rather than one.* No + multi-row run exists in the corpus, and swallowing several would risk reading + a header row as data -- the same class of bug in reverse. +- *Keep the `#REF!` rows under the `Undefined` sentinel* (option A as put to the + user). The measurements are real and would be usable for clinic-level trends, + but they would share one group key with every other unidentified patient. What + this gives up is stated plainly: 120 rows of genuine clinical measurement are + discarded, and only a workbook repair can bring them back. +- *Fix R's `+1` offset.* R is being retired ([ticket 12](12-retire-r-workspace.md)); + its bug is documented, not repaired. +- *Add a cause classifier for any of this.* There is no cell mismatch to + classify -- an unmatched row never reaches `compare_cells`. The resolution + comment is the record, as this ticket's own Standing bar anticipated. + +**Evidence.** `executed` throughout. + +- The ticket's numbers re-measured against the run at + `output/comparison/2026-08-22T084333Z` before any work: 144 / 130 / 0 / 0, + unchanged since the ticket was written. +- Every mechanism reproduced by running the code against the real workbook on + the USB drive, not read: the `Oct22` header misread reproduced through + `read_all_patient_sheets` and observed as its own "no 'patient_id' column, + skipping" warning; the `#REF!` rows' contents counted column by column; the + Mandalay row-1 space read out of all 12 sheets with `openpyxl`; R's `+1` + offset read from its own source at line 67 and confirmed against which sheet + keeps `MM_MD001` in R's frozen output (`Jan24`, alone). +- The abutting-cell rule scanned across **all 254 trackers** before it was + written, and again after, to confirm it moves exactly one sheet. +- Full pipeline re-run against the real 254-tracker set and a full comparison + against the frozen `output_r/` + (`output/comparison/2026-08-23T225035Z`): row-key divergence **144 -> 129** + cleaned and **130 -> 115** raw, the delta being exactly the 15 recovered rows + at both stages. Cell mismatches moved by **+22 only** -- `t1d_diagnosis_age` + 4,800 -> 4,814 and `fbg_baseline_mg` 11,839 -> 11,847, both onto existing + named causes -- and **`unclassified` did not move on any stage** (16 patient + cleaned, 0 patient raw, 21 product cleaned, 0 product raw). Product untouched + at both stages, patient raw's per-column snapshot byte-identical. +- 931 tests, `ruff format --check`, `ruff check`, `ty check src/` all pass. + +**Tense.** The Children's Hospital 2 loss and the silent `#REF!` drop were +**current production behaviour** until this session; both are now fixed on +`migration`. R's `MM_MD001` loss is current behaviour of the archived R +pipeline and will not be fixed. The 120 discarded `#REF!` rows remain discarded +-- that is the decision, not a defect left open. + +**What is left, and where it went.** The residual unmatched population is +**129 cleaned / 115 raw**, and every row of it is now decided: 103 are the +`#REF!` source defect, 11 are R's offset bug, 6 are YGH's spliced ID, 8 are +ticket 47's, 1 is ticket 43's. Nothing is unowned. Three source-defect findings +were written to [ticket 40](40-source-defect-findings-report.md). diff --git a/docs/wayfinder/tickets/60-audit-remaining-pre-bar-classifiers.md b/docs/wayfinder/tickets/60-audit-remaining-pre-bar-classifiers.md new file mode 100644 index 0000000..4d4fe3d --- /dev/null +++ b/docs/wayfinder/tickets/60-audit-remaining-pre-bar-classifiers.md @@ -0,0 +1,205 @@ +--- +id: 60 +title: Audit the eight pre-bar causes the first classifier pass did not reach +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-24b +claimed_at: 2026-08-24T12:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 32 +--- + +## Premise + +Rests on [the classifier re-audit](32-audit-classifiers-against-decision-bar.md), +closed, which established the scope this ticket inherits and the standard it is +judged by: + +- **Twelve** of the 50 causes in `src/a4d/migration/compare.py` predate the + map's **triage means deciding, not labelling** preference (dated by first + commit: 2026-08-11 and 2026-08-12). The other 38 were written after it, each + argued in its own triage ticket, and are **out of this audit's scope** -- + ticket 32 rejected re-deriving them. +- Four of the twelve are resolved: `off_by_one_day` and `ce_typo` deleted, + `sentinel_null` renamed to `python_sentinel_r_extraction_gap`, + `r_value_missing` bounded by `python_out_of_window_date_preserved` and + `python_absurd_excel_serial`. + +Rests also on the map's standing preference that naming a cause is not a +decision in favour of Python's value, and on ticket 32's demonstration that the +weakness is usually not in the mechanism but in the *silence about the Python +side*: `r_value_missing` was correct about R for 11,436 of 11,468 rows and wrong +to speak for the other 32. + +**Void rather than in need of rewriting** if ticket 32's scoping is overturned +-- i.e. if someone decides the post-bar 38 must be audited too, this ticket is +the wrong unit of work and the whole audit should be re-cut by registry. + +## Question + +Audit the eight remaining pre-bar causes against the two-bar standard. For each: +what the mechanism actually is, what evidence establishes it, and the explicit +verdict on whether Python is correct. Where Python is wrong or lossy, fix the +pipeline rather than keeping the label. + +Current volumes, from the 2026-08-22 run against the real 254-tracker set: + +| cause | rows | ticket 32's read | +|---|---|---| +| `r_extraction_gap` | ~31,000 | "likely fine", but later widened to serve three columns and three separately-verified R mechanisms -- the widening is the risk | +| `r_validator_rejects_multivalue` | 15,647 | "likely fine" (documented R validator bug) | +| `excel_formula_error` | 12,680 | "likely fine", both directions verified by ticket 27 | +| `row_order_divergence` | 7,480 | mechanism established; the "is Python correct" half is strong but was never written down as a verdict | +| `r_category_lookup_miss` | 866 | "likely fine" (traced to `read_product_data.R`'s unnormalized join) | +| `openpyxl_date_typed_stray_cell` | 100 | "likely fine" (source-Excel verified) | +| `buddhist_era_typo` | 6 | "likely fine", but later made symmetric -- re-check at its new scope | +| `wide_format_fragment_truncated` | 5 | "likely fine" (R's value verified a strict prefix of Python's) | + +Three things ticket 32 established that this ticket should use rather than +re-derive: + +1. **`row_order_divergence` needs a verdict written, not more evidence.** + Ticket 25 already proved per-column multiset equality across all 2,283 + `(clinic_id, product_sheet_name)` groups for `product_units_released`, and + row-identity preservation across all 11,649 product groups. What is missing + is the explicit statement that Python's chronological sort is the correct + side, plus the caveat that the `row_order_candidate` flag itself is a loose + membership test that false-fires on repeated small numeric values. + `product_balance`'s under-detection is already explained and already homed on + [ticket 36](36-triage-product-cleaned-unclassified-residual.md). +2. **`r_extraction_gap`'s lumping is the thing to test.** `classify()` cannot + see the file, so one cause covers three mechanisms across three columns. + Ticket 32's method applies directly: scan the whole population, then ask + which rows the mechanism does *not* account for. +3. **Ask what the classifier says about the Python side.** Every one of ticket + 32's four findings came from that question, not from doubting R. + +## Residual inherited from ticket 32 + +Product cleaned `unclassified` is 21 rows (11 `product_balance`, owned by ticket +36; 10 `product_entry_date`). The `product_entry_date` ten include a population +nobody has named: rows where R holds a date roughly a year ahead of Python's -- +`2028-02-26` vs `2026-02-26` (2026 NOGH), `2022-12-12` vs `2021-12-30` (2021 +VNCH), `2025-05-20` vs `2024-05-30` (2024 Surat Thani). Whether that is one +mechanism or several is unmeasured. + + +## Resolution + +**Decision.** Six of the eight causes are audited to the two-bar standard and +Python is the correct side in every one. One carried a real defect and is +fixed; one had its verdict written rather than re-evidenced; two were not +reached and are split into +[ticket 62](62-finish-the-pre-bar-classifier-audit.md). + +**The defect: `buddhist_era_typo` was naming twelve cells that are not +Buddhist-era dates.** Its whole test was `carried.year >= 2400`, which after +[ticket 61](61-decide-buddhist-era-date-conversion.md) is exactly backwards -- +the cleaned stage now *converts* genuine BE dates, so a year still sitting +above the threshold is by construction not a recoverable one. Both real +populations are ordinary year typos, verified in the workbook itself: +`3035-03-01` (2025 CDA, `Mar25!O219`) and `5025-05-19` (2025 Surat Thani, +`May25!O70`), each a genuine date-formatted Excel cell whose year is neither +Gregorian-plausible nor 543 off its tracker's. The cause is now bounded by the +sheet's own BE band, mirroring `_is_python_absurd_excel_serial`; where the +sheet year cannot be read the old behaviour stands, since an unknown tracker +year cannot disprove a BE date. + +Bounding it exposed a second false label, which this ticket owns because its +own change surfaced it: five of the twelve fell through to +`r_parse_order_cannot_read_cell`, which asserts R's parse orders failed. R's +parse orders did not fail -- R's raw output holds the Excel serial `1141523` +(and `414611` for CDA) as a string, and it is the comparison's own +`normalize_date_column` that turns those into the sentinel. That is +`python_absurd_excel_serial`'s documented shape (ticket 32 created it for +`1339576` and `411384`; CDA's `414611` is a near sibling of the latter), and it +was only ever wired to the product entry-date registry. It is now prepended to +every patient date column, which is safe only because its own BE band check +declines a genuine BE year -- and that band could not previously apply to +patient columns at all, since no patient stage sets `tracker_year_col`, so it +now falls back to `_sheet_year` the way the other patient date causes do. + +Measured end-to-end against the real 254-tracker pair, before and after, every +per-cause count in all four stages: **exactly four entries moved and nothing +else changed.** `buddhist_era_typo` 12 -> 0; patient cleaned +`python_rejects_beyond_tracker_year` 36 -> 42 (R keeps the absurd year, Python +sentinels it -- true); patient raw `python_absurd_excel_serial` 0 -> 6. +`unclassified` is unchanged at 37, so no row was pushed into the unexplained +bucket -- all twelve moved from a false label to a true one. + +**`r_extraction_gap`'s lumping is real and wider than its docstring**, which is +what ticket 32 predicted. The test is a blanket `r_value is None and py_value is +not None`; the docstring enumerates six mechanisms over seven columns; the +population spans **thirteen**. Three further mechanisms traced to source, Python +correct in all three: + +- `insulin_regimen`, 2021 Kantha Bopha, 194 rows -- `Mar21`/`Apr21` carry the + patient header at row 85 with `Q85`/`Q86` *empty* where every other sheet has + `Q13 = "Insulin Regime"`. R publishes 0 for those two sheets and 97 for each + of the others; `Mar21!Q87` is `"Self-mixed BD"` and Python recovers the name + from the sibling sheets. The documented `clinic_visit` blank-header + mechanism, second instance. +- A **blank row number on the 2026 `Annual` sheet**, 7 rows -- 2026 ISDFI rows + 32-34 and 2026 Khon Kaen row 27 are exactly the rows whose column A is empty. + This **corrects an explanation already in the registry**: the comment above + those same ISDFI cells attributes them to R reading nothing from the Annual + sheet, but R reads that file's Annual sheet perfectly well (102 of 120 + `edu_occ`). The sheet is not the mechanism; the missing row number is. +- `edu_occ`, 2026 Nakornping, 42 rows -- `Annual!E9` appends Thai to the header, + which R's Unicode-aware sanitizer keeps and Python's ASCII folding drops. + `r_non_latin_header_miss`'s mechanism absorbed here on registry order. + +**`r_validator_rejects_multivalue` covers two R defects, not one.** 570 of its +15,647 rows have a Python value with no comma at all, 510 a bare `Pre-mixed` -- +which the multi-value story cannot explain, since R publishes `Pre-mixed` +itself 5,426 times. The second mechanism is NA propagation through R's builder: +`ifelse(NA == "Y", ...)` yields `NA`, `paste` stringifies it as `"NA"`, and the +`.x[.x != ""]` filter does not remove it, so a single tick beside unfilled +siblings still produces `"pre-mixed,NA,NA,NA,NA"`. Measured: where R publishes +`Pre-mixed` the siblings are non-null in 4,134 of 4,135 rows; where R publishes +`Undefined` on a pre-mixed tick, 575 have a null sibling. R's output contains no +comma-joined value anywhere and no `Rapid-acting` at all (the `"rapic-acting"` +typo). Python reads the identical raw ticks -- verified on 2024 NPT, MM_NC009, +Apr24 -- and is correct. + +**`excel_formula_error`: Python is correct and the test's symmetry is +unexercised.** All 12,680 rows run one way (Python holds the error string, R +null) across `t1d_diagnosis_age` 8,495, `bmi` 4,170, `age` 15, and every one is +raw-stage. Executed: 0 error-string cells in those columns across every cleaned +parquet on both sides, so nothing reaches the tables and raw's extra fidelity +costs downstream nothing. + +**`row_order_divergence`: verdict written, no new evidence.** Python's +chronological sort is the correct side; both sides implement the same rank +algorithm and differ only in which rows have a parsable date to sort by. +Recorded with the caveat ticket 32 asked for -- `row_order_candidate` is a loose +value-membership test that false-fires on repeated small numeric values. + +**`wide_format_fragment_truncated`: confirmed at full population.** All 5 rows +are strict prefixes (`'2(Error'` -> `'2(Error-1)'`), which is exactly what the +docstring claims. + +**Because.** Ticket 32's method -- scan the whole population, then ask which +rows the mechanism does not account for -- found something in every cause it was +applied to. The two it did not reach are the two it was not applied to. + +**Rejected.** Re-deriving the 38 post-bar causes: ticket 32 ruled them out and +nothing here disturbs that. Leaving the five parse-order rows on their false +label because that classifier is post-bar and out of scope: this ticket's own +change put them there, so it owns them. Widening `r_extraction_gap` into +separate named causes per mechanism: the name is accurate for all thirteen +columns (R loses data Python keeps) and splitting it would churn counts without +changing a verdict -- the docstring was the thing that was wrong. + +**Evidence: executed.** Pipeline re-run over all 254 trackers, full comparison +before and after, per-cause diff across all four stages, source workbooks opened +for every cell named above. Full suite 938 passed / 1 skipped, ruff and +`ty check src/` clean. + +**Tense.** The counts and the four moved entries describe current behaviour +after this session's change, measured. `r_extraction_gap`'s and +`r_validator_rejects_multivalue`'s docstring corrections change no behaviour at +all -- no count moved for either. diff --git a/docs/wayfinder/tickets/61-decide-buddhist-era-date-conversion.md b/docs/wayfinder/tickets/61-decide-buddhist-era-date-conversion.md new file mode 100644 index 0000000..cbc7481 --- /dev/null +++ b/docs/wayfinder/tickets/61-decide-buddhist-era-date-conversion.md @@ -0,0 +1,261 @@ +--- +id: 61 +title: Decide whether a Thai clinic's Buddhist-era entry date is published as 2567 or converted to 2024 +labels: [wayfinder:grilling] +status: closed +blocked_by: [] +assignee: session-2026-08-22b +claimed_at: 2026-08-22T12:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 32 +--- + +## Premise + +Rests on [the classifier re-audit](32-audit-classifiers-against-decision-bar.md), +closed, which narrowed `_validate_entry_dates`' era exemption +(`clean/product.py`) from "any year >= 2400" to "this tracker's own +Buddhist-era year", and in doing so established the facts this question needs: + +- **22 rows** in the cleaned product table carry a genuine Buddhist-era date, + all from Thai clinics -- 2024/2023/2022/2025 Chiang Mai Maharaj Nakorn, 2024 + Phattalung. They arrive as BE-shifted Excel serials (~243,933) which + `parse_date_flexible` reads to a year in the 2565-2568 range, matching each + sheet's month exactly (`2567-11-11` in `Nov24`). +- The pipeline publishes those years **as written**: `product_data` holds an + entry date of 2567-11-11 for a November 2024 stock movement, 543 years in the + future to any downstream reader. +- Ticket 32 deliberately did not decide this. It fixed only the corrupt cases + (`5567-08-19`, `3026-04-30`, `2525-10-02`), which are now sentinelled and + logged under `implausible_era_date`. + +Rests also on the map's standing preference that where a divergence traces to a +human error in the source workbook the fix is the workbook -- which is what +makes this a genuine question rather than an obvious yes: a BE date is not a +clinic's *error*, it is the calendar the clinic uses. + +**Void rather than in need of rewriting** if the era exemption is removed +altogether, since there would then be no BE dates in output to convert. + +## Question + +Should the product pipeline convert a Buddhist-era entry date to Gregorian +before publishing it? + +Candidate answers, with what each costs: + +1. **Convert (subtract 543)** where the year lands in the tracker's own BE band + -- the band the pipeline already computes, so the test is free and cannot + fire on a Gregorian date. Downstream gets one calendar. Costs: it moves 22 + rows of real data; R does not do it, so it becomes a new, deliberate R/Python + divergence needing its own cause; and if the band is ever wrong the pipeline + silently rewrites a date. +2. **Publish as written, flag for source correction** -- emit an informational + error record so the clinics are asked to write Gregorian, and leave output + untouched until they do. Costs: BigQuery keeps 22 far-future dates for as + long as it takes. +3. **Convert and flag** -- both, treating the conversion as a recovery like + `date_recovered_from_text`, auditable through the error table rather than + silent. + +Ticket 39's precedent argues for auditability over silence: it recovered dates +from clinical notes but wrote 1,223 error records so every recovery could be +reviewed. Whether the two situations are alike is exactly what needs deciding -- +a BE year is unambiguous once identified, while a note-embedded date is a guess. + +Points to settle: + +- Does the **patient** arm have the same population? `PATIENT_BUDDHIST_ERA_CLASSIFIERS` + exists with the same 2400 threshold, and ticket 40 already records + "Buddhist-Era years typed into Gregorian date cells" as a patient-side source + defect -- so patient may need the same answer, or may already have a different + one. Measure before deciding, per the map's scan-the-population preference. +- What does the **downstream consumer** do with `product_entry_date`? If it + filters or buckets by year, a 2567 row is not merely odd-looking, it is + invisible or misfiled. +- Is a BE year in a Thai tracker the clinic's convention or an Excel locale + artifact? The serials suggest the workbook itself is in a Thai-calendar Excel, + which would mean nobody typed anything wrong. + +## Decision (user, 2026-08-22) + +**Convert.** Where a date is clearly not Gregorian and its intended value can be +determined unambiguously, the pipeline converts it like any other recovery -- +**in the cleaned stage, not in raw**, so raw keeps what the workbook actually +says. This resolves the question above in favour of candidate 3 (convert *and* +flag), consistent with [ticket 39](39-recover-dates-embedded-in-free-text.md)'s +precedent that a recovery is auditable rather than silent. + +Not yet implemented -- this records the decision, not the work. + +## Measured population (session-2026-08-22, real 254-tracker set) + +The ticket asked to measure the patient arm before deciding. Done, and **the +patient arm is the larger and more urgent half**, which the ticket's premise did +not know: + +- **Product cleaned: 22 rows**, all published today as BE years (`2567-11-11` in + a `Nov24` sheet). Visible but wrong-looking. +- **Patient: 381 cells across 95 distinct values, and every one is currently + destroyed.** Patient's `_validate_dates` clobbers any future date with the + 9999-09-09 sentinel -- so unlike product, patient publishes nothing odd + *because it publishes nothing at all*. The cells were invisible for exactly + that reason: no comparison sheet shows them, since R sentinels them too. + Scanned by re-parsing every raw patient date column with + `parse_date_flexible` and keeping years in [2400, 9999). + +Affected clinics are Thai throughout: 2024/2025/2026 Chiang Mai Maharaj Nakorn, +2026 Chulalongkorn (the largest -- whole complication-screening date columns in +BE), 2026 Nakornping, 2023 Nakornping, 2022 Hat Yai. + +**The band test the product fix already uses appears to be the right rule for +patient too, and one case argues it strongly.** 2022 Hat Yai's `t1d_diagnosis_date` +holds `2560-01-01` (12 cells). BE for 2022 is 2565, so 2560 looks like a typo -- +but it falls inside the band `[tracker_year + 543 - YEAR_FLOOR_DELTA, +tracker_year + 543]` = 2560-2565, and converting it gives **2017-01-01**, which +is exactly the date [ticket 40](40-source-defect-findings-report.md) independently +established from that patient's own D.O.B. (2013-03-17), recruitment (2017-06-01) +and age at diagnosis (4). The band recovers the right answer without being told. + +Patient needs a *wider* lower bound than product, and the data shows why: a +screening or diagnosis date legitimately predates its tracker. 2026 +Chulalongkorn records `2567-07-18` (2024) in a kidney-test column, correctly. + +**Two values, 6 cells, fall outside any band and must stay sentinelled**, +becoming source-defect findings rather than conversions: `3035-03-01` (2025 CDA, +1 cell) and `5025-05-19` (2025 Surat Thani, 5 cells). Neither decodes to a +plausible year by subtracting 543. + +**Still open before implementing:** whether `hospitalisation_date`'s two +note-embedded BE cells (`18-19/11/2567`, `19-22/5/2568`) come along. Ticket 39 +refused them on the grounds that subtracting 543 would be guessing at the +clinic's calendar -- this decision overturns that reasoning for cells the band +test identifies, but those two also carry a date *range*, which is a separate +problem ticket 39 already declined. + +## Resolution (session-2026-08-22b) + +**Decision.** Both arms convert a Buddhist-era date to Gregorian in the cleaned +stage, and log every conversion under a new `buddhist_era_converted` error code +so the shift is auditable rather than silent. Implemented, run against the real +254-tracker set, and every resulting R/Python divergence named. + +The open question the previous session left -- whether +`hospitalisation_date`'s two note-embedded BE cells (`18-19/11/2567`, +`19-22/5/2568`) come along -- is answered **no** by the user (2026-08-22): a +date *range* is a separate problem [ticket +39](39-recover-dates-embedded-in-free-text.md) already declined, and this +decision does not overturn that half. They are reported as source defects +instead ([ticket 40](40-source-defect-findings-report.md)). + +**What was built.** + +- `clean/buddhist_era.py` (new): `BUDDHIST_ERA_OFFSET`, `BUDDHIST_ERA_THRESHOLD` + and `gregorian_from_buddhist()`, shared by both arms. The shift goes via a + string rather than `pl.date`/`dt.replace` because both of those *raise* on + invalid components: 543 is not a multiple of 4, so a Buddhist leap day can + land on a non-leap Gregorian year (2568-02-29 -> 2025-02-29) and would abort + the tracker. It yields null there instead, and the cell is left for the + ordinary implausible-date handling rather than invented. +- `_convert_buddhist_era_dates` (`clean/patient.py`, step 5.4). Domain is + `get_date_columns()` minus `tracker_date` -- derived, not listed. Placed + before `_fix_age_from_dob` so no age is ever derived from a BE `dob`, and + before `_validate_dates`, which is what was destroying these cells. +- `_validate_entry_dates` (`clean/product.py`) converts inside the BE band it + already computed for ticket 32 rather than passing the date through. + +**The band, and why patient's differs from product's.** Product converts only +inside `[table_year + 543 - 5, table_year + 543]`. Patient has no lower bound +at all: a diagnosis or screening date legitimately predates its tracker by +decades, and 2026 Chulalongkorn correctly records `2567-07-18` (2024) in a +kidney-test column. The upper bound alone -- shifted year no later than the +tracker's own year -- is what rejects a value that decodes to nothing, and it +does: `3035` -> 2492 and `5025` -> 4482 both stay sentinelled. + +**Measured against the real 254-tracker set** (`a4d run --force`, both arms in +one execution, then `just compare-outputs` against `output_r`): + +- **Patient: 375 cells converted** across 11 columns and 7 Thai trackers, each + logged. The 381 the previous session measured minus the 6 that decode to + nothing -- exactly the split it predicted. +- **Product: 22 rows converted**; `product_data` now holds **zero** entry dates + with a year past 2400 (queried directly), where it previously published + stock movements dated 543 years in the future. +- **The Hat Yai case came out right without being told.** `2022_Hat Yai` + `TH_HY013`'s `t1d_diagnosis_date` now reads 2017-01-01 against a D.O.B. of + 2013-03-17 and `t1d_diagnosis_age` 4 -- the date [ticket + 40](40-source-defect-findings-report.md) had derived by hand. +- **Patient cell divergence is unchanged at 114,284**, which is the expected + result: these cells already mismatched, with Python holding the sentinel; + now it holds a date. Product cleaned rose 22,699 -> 22,735 (+36) from the + re-sort described below. **Unclassified did not move on either arm** (16 and + 21). + +**Two new causes in the comparison tool**, both needed by this map's bar that a +difference is named rather than left over: + +- `python_buddhist_era_converted` (287 cells, both arms): R keeps the BE year, + Python holds exactly the same day 543 years earlier. The test is the exact + shift -- same month, same day -- so a second divergence riding along cannot + hide behind the name. It takes over 281 cells `buddhist_era_typo` used to + hold, 12 `python_rejects_beyond_tracker_year` held, 16 that + `python_out_of_window_date_preserved` was over-claiming, and 1 that + `note_dates_read_differently` had swallowed. It is prepended for every + `get_date_columns()` column, not hand-listed. +- `buddhist_era_conversion_row_order` (3 cells): converting moves the row. + Product rows pair by ordinal position within (clinic, sheet), and R sorts an + unconverted 2565 date to the end of its group while Python sorts 2022 + chronologically. Verified on the real pair -- 2022 Chiang Mai Maharaj + Nakorn, `Dec22`, "Accu-Chek Instant Test Strips": R's ordinal 11 holds + 2565-12-24 where Python holds 2022-12-28, and Python's ordinal 9 holds the + converted 2022-12-24. Plain value membership cannot see this (R's value is + by construction absent from Python's group), so `compare_cells` now also + tests membership of the *shifted* value, via a new + `era_shift_row_order_candidate` flag. + +That same re-sort also moved 36 cells into causes that already existed +(`row_order_divergence` +21 across four columns, `r_value_missing` +17, +`derived_running_total_row_order` +4, offset by +`python_out_of_window_date_preserved` -16). Stated as a consequence of this +change, not as new evidence about R. + +**Rejected.** + +- *Publish as written and flag only* (candidate 2): rejected by the user's + decision. On product it leaves BigQuery holding dates 543 years out; on + patient it is not even available, because publishing as written is what + `_validate_dates` destroys. +- *Convert in the raw stage*: rejected by the decision -- raw is a faithful + record of what the workbook says, and the whole audit trail depends on that. +- *Convert the two hospitalisation ranges too*: rejected this session (above). +- *Widen the patient band downward with an explicit floor* (e.g. 1900): + considered and dropped as dead weight. The 2400 threshold already implies a + floor of 1857, and no cell sits near it. +- *Let `row_order_divergence` absorb the three displaced product cells* by + normalizing R's BE value before the membership test: rejected because the + cells' *values* still differ for a reason worth naming; folding them into a + generic order cause would have hidden the conversion behind it. + +**Evidence.** Everything above is **executed**: a full `a4d run --force` over +the 254-tracker set, `duckdb` queries against the resulting parquets and error +table, two `just compare-outputs` runs, and 927 passing tests (`ruff`, `ty +check src/` clean). The one judgement is the band rule itself -- that a year +past 2400 which decodes to no later than the tracker year *is* a Buddhist-era +date. It is recorded as an assumption on the map. + +**Tense.** All counts describe current behaviour on `migration` after this +session's commit, not a proposal. + +### Not done here + +- The **2 range cells** and the **6 undecodable era cells** are now [ticket + 40](40-source-defect-findings-report.md)'s to report; that ticket's body was + updated in this session, including retiring "BE years in date cells" as a + defect class now that they are converted. +- **`buddhist_era_typo` still exists** at its new scope of exactly 6 cells (the + ones Python cannot convert). [Ticket + 60](60-audit-remaining-pre-bar-classifiers.md) lists it as one of the eight + pre-bar causes to audit, and its table's "6 rows" now matches what the + classifier actually holds. diff --git a/docs/wayfinder/tickets/62-finish-the-pre-bar-classifier-audit.md b/docs/wayfinder/tickets/62-finish-the-pre-bar-classifier-audit.md new file mode 100644 index 0000000..7761b8f --- /dev/null +++ b/docs/wayfinder/tickets/62-finish-the-pre-bar-classifier-audit.md @@ -0,0 +1,231 @@ +--- +id: 62 +title: Finish the pre-bar classifier audit — the two causes and the one bulk population it did not reach +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-24c +claimed_at: 2026-08-24T16:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 60 +--- + +## Premise + +Rests on [the classifier re-audit](32-audit-classifiers-against-decision-bar.md), +closed, which scoped the audit to the **12** causes predating the map's *triage +means deciding, not labelling* bar and explicitly ruled the other 38 out. + +Rests on [the eight pre-bar causes](60-audit-remaining-pre-bar-classifiers.md), +closed, which audited six of the eight and left this remainder. It also +established the method that keeps finding things: scan the whole population, +then ask which rows the stated mechanism does *not* account for. Applied to +`r_extraction_gap` it found three undocumented mechanisms; applied to +`r_validator_rejects_multivalue` it found a second R defect; applied to +`buddhist_era_typo` it found the cause was naming twelve cells that are not +Buddhist-era at all. + +**Void rather than in need of rewriting** if ticket 32's scoping is overturned +-- if the post-bar 38 must be audited too, this is the wrong unit of work and +the whole audit should be re-cut by registry. + +## Question + +Three populations, all judged by the same two bars — what the mechanism +actually is, and the explicit verdict on whether Python is correct. + +1. **`r_category_lookup_miss`** (866 rows, `product_category`, product cleaned). + Ticket 32 read it as "likely fine", traced to `read_product_data.R`'s + case- and whitespace-sensitive `add_product_categories()` join where + `src/a4d/reference/products.py` normalizes both sides. Never re-measured + against the current 254-tracker set. The population test applies directly: + is every one of the 866 explained by a normalization difference, or does + some other shape ride along? + +2. **`openpyxl_date_typed_stray_cell`** (100 rows). Its docstring names only the + raw product columns `product_units_received` and `product_received_from`, + but ticket 60 found it also fires on **20 patient rows** it does not mention + — `t1d_diagnosis_age` (16), `blood_pressure_mmhg` (3), `testing_frequency` + (1). The values look like the same mechanism (R holds the raw Excel serial, + e.g. `20668.0`; Python holds the date the cell's own format declares, e.g. + `1956-08-01`), but the *source cells were not re-opened* for the patient + rows, so that is inference from shape, not verification. Note the patient + values are nonsense as ages — the source cell is a date-formatted cell in an + age column, which likely also belongs in + [ticket 40](40-source-defect-findings-report.md). + +3. **`r_extraction_gap`'s bulk** — `recruitment_date` (28,009) and + `edu_occ_updated` (2,770), together ~98% of the cause and the two + populations ticket 60 did **not** re-measure. Both have a documented, + source-verified mechanism; what is unmeasured is whether the mechanism + accounts for the whole population, which is exactly the question that + turned up three new mechanisms in the other 11% of the same cause. + +## Why this still needs `r-archive/` + +All three questions are about what R's own code does, so the R source has to +stay until they are answered — the reason +[ticket 12](12-retire-r-workspace.md) is blocked on this ticket rather than on +its predecessor. + +## Resolution + +**Decision.** All three populations are audited to the two-bar standard and +**Python is the correct side in every one**. No pipeline change was needed or +made — this is the first ticket in the audit chain to find no defect. What it +found instead is that all three *explanations* were wrong or incomplete, so the +three docstrings in `src/a4d/migration/compare.py` are rewritten against +measured mechanisms. The audit that ticket 32 scoped to twelve pre-bar causes is +now complete. + +**Because.** Ticket 60's method — scan the whole population, then ask which rows +the stated mechanism does not account for — found something in all three, again. + +**1. `r_category_lookup_miss` (866 rows) is two mechanisms, and the larger one +is not about R's join at all.** The docstring blamed a "case or whitespace" +difference against a join R does unnormalized. Bucketing all 866 by the exact +byte difference between R's string, Python's string and the reference entry: + +- **652 rows / 26 names: CRLF vs LF.** The reference workbook stores embedded + line breaks as a bare `\n` — its Stock_Summary sheet XML holds 29 LF bytes and + **zero** CR bytes — while the trackers store `\r\n` (2021 Surat Thani's + `sharedStrings.xml`: 80 CRLF pairs). readxl returns each faithfully, so R + compares `'Accu-Chek Performa Test Strips \r\n(50s/ bottle)'` against a + reference entry spelled with `\n` and misses. openpyxl normalises CRLF to LF + on read, so Python's two sides match byte-for-byte. The reference sheet + already *contains* all 26 names verbatim — R should have joined them, and the + reason it does not has nothing to do with case or with whitespace-sensitivity. + This is the same reader difference ticket 22 added + `normalize_whitespace_column` for, resurfacing as a lookup failure. +- **214 rows / 3 names: genuine case.** `'(singles)'` against the reference's + `'(Singles)'` on NovoFine (98) and NIPRO (25), and `'ACCU-CHEK Performa + Glucometer Set'` against `'Accu-Chek Performa Glucometer Set'` (91). Verified + in R's source that the join is case-sensitive on values: + `add_product_categories` (read_product_data.R:481) is a bare + `dplyr::left_join`, and `load_product_reference_data` (:438) lowercases only + `colnames`. This is the half the docstring had right. + +Python correct in both; R loses a category its own reference data can supply. +The names R and Python fail on *together* (`Contour Plus Elite Meter`, +`Aphatyl Vitamin`, …) are genuinely absent from the reference and produce no +divergence — which is why they are not in the 866. + +**2. `openpyxl_date_typed_stray_cell` (100 rows) states one verdict for three +shapes, and for two-thirds of the population that verdict is backwards.** The +detection is sound — every cell behind the cause was opened and genuinely +carries a date/time number format — but "Python's value is the faithful one, +R's is a column-wide coercion artifact" holds for only 14 rows: + +- **A real date in a quantity column, 14 rows.** Serial in the 43,566-45,652 + range. Ticket 24's verdict stands; `_is_stray_date_zeroed` covers the cleaned + face, where R carries 43,708 into the stock ledger and Python zeroes it. +- **A quantity or a zero in a date-formatted cell, 66 rows** (48 holding 0, read + back as `00:00:00`; 18 holding a genuine count of 5-200, read back as a 1900 + date). Here **R's serial is the number the clinician typed and Python's date + is the misleading rendering**. Source-verified at 2023 Yangon General + `Apr23!G25` and siblings: column G carries a `d-mmm-yy` format over cells + holding 150, 200, 30, 5 and 0, alongside `Entry Date`, `Balance` and + `Units Received` headers at row 27. It costs nothing, because the divergence + does not survive cleaning — both sides publish identical cleaned values for + every affected `(product, sheet)` group (executed: 2023 YGH `Apr23` + Insunova/NovoRapid/Insupen/Easydrip all 5/150/30/200 on both sides). A + raw-stage representation artifact, so the pipeline is deliberately unchanged, + on ticket 32's `parse_date_flexible` precedent. +- **The patient arm, 20 rows / 5 source cells / 5 trackers**, which the + docstring never mentioned. Each is a value Excel silently auto-converted on + entry, all five re-opened in the workbook and confirmed date-formatted: + `10/60` → Oct-1960 in a "Blood Pressure (mm HG)" column (2020 Mahosot + `Nov20!R152`/`Dec20!R153`, 2022 Mahosot `Jun22!Z323`, all `mmm-yy`); `1-2` → + 1-Feb in "Testing Frequency (per day)" (2021 Khon Kaen `Mar21!P60`, `d-mmm` — + and that column holds `' 1 - 2'` four rows above, which is what the clinician + meant); and a diagnosis *date* in "Age at Diagnosis" (2023 Chiang Mai + `Patient List!H14`, 2023 Yangon General `Patient List!H76`). **Python is + unambiguously correct and visibly so**: it nulls them, while R publishes + diagnosis ages of **20,668 years** (TH_CP005, 12 sheets) and **42,859 years** + (MM_YC043, 4 sheets). All five already reach the errors table as + `type_conversion` with the original value intact, so + [ticket 40](40-source-defect-findings-report.md) needs nothing added. + +**3. `r_extraction_gap`'s bulk: the stated mechanism is false for the larger +half, and the true one is much sharper.** The docstring said R's extraction +"fails to populate `recruitment_date` for the large majority of patients". R in +fact populates **53,331 of its 82,771 raw rows (64%)**, and the divergence is +**perfectly file-level and all-or-nothing**: over the 239 comparable files, 91 +where R reads *zero* recruitment dates, 148 with no divergence, and **zero** +files where R reads some and misses others. + +The cause is a header cell whose text ends in a space, which forces Excel to +write `xml:space="preserve"`. Executed: 2023 CDA's `Patient List!L8` is literally +`Date of Recruitment +`. R reads its headers with `openxlsx::read.xlsx` +(script1_helper_read_patient_data.R:15), whose inline-string parsing folds that +attribute into the header text, so `make.names` emits the column +**`xmlspacepreservedateofrecruitmentmmmyy`** and no synonym matches it. The +correlation is exact: **all 91 affected files carry that column in R's own raw +parquet, and none of the other 148 do.** openpyxl parses the attribute correctly +and reads `'Date of Recruitment '`. The same defect eats at least one further +column R never maps (`xmlspacepreserveinsurancestatusnanssfeqeqpending`). + +`edu_occ_updated` (2,770 rows, 16 files) is also whole-file (13 files at zero), +but unlike `recruitment_date` it is **not one mechanism**: R's raw output carries +three distinct junk names for it — `xlevelofeducationoroccupationdate` (the +documented leading-space fixup), `levelofeducationoroccupationอาชพหรอชนเรยน` and +`...dateupdatedddmmmyyyy` (ticket 60's Thai-header mechanism). Fifteen files +carry a junk name *and* still resolve the column, so unlike recruitment_date the +junk name alone does not predict the loss. + +**One of this ticket's own premises was false, and cost nothing to check.** It +said the patient rows' "source cells were not re-opened, so that is inference +from shape". They were — ticket 50 opened them and recorded the exact cells in +the `blood_pressure_mmhg`/`testing_frequency` note in +`scripts/compare_outputs.py`. The genuine gap was that the *classifier's own +docstring* never mentioned the patient arm. This session re-opened all five +independently and found the same cells, plus 2022 Mahosot `Jun22!Z323`. + +**Rejected.** +- *Renaming `openpyxl_date_typed_stray_cell`.* "Stray" over-reads for 66 of its + 100 rows — the cell is not stray, only its number format is. But `openpyxl` + and `date_typed` hold for all 100, ticket 60's precedent is to fix the + docstring where the name is accurate, and the name is referenced across eight + closed tickets and two pipeline source comments that a rename would orphan. + Recorded in the docstring instead. +- *Splitting `r_extraction_gap` per mechanism.* Same reason ticket 60 rejected + it: the name is true of all thirteen columns and splitting churns counts + without changing a verdict. +- *Making Python's raw stage read the shape-B cells as numbers.* It would move + production raw output to fix a divergence that cleaning already erases — the + cleaned parquets are identical on both sides today. +- *Normalising line endings in Python's category join.* Correct in principle and + deliberately not done here: it changes a production join for a case nothing in + the current data triggers, and it needs its own measurement. Recorded as fog + on the map instead. +- *Auditing the 38 post-bar causes.* Unchanged from tickets 32 and 60. + +**Evidence: executed.** Full 254-tracker pipeline run at `a5b2a09` plus a full +`just compare-outputs` against the frozen `output_r/` established the session +baseline, and it reproduced ticket 60's reported deltas exactly +(`buddhist_era_typo` 12 → 0 across both patient stages, `python_absurd_excel_serial` +0 → 6 raw, `python_rejects_beyond_tracker_year` 36 → 42 cleaned) — confirming the +measurement environment before anything was concluded from it. All population +counts come from that run's Excel reports and JSON snapshots; all mechanisms +from opening the source workbooks (and, for the line endings, the raw sheet XML) +and from reading R's own source. A second full comparison after the change +confirms **all four stages report "no change from the previous run" and zero +per-cause movement** — which is the proof that a docstring-only session moved +nothing. Full suite 938 passed / 1 skipped, ruff and `ty check src/` clean. + +**Tense.** Every count describes current behaviour, measured this session. The +docstring claims quoted as wrong ("large majority of patients", "case or +whitespace", "Python's value is the faithful one") describe what the registry +*said* before this session, not what the pipeline does. + +**What this does to [ticket 12](12-retire-r-workspace.md): it does not unblock +it.** Re-deriving the list of open tickets that still need the R *source*, per +the map's Notes, leaves exactly one: [Monthly rows with a misspelled ID silently +lose their Patient List demographics](58-patient-list-join-uses-unfixed-id.md), +whose question 3 is literally "Check R — R's own join ordering may or may not +have the same gap". Tickets 6, 9, 16, 34, 35, 40 and 41 were each checked and +need no R. So `blocked_by` moves from `[62]` to `[58]`, and the count of tickets +standing between this map and retiring R stays at one. diff --git a/docs/wayfinder/tickets/63-name-the-cleaned-stage-static-join-divergence.md b/docs/wayfinder/tickets/63-name-the-cleaned-stage-static-join-divergence.md new file mode 100644 index 0000000..b3ae769 --- /dev/null +++ b/docs/wayfinder/tickets/63-name-the-cleaned-stage-static-join-divergence.md @@ -0,0 +1,166 @@ +--- +id: 63 +title: The cleaned stage has 4,949 cells with no cause, because the ID spelling that explains them is gone by then +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-24e +claimed_at: 2026-08-24T22:00:00+02:00 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 58 +--- + +## Premise + +Rests entirely on [Monthly rows with a misspelled ID silently lose their +Patient List demographics](58-patient-list-join-uses-unfixed-id.md), closed, +which made both whole-tracker joins key on a normalized `patient_id` and so +recovered 680 rows / 6,863 Patient List cells that R still publishes as null. + +That decision also established the constraint this ticket exists inside: the +new cause `r_static_join_misses_respelled_id` can only fire on the **raw** +stage, because its discriminator is the month row's own ID still carrying the +hyphen or transfer-clinic suffix -- and cleaning normalizes exactly that, on +both sides. Raw-stage `unclassified` is now **0**; cleaned-stage went **16 -> +3,593**. + +Also rests on the map's standing bar (**triage means deciding, not +labelling**): a cause carrying the right label over the wrong reason is what +[ticket 62](62-finish-the-pre-bar-classifier-audit.md) found three times over, +so widening `r_extraction_gap` to swallow these is explicitly not the answer. + +**What would void this ticket rather than rewrite it:** ticket 58's join fix +being reverted, since without it these 3,593 cells do not exist. + +## Question + +1. **Find a discriminator that survives cleaning.** Candidates: have the + comparison carry the raw spelling forward as a diagnostic flag on + `CellMismatch` (the shape `row_order_candidate` and + `group_endpoint_matches` already use); or derive the affected identities + once per file and match on those; or have cleaning record the + pre-normalization ID in a column. Each has a different reach -- the first + two touch only the comparison, the third changes production output. +2. **Bound the three columns the corpus-wide wiring over-claimed on.** Ticket + 58 measured that `t1d_diagnosis_age` (1,069 rows against a join-miss + population of 679), `bmi` (144) and `age` (1) also carry R-null/Python-present + cells for these patients from some *other* mechanism, because the month + sheets carry those columns too. They currently keep their pre-existing + causes. Measure what those other rows actually are and whether their + current cause states the right reason -- this is a ticket-32-shaped + question, not a wiring question. +3. **Check what absorbed the new cells before the bounded wiring landed.** + `recruitment_date` went 28,085 -> 28,762 in the first post-fix comparison + with no `unclassified` movement, which means an existing cause + (`r_extraction_gap`, whose documented mechanism is R's `xml:space` header + defect) silently took 677 cells that belong to the join miss. Confirm no + other column is still in that state on the cleaned side. + +## Standing bar + +Per the map's **triage means deciding, not labelling**: measure before +proposing, and a classifier is only written once the mechanism is traced. + +## Resolution + +**Decision.** The discriminator is derived per file from the run's own **raw** +output, comparison-side only (question 1's first candidate). `static_join_missed_ids` +(`src/a4d/migration/compare.py`) reads each raw parquet's `patient_id` column +and returns the *normalized* spelling of every ID that was written +non-canonically -- the last place the two spellings coexist. `compare_cells` +sets `CellMismatch.static_join_missed_id` from that set, and +`_is_r_static_join_misses_respelled_id` accepts it as an alternative to the raw +stage's own per-cell test. Production output is untouched. + +**The ticket's headline number was understated.** It said cleaned +`unclassified` went 16 -> 3,593. Measured across the four snapshots of +2026-08-24: it is **16 -> 4,949**, stable over five runs. Raw is 0. Ticket 58's +framing survives; only its size was wrong, and the title is corrected. + +**Two guards had to be added before the cause was true, and both were found by +measuring rather than by reasoning.** + +1. **A column R lost across the whole file is a different mechanism.** Wired + naively, the prepended cause claimed 9,302 cells across **43 files** -- far + beyond the 6 files ticket 58 recovered. 2024 Preah Kossamak reads **zero** + `recruitment_date` in 863 rows and 2024 Yangon General zero in 1,057, which + is `r_extraction_gap`'s file-level `xml:space` header defect (ticket 62); + 2024 Mahosot reads 672 of 1,068, so *its* nulls really are per-patient. New + `CellMismatch.r_column_empty_in_file`, measured on R's own frame, makes the + cause decline the first case. It removed 2,336 cells whose reason is R's + header, not R's key -- the exact ticket-32 failure mode of a right label + over a wrong mechanism. The raw stage never needed it: a column R never + mapped is absent there and is reported as column divergence. +2. **`age` is derived, so it needed its own cause, not this one.** New + `r_age_not_derived_without_dob`, wired to `age` alone and keyed on new + `CellMismatch.row_dob_recovered` (this row's `dob` null on R, present on + Python) rather than on the file-level set -- the birth date is what the + derivation actually consumes. + +**Question 2 is answered, and two of its three columns needed nothing.** +- `age`: **113 cells**, all on join-miss identities, in 3 files. 53 differ + numerically (-2, -1, +1, +4) and 60 are R-null. One mechanism for both: + `_fix_age_from_dob` overrides the sheet's Age cell with the D.O.B.-derived + age; R's `dob` is null for **all 275 rows** of the affected patients in its + frozen 2023 Mahosot output, so R keeps the clinic's typed figure or nothing. + LA_MH060's recovered D.O.B. is 2009-01-16, so `Jan23` is 14 -- Python's + value; R publishes 13, the clinic's own, updated a month late. **Python is + the correct side.** +- `bmi`: **0 cells** on join-miss identities in the current cleaned + comparison. Ticket 58's measured 144 does not survive; nothing to bound. +- `t1d_diagnosis_age`: 1,078 cells on these identities, on + `r_never_derives_diagnosis_age` (683), `r_numeric_error_sentinel` (391) and + `source_date_in_diagnosis_age` (4). The reason is **right but was + incomplete**: these are divergences ticket 58 *created* (2024 Mahosot's + LA_MH056 is null on `dob`, `t1d_diagnosis_date` and `t1d_diagnosis_age` in R + and 2014-02-22 / 2021-06-22 / 7 in Python), yet R would publish nothing here + even holding both dates, because its `fix_t1d_diagnosis_age` call site is + commented out. Docstring corrected; **no count moved** -- ticket 62's shape. + +**Question 3 is answered, and more was absorbed than the ticket suspected.** +It flagged `recruitment_date` (677). Measured, four causes were silently +holding join-miss cells on the cleaned side, and the guard-corrected +reassignment is: `recruitment_date|r_extraction_gap` 28,686 -> 28,015 (671), +`fbg_baseline_mg|r_join_suffix_collision` 9,521 -> 8,844 (677), +`edu_occ|r_extraction_gap` 283 -> 59 (224), `edu_occ_updated|r_extraction_gap` +2,993 -> 2,769 (224), and `blood_pressure_{sys,dias,updated}|r_extraction_gap` +21 -> 12 (9 each). Every one of those now matches the **raw** stage's count for +the same column cell-for-cell (671, 677, 223, 223, 8), which is the strongest +available evidence the reassignment is right: two independently-derived +discriminators agreeing. + +**Rejected.** +- *Publishing the pre-normalization ID as a cleaned column* -- the most direct + discriminator, but it changes the 83-column schema that matches R's output, + a production change serving a migration-only tool due to be deleted with R. +- *Emitting an error record when the pipeline normalizes a join key* -- cleaner + provenance, but it adds a production error record with no consumer after + retirement and needs a full 254-tracker re-run before the comparison sees it. +- *Widening `r_extraction_gap`* -- explicitly ruled out by the premise, and the + measurement above shows why: it was already absorbing 4,482 cells under the + wrong reason. +- *Excluding an ambiguous identity from the set.* 2026 Surat Thani spells the + same patient both `TH_ST029` and `TH-ST029` in `May26`; after normalization + the two rows are indistinguishable, so 5 cells (1 row x 5 columns) are + claimed that the raw stage labels `r_extraction_gap`. Excluding the identity + would cost two correct classifications to avoid one wrong one, so it is named + in the docstring instead. + +**Evidence: executed.** Every count above is from a real 254-tracker comparison +against the frozen R baseline on the USB drive, not from reading code. The two +source workbooks (2026 Surat Thani's `Patient List` and `Annual`) were opened +directly; R's and Python's frozen parquets were queried for the D.O.B. and +diagnosis-date claims. Full suite 980 passed / 1 skipped; ruff clean; `ty` +unchanged at 44 pre-existing diagnostics (verified by stashing). + +**Tense.** Every count describes current behaviour after this session's change, +except the "before" figures, which describe the run at `2026-08-24T182929Z`. + +**Result: cleaned `unclassified` 4,949 -> 16, the pre-session baseline; raw +stays 0; both product stages unmoved; and the four-stage total is unchanged at +121,742 / 33,488 / 22,735 / 89 -- so nothing was invented, lost, or quietly +pushed into the unexplained bucket.** The 16 remaining are the pre-existing +cells owned by fog patches (11 `insulin_subtype`, 2 `fbg_updated_mmol`, 2 +`remote_followup`, 1 `bmi_date`). diff --git a/docs/wayfinder/tickets/64-documentation-overhaul-drop-r-framing.md b/docs/wayfinder/tickets/64-documentation-overhaul-drop-r-framing.md new file mode 100644 index 0000000..97362d4 --- /dev/null +++ b/docs/wayfinder/tickets/64-documentation-overhaul-drop-r-framing.md @@ -0,0 +1,174 @@ +--- +id: 64 +title: Rewrite every docstring and doc that explains the code by what R did +labels: [wayfinder:task] +status: closed +blocked_by: [] +assignee: session-2026-08-24g +claimed_at: 2026-08-24 +resolution: decided +evidence: executed +closed_by: null +spawned_by: 12 +--- + +## Premise + +[Retire R from the workspace once the pipeline is fully verified +Python-only](12-retire-r-workspace.md) is closed: `r-archive/` is deleted, and +the `r-archive-removed` tag is the only way to reach it. That is what makes +this ticket possible and necessary at once -- possible, because R is no longer +a live reference anyone is expected to consult; necessary, because the code +still explains itself by pointing at a directory that no longer exists. + +Ticket 12 measured the size of it: **35 citations of R source files across 12 +Python modules**, not the 8 in `src/a4d/migration/compare.py` its own inventory +claimed. The spread is what matters -- `compare.py` holds 17, but +`clean/patient.py` holds 10, and `extract/patient.py`, `clean/date_parser.py`, +`clean/schema.py`, `clean/schema_product.py`, `clean/product.py`, +`clean/validators.py` and `tables/product.py` carry the rest. Two test files +cite R as well. So this reaches production cleaning code, not just migration +tooling. + +Rests also on the map's **standing triage bar** (Notes, set 2026-08-12g and +refined 2026-08-13): a difference is only understood when its *mechanism* is +traced to source. Fifty-four closed tickets did exactly that work, and the +mechanisms they established -- with real files, real cells, real counts -- are +the material this ticket writes the new documentation from. The evidence for +almost every rule in the pipeline already exists in the closed tickets and in +[MR_DESCRIPTION.md](../../archive/MR_DESCRIPTION.md); this ticket moves it to +where a reader of the code will actually find it. + +Would be void, not merely rewritten, if `r-archive/` were restored to the +working tree as a live reference. + +## Question + +The user's framing (2026-08-24): documentation that justifies a behaviour by +"to match R" was fine as *working* documentation during the migration, and is +now simply **wrong** -- it explains the code by something that no longer +exists, and it says nothing about why the code is the way it is. Comments and +docs should describe the current state and the real reason for a design, with +concrete examples and data. + +Decide and execute: + +1. **What replaces an R citation.** A docstring reading "R's `fix_fbg` matches + its category words as substrings, so Python anchors them" has to become a + statement about Python alone that still carries the reason. The candidate + shape is: what the rule does, the source evidence that made it necessary + (the tracker, the sheet, the cell, the count), and what breaks without it. + Confirm that shape against a handful of the hardest sites before applying it + to 35. + +2. **Which R references are history rather than explanation.** The cause + registry in `compare.py` documents R/Python *divergences*, and the + comparison tool's whole purpose was comparing against R. Its classifier + docstrings may legitimately name R, since R is their subject -- but decide + that explicitly rather than by default, and decide what happens to the tool + itself once the migration closes (see the fog entry on formally retiring R). + +3. **How far this reaches beyond docstrings.** `docs/archive/` holds + `MIGRATION_GUIDE.md`, `PYTHON_IMPROVEMENTS.md`, + `PRODUCT_DATA_PIPELINE_FEATURE.md` and `MR_DESCRIPTION.md` -- all + R-framed by construction. Some are genuinely historical records of the + migration and should stay that way; some are read as current guidance and + should not. Sort them, and decide whether `docs/CLAUDE.md`'s "Migration + Status" section survives at all once the migration is done. + +4. **Guard against regression.** Nothing currently stops a new "matches R + behaviour" docstring being written. Decide whether a check belongs in the + lint/CI set that [ticket 34](34-local-ci-parity-guard.md) is aligning, or + whether the instruction in `CLAUDE.md` is enough. + +Note the precedent from [round 4](51-triage-patient-cleaned-residual-4.md): +`_validate_dates` carried a docstring claiming its future-date guard "matches R +pipeline behavior" when R has no such guard at all. A false R citation is not +only stale, it can be **wrong about R** -- so this is a rewrite against measured +behaviour, not a find-and-replace. + +## Resolution (session-2026-08-24g) + +**Decision.** Every module outside `src/a4d/migration/` now explains itself in +its own terms. **353 R references in `src/` and 88 in `tests/` are down to the +179 + 42 inside the comparison package**, which is exempt by decision (below). +Outside it: `src/` 174 -> **0**, `tests/` 46 -> **0**, user-facing markdown 65 +-> **0**. + +The replacement shape, confirmed against the hardest sites before being applied +broadly: *what the rule does, the source evidence that forced it (tracker, +sheet, cell, count), and what breaks without it.* Examples -- +`transform_cm_to_m`'s threshold stopped being "matching R" and became "120 +cells reading 2.43/6.9/13.0 were published as 0.069 metres"; the whitespace +strip stopped citing `readxl`'s `trim_ws` and now names Kantha Bopha 2019 +KH_KB023's `'F '` cell and the 72 rows of `sex` it cost; the optional closing +parenthesis stopped citing R's regex and now says 25 of 30 source cells are +written `180(May-2017`. + +**`src/a4d/migration/` is exempt, and marked historical rather than merely +excused** (option 2 of three, user's choice). Its module docstring now opens by +saying the package is a snapshot of finished work, that no new R output will +ever be produced so the tool will not run again, that it is kept because its +cause registry is the written evidence behind the migration's conclusions, and +that the R source is reachable at `git show r-archive-removed^:...`. The script +itself is unchanged. + +**`docs/migration/` moved to `docs/archive/`** (user's instruction, mid-session) +and gained a `README.md` index: what each document is, what superseded it, and +where to look instead for current guidance. Every document carries a banner and +was brought to the true current state first rather than being frozen while +wrong -- `MIGRATION_GUIDE.md`'s "Phases 0-9" status became the real final state, +`MR_DESCRIPTION.md`'s "240 commits / 664 tests" became 285 commits / 981 tests, +and `PYTHON_IMPROVEMENTS.md` now says in its own header that ticket 7 found it +undercounting, and points at the two derived inventories that replaced it. + +**`docs/VALIDATION_SUMMARY.md` was deleted**, not banner-ed: it is an +R-vs-Python verdict over **174** trackers, superseded by the 254-tracker +comparison, so it was both R-framed and factually stale. Nothing referenced it. + +**A regression guard was added** (question 4), as +`tests/test_docs_have_no_r_framing.py` rather than a new lint hook -- it runs in +CI already and needs no coordination with [ticket +34](34-local-ci-parity-guard.md). It derives its file list from the tree rather +than hardcoding one, exempts the migration package and itself, and assembles +its own patterns from parts so it does not match its own source. Verified by +deliberately introducing a violation and watching it fail with the file and +line, then restoring. + +**Because.** A citation of a deleted directory is not a weaker explanation than +one of measured behaviour -- it is not an explanation at all, and the map's own +standing triage bar already required the mechanism to be traced to source. The +evidence existed in 56 closed tickets; this moves it to where a reader of the +code will find it. + +**Rejected.** +- *Retire `compare.py` with R.* Would delete the audit trail behind every cause + the map decided, and the frozen `output_r/` baseline still exists. +- *Exempt `compare.py` silently, repointing only its path citations.* Leaves a + reader unable to tell whether the module is current guidance -- the exact + defect being fixed everywhere else. +- *Rewrite the `docs/archive/` documents rather than freezing them.* They are + records **of** the migration and R-framed by construction; rewriting them + would destroy the thing they are. +- *Find-and-replace.* Ruled out by [round + 4](51-triage-patient-cleaned-residual-4.md)'s precedent: `_validate_dates` + claimed its future-date guard "matches R pipeline behavior" when R had no + such guard, so a stale citation can be wrong about R and each site needed + reading. + +**Deliberately not changed: two published log values that name R.** +`src/a4d/extract/product.py` emits `function_name="read_product_data_step1"`, +the only one of twenty `function_name` values that is not the emitting Python +function, and three sites emit `script="script1"`/`"script3"`. These are **data +in the BigQuery `logs` table**, not comments, so changing them changes published +output and could break a consumer. They are a real inconsistency and they are +the residue of this ticket -- recorded rather than silently fixed, for the user +to decide. + +**Evidence.** Executed. Population measured by grep before any edit (353/88/65, +by file), re-measured after. Full suite **1,080 passed, 1 skipped, 86% +coverage** -- 980 before, plus the guard's 100 parametrised cases; no existing +test changed behaviour. `ruff check`, `ruff format --check` and `ty check src/` +all clean. The guard was proven non-vacuous by introducing a violation. + +**Tense.** All of the above is current state, committed. diff --git a/docs/wayfinder/tickets/65-logs-table-r-named-values.md b/docs/wayfinder/tickets/65-logs-table-r-named-values.md new file mode 100644 index 0000000..e9b54ce --- /dev/null +++ b/docs/wayfinder/tickets/65-logs-table-r-named-values.md @@ -0,0 +1,70 @@ +--- +id: 65 +title: Two values published into the logs table still name R scripts +labels: [wayfinder:grilling] +status: open +blocked_by: [] +assignee: null +claimed_at: null +resolution: null +evidence: null +closed_by: null +spawned_by: 64 +--- + +## Premise + +[Rewrite every docstring and doc that explains the code by what R +did](64-documentation-overhaul-drop-r-framing.md) is closed: every comment and +docstring outside `src/a4d/migration/` now explains the pipeline in its own +terms, and a CI guard stops new ones appearing. + +That ticket deliberately stopped at the boundary between *documentation* and +*published data*, and this is what it found on the far side. These are not +comments -- they are field values written into the BigQuery `logs` table on +every run, so changing them changes output a consumer may depend on. Ticket 64 +recorded them rather than fixing them silently. + +Rests also on [retiring R](12-retire-r-workspace.md): `r-archive/` is deleted, +so `script1` and `read_product_data_step1` name files that are no longer in the +repo. + +Void, rather than merely rewritten, if the `logs` table is dropped or +restructured for another reason first. + +## Question + +Two populations, and they may deserve different answers. + +1. **`function_name="read_product_data_step1"`** + (`src/a4d/extract/product.py`, emitted by `_count_orphan_released_units`). + Every one of the other twenty `function_name` values in the codebase is the + Python function that emitted the record -- `cut_numeric_value`, + `_validate_dates`, `read_all_patient_sheets`. This one names an R script + instead, so the field means something different in this one row type. Note + `check_entry_dates` in `clean/product.py` is a milder version of the same + thing: descriptive, but not the emitting function's name either. + +2. **`script="script1"` / `script="script3"`** (three sites in + `extract/product.py`, one in `tables/product.py`). The `script` field + defaults to `"clean"` -- a stage name -- so the numbered values are a second + vocabulary inside one column, and the numbers refer to R scripts. + +What has to be decided: + +- **Is anything actually consuming these?** Measure before deciding: query the + `logs` table for the distinct `script` and `function_name` values and their + row counts, and establish whether any dashboard, saved query or downstream + job filters on them. The answer probably decides the rest. +- **If nothing consumes them**, the cheap fix is to make both fields honest: + `function_name` becomes the emitting Python function, `script` becomes the + stage (`extract` / `clean` / `tables`) consistently. That is a one-line + change per site plus a test assertion. +- **If something does consume them**, decide whether the rename is worth a + coordinated change, or whether these values are simply frozen vocabulary that + should be documented as such where the field is defined (`errors.py`). +- **Whether this should land before [promoting `migration` into + `dev`](06-promote-migration-to-dev.md)** or after. It is deliberately not + wired as a blocker: it changes published data rather than correctness, and + the promotion gate was argued on documentation being wrong, which this no + longer is. diff --git a/justfile b/justfile new file mode 100644 index 0000000..78d4eef --- /dev/null +++ b/justfile @@ -0,0 +1,230 @@ +# a4d Python Pipeline - Development Commands + +# Default recipe (show available commands) +default: + @just --list + +PROJECT := "a4dphase2" +DATASET := "tracker" +REGISTRY := "asia-southeast2-docker.pkg.dev/a4dphase2/a4d/pipeline" +IMAGE := REGISTRY + ":latest" + +# ── Environment ─────────────────────────────────────────────────────────────── + +# Install dependencies and sync environment +sync: + uv sync --all-extras + +# Update dependencies +update: + uv lock --upgrade + +# Show project info +info: + @echo "Python version:" + @uv run python --version + @echo "" + @echo "Installed packages:" + @uv pip list + +# Clean cache and build artifacts +clean: + rm -rf .ruff_cache .pytest_cache htmlcov .coverage dist build src/*.egg-info + find . -type d -name __pycache__ -exec rm -rf {} + + find . -type f -name "*.pyc" -delete + +# ── Code Quality ────────────────────────────────────────────────────────────── + +# Format code with ruff +format: + uv run ruff format . + +# Check code formatting without modifying files +format-check: + uv run ruff format --check . + +# Auto-fix linting issues +fix: + uv run ruff check --fix . + +# Run ruff linting +lint: + uv run ruff check . + +# Run type checking with ty +check: + uv run ty check src/ + +# Run all CI checks (format, lint, type, test) +ci: format-check lint check test + +# ── Testing ─────────────────────────────────────────────────────────────────── + +# Run unit tests (skip slow/integration) +test: + uv run pytest -m "not slow" + +# Run tests without coverage (faster, fail fast) +test-fast: + uv run pytest -m "not slow" --no-cov -x + +# Run all tests including slow/integration +test-all: + uv run pytest + +# Run integration tests only +test-integration: + uv run pytest -m integration + +# Install pre-commit hooks +hooks: + uv run pre-commit install + +# Run pre-commit on all files +hooks-run: + uv run pre-commit run --all-files + +# ── Local Pipeline ──────────────────────────────────────────────────────────── + +# Process a single patient tracker file (no GCS) +run-file FILE: + uv run a4d run patient --file "{{FILE}}" + +# Process a single product tracker file (no GCS); ad-hoc/debug only — use `just run` for full runs +run-file-product FILE: + uv run a4d run product --file "{{FILE}}" + +# Process local patient files only, no GCS (paths with spaces: use --file recipes instead) +run-local *ARGS: + uv run a4d run patient {{ARGS}} + +# Process local product files only, no GCS; ad-hoc/debug only — use `just run` for full runs +run-local-product *ARGS: + uv run a4d run product {{ARGS}} + +# Rebuild patient+product+clinic+logs tables from existing cleaned output +create-tables *ARGS: + uv run a4d create tables {{ARGS}} + +# Download from GCS, process locally, no upload +run-download *ARGS: + uv run a4d run --skip-upload {{ARGS}} + +# Full pipeline: download from GCS, process, upload to GCS + BigQuery +run *ARGS: + uv run a4d run {{ARGS}} + +# Diff a Python output directory against the frozen R baseline (migration-only, ticket 15) +# Named params (not *ARGS) so paths with spaces (e.g. the USB drive) survive just's interpolation +compare-outputs r_dir py_dir output_dir="output/comparison" *ARGS: + uv run python scripts/compare_outputs.py \ + --r-dir "{{r_dir}}" --py-dir "{{py_dir}}" --output-dir "{{output_dir}}" {{ARGS}} + +# ── Docker ──────────────────────────────────────────────────────────────────── + +# --provenance=false: suppress BuildKit attestation manifests so the registry +# shows one image entry instead of three (image + attestation + index) +# Build Docker image tagged as :latest and : +docker-build: + #!/usr/bin/env bash + set -euo pipefail + GIT_SHA=$(git rev-parse --short HEAD) + docker build --provenance=false --platform=linux/amd64 \ + -t {{IMAGE}} \ + -t {{REGISTRY}}:${GIT_SHA} \ + -f Dockerfile . + +# Smoke test: verify the image starts and the CLI is reachable +docker-smoke: + docker run --rm {{IMAGE}} uv run a4d --help + +# Push both :latest and : tags to Artifact Registry +docker-push: docker-build + #!/usr/bin/env bash + set -euo pipefail + GIT_SHA=$(git rev-parse --short HEAD) + docker push {{IMAGE}} + docker push {{REGISTRY}}:${GIT_SHA} + echo "Pushed: {{IMAGE}} and {{REGISTRY}}:${GIT_SHA}" + +# Delete all images from Artifact Registry except :latest +docker-clean: + #!/usr/bin/env bash + set -euo pipefail + LATEST=$(gcloud artifacts docker images describe {{IMAGE}} \ + --project={{PROJECT}} --format="value(image_summary.digest)") + echo "Keeping: {{IMAGE}} ($LATEST)" + gcloud artifacts docker images list {{REGISTRY}} \ + --include-tags --project={{PROJECT}} \ + --format="value(digest)" \ + | while read -r digest; do + if [ "$digest" != "$LATEST" ]; then + echo "Deleting $digest..." + gcloud artifacts docker images delete "{{REGISTRY}}@$digest" \ + --project={{PROJECT}} --quiet --delete-tags 2>/dev/null || true + fi + done + echo "Done." + +# List images in Artifact Registry with tags and digests +docker-list: + gcloud artifacts docker images list {{REGISTRY}} \ + --include-tags \ + --project={{PROJECT}} + +# ── GCP / Cloud Run ─────────────────────────────────────────────────────────── + +# Creates dated snapshots e.g. patient_data_static_20260227 with 7-day expiry. +# Snapshot all BigQuery pipeline tables (safe to run before deploy) +backup-bq: + #!/usr/bin/env bash + set -euo pipefail + DATE=$(date +%Y%m%d) + EXPIRY="TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 7 DAY)" + # Every table load_pipeline_tables deletes and recreates on each run. + # Keep in sync with PARQUET_TO_TABLE in src/a4d/gcp/bigquery.py when adding new pipelines. + TABLES="patient_data_static patient_data_monthly patient_data_annual product_data clinic_data_static logs errors tracker_metadata" + for TABLE in $TABLES; do + if bq show --quiet {{PROJECT}}:{{DATASET}}.${TABLE} 2>/dev/null; then + SNAP="${TABLE}_${DATE}" + echo "Snapshotting ${TABLE} -> ${SNAP}..." + bq query --use_legacy_sql=false --project_id={{PROJECT}} \ + "CREATE SNAPSHOT TABLE \`{{PROJECT}}.{{DATASET}}.${SNAP}\` + CLONE \`{{PROJECT}}.{{DATASET}}.${TABLE}\` + OPTIONS(expiration_timestamp = ${EXPIRY})" + else + echo "Skipping ${TABLE} (does not exist yet)" + fi + done + echo "Done. Snapshots expire in 7 days." + +# Build, push and update the Cloud Run Job to use the latest image +deploy: docker-push + gcloud run jobs update a4d-pipeline \ + --image={{IMAGE}} \ + --region=asia-southeast2 + +# Execute the Cloud Run Job +run-job: + gcloud run jobs execute a4d-pipeline --region=asia-southeast2 + +# Stream logs from the Cloud Run Job (Ctrl-C to stop) +logs-job: + gcloud beta logging tail 'resource.type="cloud_run_job" AND resource.labels.job_name="a4d-pipeline"' \ + --project={{PROJECT}} \ + --format="value(textPayload)" + +# Show current resource settings (CPU, memory, timeout, parallelism) for the Cloud Run Job +job-settings: + gcloud run jobs describe a4d-pipeline \ + --region=asia-southeast2 \ + --project={{PROJECT}} \ + --format="yaml(spec.template.spec.template.spec.containers[0].resources, spec.template.spec.template.spec.timeoutSeconds, spec.template.spec.parallelism, spec.template.spec.taskCount)" + +# Roll back Cloud Run Job to a specific git SHA +# Usage: just rollback abc1234 +rollback SHA: + gcloud run jobs update a4d-pipeline \ + --image={{REGISTRY}}:{{SHA}} \ + --region=asia-southeast2 + @echo "Rolled back to {{REGISTRY}}:{{SHA}}" diff --git a/man/a4d-package.Rd b/man/a4d-package.Rd deleted file mode 100644 index 6121349..0000000 --- a/man/a4d-package.Rd +++ /dev/null @@ -1,15 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/a4d-package.R -\docType{package} -\name{a4d-package} -\alias{a4d} -\alias{a4d-package} -\title{a4d: Process tracker data for the DataCross A4D project} -\description{ -Functionality to process, clean and ingest tracker data (excel files). -} -\author{ -\strong{Maintainer}: Michael Aydinbas \email{michael.aydinbas@gmail.com} - -} -\keyword{internal} diff --git a/man/add_product_categories.Rd b/man/add_product_categories.Rd deleted file mode 100644 index 396e0dd..0000000 --- a/man/add_product_categories.Rd +++ /dev/null @@ -1,22 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_product_data.R -\name{add_product_categories} -\alias{add_product_categories} -\title{Inserts product categories if contained in a provided mapping} -\usage{ -add_product_categories(inventory_data, product_category_mapping) -} -\arguments{ -\item{inventory_data}{A product inventory dataframe containing a "product" column} - -\item{product_category_mapping}{A dataframe mapping "product" to "category"} -} -\value{ -A dataframe containing the inventory data with an additional product -category column. -} -\description{ -Product categories are added in the sense of a left join, -meaning missing categories will result in NA values while products are never -dropped. -} diff --git a/man/adjust_column_classes.Rd b/man/adjust_column_classes.Rd deleted file mode 100644 index 00b49ac..0000000 --- a/man/adjust_column_classes.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{adjust_column_classes} -\alias{adjust_column_classes} -\title{adjust_column_classes} -\usage{ -adjust_column_classes(product_df) -} -\description{ -Adjust classes of dataframe names -} diff --git a/man/calculate_most_frequent.Rd b/man/calculate_most_frequent.Rd deleted file mode 100644 index 9525c7e..0000000 --- a/man/calculate_most_frequent.Rd +++ /dev/null @@ -1,31 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script3_create_table_product_data.R -\name{calculate_most_frequent} -\alias{calculate_most_frequent} -\title{Calculate the Most Frequent Value in a Data Frame} -\usage{ -calculate_most_frequent(df, group_column, value_column, new_column) -} -\arguments{ -\item{df}{A data frame.} - -\item{group_column}{The name of the column to group by.} - -\item{value_column}{The name of the column to calculate the most frequent value from.} - -\item{new_column}{The name of the new column to add to the data frame.} -} -\value{ -A data frame with an additional column containing the most frequent value of the specified column for each group. -} -\description{ -This function calculates the most frequent value of a specified column for each group in a data frame. -It then adds a new column to the data frame with these most frequent values. -} -\examples{ -\dontrun{ -library(dplyr) -data(mtcars) -result <- calculate_most_frequent(mtcars, "cyl", "gear", "most_frequent_gear") -} -} diff --git a/man/check_allowed_values.Rd b/man/check_allowed_values.Rd deleted file mode 100644 index e857ce2..0000000 --- a/man/check_allowed_values.Rd +++ /dev/null @@ -1,38 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{check_allowed_values} -\alias{check_allowed_values} -\title{Check value against list of allowed values.} -\usage{ -check_allowed_values( - x, - valid_values, - id, - replace_invalid = TRUE, - error_val = "Undefined", - col = "" -) -} -\arguments{ -\item{x}{character value to check.} - -\item{valid_values}{list of valid options.} - -\item{id}{patient id.} - -\item{replace_invalid}{A flag controlling whether to also replace invalid .} - -\item{error_val}{value that replaces invalid values.} - -\item{col}{column name when used with mutate/across.} -} -\value{ -Either x or "Other". -} -\description{ -Invalid values are logged and depending on the replace invalid flag -also replaced with an error value. When checking for the validity of values -the case of the string is not considered nor are spaces and non-alphanumeric -characters. If the value matches an entry in the allowed_values list -that match is returned as the result. -} diff --git a/man/check_entry_dates.Rd b/man/check_entry_dates.Rd deleted file mode 100644 index 7cad7eb..0000000 --- a/man/check_entry_dates.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_product_data.R -\name{check_entry_dates} -\alias{check_entry_dates} -\title{Check entry dates in product data} -\usage{ -check_entry_dates(df, Sheet) -} -\arguments{ -\item{df}{A dataframe containing product data.} - -\item{Sheet}{The name of the sheet where the product data is located.} -} -\value{ -This function does not return a value. It logs a warning message if there are any dates in 'product_entry_date' that don't match the month/year on the sheet. -} -\description{ -This function checks if the entry dates for the balance match the month/year on the sheet. If any discrepancies are found, it logs a warning message with the number of mismatched dates and their corresponding 'product_entry_date' values. -} diff --git a/man/check_negative_balance.Rd b/man/check_negative_balance.Rd deleted file mode 100644 index ec4018c..0000000 --- a/man/check_negative_balance.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_product_data.R -\name{check_negative_balance} -\alias{check_negative_balance} -\title{Check negative values in 'product_balance' column} -\usage{ -check_negative_balance(df, Sheet) -} -\arguments{ -\item{df}{A dataframe containing product data.} - -\item{Sheet}{The name of the sheet where the product data is located.} -} -\value{ -This function does not return a value. It logs a warning message if there are any negative values in the 'product_balance' column. -} -\description{ -This function checks for negative values in the 'product_balance' column of a given dataframe. If any negative values are found, it logs a warning message with the number of negative values and their corresponding 'product_balance' values. -} diff --git a/man/check_patterns_in_column.Rd b/man/check_patterns_in_column.Rd deleted file mode 100644 index 08c432c..0000000 --- a/man/check_patterns_in_column.Rd +++ /dev/null @@ -1,24 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/wide_format_2_long_format.R -\name{check_patterns_in_column} -\alias{check_patterns_in_column} -\title{Check for Specific Patterns in a Dataframe Column} -\usage{ -check_patterns_in_column(df, column_name) -} -\arguments{ -\item{df}{A dataframe to check for the specific patterns.} - -\item{column_name}{The name of the column to check for the patterns.} -} -\value{ -A logical value indicating whether any cell in the specified column contains the patterns. -} -\description{ -This function checks if any cell in a specific column of a dataframe contains a pattern with a hyphen followed by a digit, or a comma (which could be leading with a space). It returns TRUE if any cell in the column contains the pattern, and FALSE otherwise. -} -\examples{ -data(mtcars) -check_patterns_in_column(mtcars, "mpg") - -} diff --git a/man/check_wide_format_columns.Rd b/man/check_wide_format_columns.Rd deleted file mode 100644 index 94b703d..0000000 --- a/man/check_wide_format_columns.Rd +++ /dev/null @@ -1,23 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/wide_format_2_long_format.R -\name{check_wide_format_columns} -\alias{check_wide_format_columns} -\title{Check if dataframe contains required columns} -\usage{ -check_wide_format_columns(df) -} -\arguments{ -\item{df}{A dataframe to check for the presence of required columns.} -} -\value{ -A logical value indicating whether the dataframe contains the required columns. -} -\description{ -This function checks if a dataframe contains the columns 'Total Units Released' and 'Units Released per person'. -(for detection the Mandalay Children's Hospital 2020-2021 wide-format) -It returns TRUE if both columns are present, and FALSE otherwise. -} -\examples{ -df <- data.frame("Total Units Released" = c(1, 2, 3), "Units Released per person" = c(4, 5, 6), "Other Column" = c(7, 8, 9)) -check_wide_format_columns(df) # Returns: TRUE -} diff --git a/man/clean_receivedfrom.Rd b/man/clean_receivedfrom.Rd deleted file mode 100644 index c7c365f..0000000 --- a/man/clean_receivedfrom.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{clean_receivedfrom} -\alias{clean_receivedfrom} -\title{clean_receivedfrom} -\usage{ -clean_receivedfrom(product_df) -} -\description{ -Clean column "product_received_from" from character values -} diff --git a/man/clean_unitsreceived.Rd b/man/clean_unitsreceived.Rd deleted file mode 100644 index fe0c6d0..0000000 --- a/man/clean_unitsreceived.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{clean_unitsreceived} -\alias{clean_unitsreceived} -\title{clean_unitsreceived} -\usage{ -clean_unitsreceived(product_df) -} -\description{ -Clean column "product_units_received" from character values and rows with values but no further information -} diff --git a/man/compare_lists.Rd b/man/compare_lists.Rd deleted file mode 100644 index ef6529e..0000000 --- a/man/compare_lists.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_product_data.R -\name{compare_lists} -\alias{compare_lists} -\title{Compare two lists and return unmatched strings} -\usage{ -compare_lists(list1, list2) -} -\arguments{ -\item{list1}{The first list of strings to be compared.} - -\item{list2}{The second list of strings to be compared.} -} -\value{ -A vector of strings that are present in 'list1' but not in 'list2'. -} -\description{ -This function compares two lists and returns the strings that are present in the first list but not in the second list. -} diff --git a/man/compute_balance.Rd b/man/compute_balance.Rd deleted file mode 100644 index ea67e99..0000000 --- a/man/compute_balance.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{compute_balance} -\alias{compute_balance} -\title{compute_balance} -\usage{ -compute_balance(product_df, year) -} -\description{ -Computes balance based on start balance value, units released, units returned, units received.- -} diff --git a/man/compute_balance_cleanrows.Rd b/man/compute_balance_cleanrows.Rd deleted file mode 100644 index 16d0b00..0000000 --- a/man/compute_balance_cleanrows.Rd +++ /dev/null @@ -1,13 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{compute_balance_cleanrows} -\alias{compute_balance_cleanrows} -\title{compute_balance_cleanrows} -\usage{ -compute_balance_cleanrows(product_df) -} -\description{ -Preparation function for compute_balance and compute_balance_status. - -Remove rows without any meaningful information regarding units (release, return, received) or entry status -} diff --git a/man/compute_balance_status.Rd b/man/compute_balance_status.Rd deleted file mode 100644 index 8d0b3df..0000000 --- a/man/compute_balance_status.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{compute_balance_status} -\alias{compute_balance_status} -\title{compute_balance_status} -\usage{ -compute_balance_status(product_df) -} -\description{ -Computate balance status (start vs. change vs. end) per product -} diff --git a/man/convert_to.Rd b/man/convert_to.Rd deleted file mode 100644 index 9468144..0000000 --- a/man/convert_to.Rd +++ /dev/null @@ -1,27 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{convert_to} -\alias{convert_to} -\title{Try to convert character x into a value of target type.} -\usage{ -convert_to(x, cast_fnc, error_val, col_name = "", id = "") -} -\arguments{ -\item{x}{character value.} - -\item{cast_fnc}{function used to convert character values into target type.} - -\item{error_val}{error value to use when cast_fnc fails.} - -\item{col_name}{column name if used with across. Optional, defaults to "".} - -\item{id}{patient id, optional, defaults to "".} -} -\value{ -converted value -} -\description{ -Use cast_fnc to cast x to target type. -If that cast fails, output error_val instead. -If x is already NA, output NA. -} diff --git a/man/correct_decimal_sign.Rd b/man/correct_decimal_sign.Rd deleted file mode 100644 index e83ba37..0000000 --- a/man/correct_decimal_sign.Rd +++ /dev/null @@ -1,17 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{correct_decimal_sign} -\alias{correct_decimal_sign} -\title{Correct decimal sign.} -\usage{ -correct_decimal_sign(x) -} -\arguments{ -\item{x}{character representation of a number.} -} -\value{ -character representation with a correct decimal sign. -} -\description{ -Correct decimal sign. -} diff --git a/man/count_na_rows.Rd b/man/count_na_rows.Rd deleted file mode 100644 index 5df2dfe..0000000 --- a/man/count_na_rows.Rd +++ /dev/null @@ -1,21 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_product_data.R -\name{count_na_rows} -\alias{count_na_rows} -\title{Count rows with missing patient's name next to the released units} -\usage{ -count_na_rows(df, units_released_col, released_to_col) -} -\arguments{ -\item{df}{A dataframe containing patient data.} - -\item{units_released_col}{The name of the column that contains information about the units released.} - -\item{released_to_col}{The name of the column that contains information about who the units were released to (patient's name).} -} -\value{ -The number of rows where the patient's name is missing next to the released units. -} -\description{ -This function checks if the patient's name is missing next to the released units in a given dataframe. It counts the number of such rows. -} diff --git a/man/create_new_rows.Rd b/man/create_new_rows.Rd deleted file mode 100644 index cc34201..0000000 --- a/man/create_new_rows.Rd +++ /dev/null @@ -1,33 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/wide_format_2_long_format.R -\name{create_new_rows} -\alias{create_new_rows} -\title{Create New Rows in a DataFrame} -\usage{ -create_new_rows(df) -} -\arguments{ -\item{df}{A dataframe containing the original data.} -} -\value{ -A modified dataframe with additional rows. -} -\description{ -This function creates new rows in a given dataframe based on specific conditions. -To convert wide-format area in the 2017 - 2020 data to long-format area. -} -\details{ -The function checks if 'Total Units Released' and 'Units Released per person' -are not empty and not equal. If this condition is met, it adds new rows to the dataframe -based on values in columns between 'Released To' and 'Units Released per person'. -If an error occurs during processing, the original dataframe is returned. -} -\examples{ -df <- data.frame( - "Date" = c("2024-03-01", "2024-03-02"), - "Total Units Released" = c(100, 200), - "Units Released per person" = c(10, 20), - "Released To (select from drop down list)" = c("A", "B") -) -new_df <- create_new_rows(df) -} diff --git a/man/create_table_clinic_static_data.Rd b/man/create_table_clinic_static_data.Rd deleted file mode 100644 index 5bcc8f7..0000000 --- a/man/create_table_clinic_static_data.Rd +++ /dev/null @@ -1,14 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script3_create_table_clinic_static_data.R -\name{create_table_clinic_static_data} -\alias{create_table_clinic_static_data} -\title{Create clinic static data table} -\usage{ -create_table_clinic_static_data(output_root) -} -\arguments{ -\item{output_root}{The root directory of the output folder.} -} -\description{ -This function reads the clinic data from an Excel file, fills missing values, and exports the data as a parquet file. -} diff --git a/man/create_table_longitudinal_data.Rd b/man/create_table_longitudinal_data.Rd deleted file mode 100644 index 649a696..0000000 --- a/man/create_table_longitudinal_data.Rd +++ /dev/null @@ -1,30 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in -% R/script3_create_table_patient_data_changes_only.R -\name{create_table_longitudinal_data} -\alias{create_table_longitudinal_data} -\title{Create CSV with longitudinal patient data for a single variable.} -\usage{ -create_table_longitudinal_data( - patient_data_files, - input_root, - output_root, - variable, - name -) -} -\arguments{ -\item{patient_data_files}{list of CSV files with cleaned patient data from step 2.} - -\item{input_root}{root directory of the input CSV files.} - -\item{output_root}{root directory of the output folder.} - -\item{variable}{name of the column that should be exported.} - -\item{name}{name used to create the export file name.} -} -\description{ -Read in all cleaned patient data CSV and create a single data.frame. -Group this data by id and take only the months when there is a change in the medical data. -} diff --git a/man/create_table_patient_data_annual.Rd b/man/create_table_patient_data_annual.Rd deleted file mode 100644 index 416a7c6..0000000 --- a/man/create_table_patient_data_annual.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script3_create_table_patient_data_annual.R -\name{create_table_patient_data_annual} -\alias{create_table_patient_data_annual} -\title{Create CSV with annual patient data} -\usage{ -create_table_patient_data_annual(patient_data_files, input_root, output_root) -} -\arguments{ -\item{patient_data_files}{list of CSV files with cleaned patient data from step 2.} - -\item{input_root}{root directory of the input CSV files.} - -\item{output_root}{root directory of the output folder.} -} -\description{ -Read in all cleaned patient data CSV and create a single data.frame. -Group this data by id and take the latest available data (latest year and month). -} diff --git a/man/create_table_patient_data_monthly.Rd b/man/create_table_patient_data_monthly.Rd deleted file mode 100644 index b2136f6..0000000 --- a/man/create_table_patient_data_monthly.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script3_create_table_patient_data.R -\name{create_table_patient_data_monthly} -\alias{create_table_patient_data_monthly} -\title{Create CSV with monthly patient data} -\usage{ -create_table_patient_data_monthly(patient_data_files, input_root, output_root) -} -\arguments{ -\item{patient_data_files}{list of CSV files with cleaned patient data from step 2.} - -\item{input_root}{root directory of the input CSV files.} - -\item{output_root}{root directory of the output folder.} -} -\description{ -Read in all cleaned patient data CSV and create a single data.frame. -Only take dynamic columns. -} diff --git a/man/create_table_patient_data_static.Rd b/man/create_table_patient_data_static.Rd deleted file mode 100644 index a626229..0000000 --- a/man/create_table_patient_data_static.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script3_create_table_patient_data_static.R -\name{create_table_patient_data_static} -\alias{create_table_patient_data_static} -\title{Create CSV with static patient data} -\usage{ -create_table_patient_data_static(patient_data_files, input_root, output_root) -} -\arguments{ -\item{patient_data_files}{list of CSV files with cleaned patient data from step 2.} - -\item{input_root}{root directory of the input CSV files.} - -\item{output_root}{root directory of the output folder.} -} -\description{ -Read in all cleaned patient data CSV and create a single data.frame. -Group this data by id and take the latest available data (latest year and month). -} diff --git a/man/create_table_product_data.Rd b/man/create_table_product_data.Rd deleted file mode 100644 index a11aa20..0000000 --- a/man/create_table_product_data.Rd +++ /dev/null @@ -1,28 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script3_create_table_product_data.R -\name{create_table_product_data} -\alias{create_table_product_data} -\title{Create a Single CSV for Table Product Data} -\usage{ -create_table_product_data(input_root, output_root) -} -\arguments{ -\item{input_root}{A string specifying the path to the directory containing the input .parquet files.} - -\item{output_root}{A string specifying the path to the directory where the output .parquet file should be written.} -} -\value{ -This function does not return a value. It writes the merged data to a new CSV file -(with reordered columns according to the list of fields) in the output_root directory. -} -\description{ -This function reads all parquet files in a specified directory, merges them into a single data frame, -and writes the merged data to a new file with reordered and processed columns according to the list of fields. -If there are any columns that are present in some files but not others, -this function will add those missing columns to the data frames where they are missing and fill them with NA. -} -\examples{ -\dontrun{ -create_table_product_data("path/to/input/directory", "path/to/output/directory") -} -} diff --git a/man/cut_numeric_value.Rd b/man/cut_numeric_value.Rd deleted file mode 100644 index 506089b..0000000 --- a/man/cut_numeric_value.Rd +++ /dev/null @@ -1,23 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{cut_numeric_value} -\alias{cut_numeric_value} -\title{If x is outside its allowed range, return error value.} -\usage{ -cut_numeric_value(x, min, max, col_name = "") -} -\arguments{ -\item{x}{numeric value to check.} - -\item{min}{lower bound.} - -\item{max}{upper bound.} - -\item{col_name}{column name if used with mutate/across.} -} -\value{ -either x or error value. -} -\description{ -If x is outside its allowed range, return error value. -} diff --git a/man/export_data_as_parquet.Rd b/man/export_data_as_parquet.Rd deleted file mode 100644 index 1d5c483..0000000 --- a/man/export_data_as_parquet.Rd +++ /dev/null @@ -1,30 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_main.R -\name{export_data_as_parquet} -\alias{export_data_as_parquet} -\title{Export data as parquet to a given destination.} -\usage{ -export_data_as_parquet(data, filename, output_root, suffix) -} -\arguments{ -\item{data}{Data frame to export as parquet file.} - -\item{filename}{Output file name.} - -\item{output_root}{Root output directory.} - -\item{suffix}{Suffix will be appended to the original file name (e.g. "patient_data").} -} -\description{ -Export data as parquet to a given destination. -} -\examples{ -\dontrun{ -export_data( - data = df_raw_product, - filename = tracker_name, - output_root = output_root, - suffix = "_product_data" -) -} -} diff --git a/man/extract_date_from_measurement.Rd b/man/extract_date_from_measurement.Rd deleted file mode 100644 index 13deedb..0000000 --- a/man/extract_date_from_measurement.Rd +++ /dev/null @@ -1,21 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{extract_date_from_measurement} -\alias{extract_date_from_measurement} -\title{Split a column value into value and date.} -\usage{ -extract_date_from_measurement(df, colname) -} -\arguments{ -\item{df}{data frame with patient data.} - -\item{colname}{column to split into value and date.} -} -\value{ -data frame with the original column and a new colname_date column. -} -\description{ -Assumes that the measurement entry is in the format "8.53 (28/8/2017)". -Uses regex to split this string into the value part and the date part, -for example in "8.53 " and "28/8/2017". -} diff --git a/man/extract_first_raw_regimen.Rd b/man/extract_first_raw_regimen.Rd deleted file mode 100644 index 450fa65..0000000 --- a/man/extract_first_raw_regimen.Rd +++ /dev/null @@ -1,17 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{extract_first_raw_regimen} -\alias{extract_first_raw_regimen} -\title{Extract the part of a string before a comma} -\usage{ -extract_first_raw_regimen(string) -} -\arguments{ -\item{string}{The input string} -} -\value{ -The extracted part. -} -\description{ -Extract the part of a string before a comma -} diff --git a/man/extract_month.Rd b/man/extract_month.Rd deleted file mode 100644 index 1b2cf4d..0000000 --- a/man/extract_month.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{extract_month} -\alias{extract_month} -\title{extract_month} -\usage{ -extract_month(sheetname) -} -\description{ -Extract month as digit (e.g., 12 for december) based on excel tab name -} diff --git a/man/extract_patient_data.Rd b/man/extract_patient_data.Rd deleted file mode 100644 index 11d04b6..0000000 --- a/man/extract_patient_data.Rd +++ /dev/null @@ -1,22 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script1_helper_read_patient_data.R -\name{extract_patient_data} -\alias{extract_patient_data} -\title{Extract the patient data from a month sheet of a tracker file.} -\usage{ -extract_patient_data(tracker_data_file, sheet, year) -} -\arguments{ -\item{tracker_data_file}{Excel tracker file.} - -\item{sheet}{Excel sheet name.} - -\item{year}{year of this tracker.} -} -\value{ -data.frame with the patient data -} -\description{ -Searches for first and last row that starts with country and clinic code -and extracts the data from this range as data.frame -} diff --git a/man/extract_product_data.Rd b/man/extract_product_data.Rd deleted file mode 100644 index 6d068b8..0000000 --- a/man/extract_product_data.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{extract_product_data} -\alias{extract_product_data} -\title{extract_product_data} -\usage{ -extract_product_data(monthly_tracker_df) -} -\description{ -Reads product data from a monthly file based on extraction logic -} diff --git a/man/extract_product_multiple.Rd b/man/extract_product_multiple.Rd deleted file mode 100644 index c6bd59d..0000000 --- a/man/extract_product_multiple.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{extract_product_multiple} -\alias{extract_product_multiple} -\title{extract_product_multiple} -\usage{ -extract_product_multiple(product_df) -} -\description{ -Splits product name cells with multiple values into separate rows. Uses numeric product content in cell for columns units_received or units_released -} diff --git a/man/extract_regimen.Rd b/man/extract_regimen.Rd deleted file mode 100644 index 0b7b481..0000000 --- a/man/extract_regimen.Rd +++ /dev/null @@ -1,17 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{extract_regimen} -\alias{extract_regimen} -\title{Maps Insulin regiment with allowed values.} -\usage{ -extract_regimen(raw_input) -} -\arguments{ -\item{raw_input}{The string that needs to be mapped} -} -\value{ -The mapping result -} -\description{ -The mapping is done using regular expressions. -} diff --git a/man/extract_unit_capacity.Rd b/man/extract_unit_capacity.Rd deleted file mode 100644 index 2ecf04d..0000000 --- a/man/extract_unit_capacity.Rd +++ /dev/null @@ -1,29 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_product_data.R -\name{extract_unit_capacity} -\alias{extract_unit_capacity} -\title{Extract Unit Capacity from a Specified Column} -\usage{ -extract_unit_capacity(df, column_name) -} -\arguments{ -\item{df}{A dataframe that contains the column to extract unit capacity from.} - -\item{column_name}{The name of the column to extract unit capacity from.} -} -\value{ -A dataframe with an additional column 'product_unit_capacity' that contains the extracted unit capacity. -} -\description{ -This function extracts the unit capacity from a specified column in a dataframe. -It assumes that the unit capacity is represented by numbers immediately before 's or s within parentheses. -All strings which contain the word 'singles' extracted as 1. -Non-numeric characters are removed and the extracted values are converted to numeric. -NA values are replaced with 1. -} -\examples{ -\dontrun{ -df <- data.frame(product = c("Product A (2s)", "Product B (3's)", "Product C")) -df <- extract_unit_capacity(df, "product") -} -} diff --git a/man/extract_year_from_age.Rd b/man/extract_year_from_age.Rd deleted file mode 100644 index 8425001..0000000 --- a/man/extract_year_from_age.Rd +++ /dev/null @@ -1,17 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{extract_year_from_age} -\alias{extract_year_from_age} -\title{Extract year from age descriptions like "10y5m"} -\usage{ -extract_year_from_age(age) -} -\arguments{ -\item{age}{text representation of diagnosis age} -} -\value{ -year of given age as character. -} -\description{ -Example: "1y5m" -> "1" -} diff --git a/man/find_string_cols.Rd b/man/find_string_cols.Rd deleted file mode 100644 index 63a97c2..0000000 --- a/man/find_string_cols.Rd +++ /dev/null @@ -1,23 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/wide_format_2_long_format.R -\name{find_string_cols} -\alias{find_string_cols} -\title{Find Columns Containing Specific String} -\usage{ -find_string_cols(df, string) -} -\arguments{ -\item{df}{A dataframe in which to search for the string.} - -\item{string}{The string to search for in the column names.} -} -\value{ -A character vector of matching column names. -} -\description{ -This function returns the column names of a dataframe that contain a specific string. -} -\examples{ -data(mtcars) -find_string_cols(mtcars, "mpg") -} diff --git a/man/fix_age.Rd b/man/fix_age.Rd deleted file mode 100644 index 233fd9e..0000000 --- a/man/fix_age.Rd +++ /dev/null @@ -1,26 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{fix_age} -\alias{fix_age} -\title{Fix age column.} -\usage{ -fix_age(age, dob, tracker_year, tracker_month, id) -} -\arguments{ -\item{age}{patient age.} - -\item{dob}{patient date of birth.} - -\item{tracker_year}{year the tracker was filled out.} - -\item{tracker_month}{month the tracker was filled out.} - -\item{id}{patient id.} -} -\value{ -fixed age. -} -\description{ -Based on date of birth calculate the current age of the patient. -If this age is different than the given patient age, we take the calculated one. -} diff --git a/man/fix_bmi.Rd b/man/fix_bmi.Rd deleted file mode 100644 index c58becd..0000000 --- a/man/fix_bmi.Rd +++ /dev/null @@ -1,23 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{fix_bmi} -\alias{fix_bmi} -\title{Calculate bmi value based on weight and height.} -\usage{ -fix_bmi(weight, height, id) -} -\arguments{ -\item{weight}{numeric weight value.} - -\item{height}{numeric height value.} - -\item{id}{patient id.} -} -\value{ -calculated bmi value. -} -\description{ -Formula: weight (kg) / height^2 (meters^2) -bmi is set to NA if either weight or height are NA. -bmi is set to error val if either weight or height are error val. -} diff --git a/man/fix_digit_date.Rd b/man/fix_digit_date.Rd deleted file mode 100644 index d7d1e7d..0000000 --- a/man/fix_digit_date.Rd +++ /dev/null @@ -1,26 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{fix_digit_date} -\alias{fix_digit_date} -\title{WIP: Convert dates that are stored as 5-digit number from Excel to date format} -\usage{ -fix_digit_date(date) -} -\arguments{ -\item{date}{date as five number digit.} -} -\value{ -Standardized date in a character format. -} -\description{ -WIP: Convert dates that are stored as 5-digit number from Excel to date format -} -\examples{ -\dontrun{ -fix_digit_date("44939") # should return "2023-01-13" -} -} -\seealso{ -https://stackoverflow.com/a/50274507: depending on whether read in -file was created on Mac or Windows, the origin date might need to be adjusted. -} diff --git a/man/fix_fbg.Rd b/man/fix_fbg.Rd deleted file mode 100644 index e01a2ac..0000000 --- a/man/fix_fbg.Rd +++ /dev/null @@ -1,17 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{fix_fbg} -\alias{fix_fbg} -\title{Replace common textual descriptions with numeric values} -\usage{ -fix_fbg(fbg) -} -\arguments{ -\item{fbg}{character value.} -} -\value{ -fbg transformed value. -} -\description{ -Replace common textual descriptions with numeric values -} diff --git a/man/fix_id.Rd b/man/fix_id.Rd deleted file mode 100644 index 02aa0fb..0000000 --- a/man/fix_id.Rd +++ /dev/null @@ -1,17 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{fix_id} -\alias{fix_id} -\title{check patient id and truncate it if necessary.} -\usage{ -fix_id(id) -} -\arguments{ -\item{id}{patient id.} -} -\value{ -Either original id, truncated id, or error value. -} -\description{ -check patient id and truncate it if necessary. -} diff --git a/man/fix_sex.Rd b/man/fix_sex.Rd deleted file mode 100644 index 5dcdefb..0000000 --- a/man/fix_sex.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{fix_sex} -\alias{fix_sex} -\title{Replace sex synonyms with either M, F or Other.} -\usage{ -fix_sex(sex, id) -} -\arguments{ -\item{sex}{patient gender.} - -\item{id}{patient id.} -} -\value{ -Either M, F or Other. -} -\description{ -Replace sex synonyms with either M, F or Other. -} diff --git a/man/fix_t1d_diagnosis_age.Rd b/man/fix_t1d_diagnosis_age.Rd deleted file mode 100644 index 3050f91..0000000 --- a/man/fix_t1d_diagnosis_age.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{fix_t1d_diagnosis_age} -\alias{fix_t1d_diagnosis_age} -\title{Try to replace text descriptions for age with proper numerical values.} -\usage{ -fix_t1d_diagnosis_age(t1d_diagnosis_age, id) -} -\arguments{ -\item{t1d_diagnosis_age}{patient age when diagnosed with T1D.} - -\item{id}{patient id} -} -\value{ -Corrected value with text replacement. -} -\description{ -Try to replace text descriptions for age with proper numerical values. -} diff --git a/man/fix_testing_frequency.Rd b/man/fix_testing_frequency.Rd deleted file mode 100644 index 4439bf4..0000000 --- a/man/fix_testing_frequency.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{fix_testing_frequency} -\alias{fix_testing_frequency} -\title{Fix problems with ranges and decimal numbers in testing_frequency.} -\usage{ -fix_testing_frequency(test_frq, id) -} -\arguments{ -\item{test_frq}{character value for testing frequency} - -\item{id}{patient id} -} -\value{ -validated test_frq value -} -\description{ -Fix problems with ranges and decimal numbers in testing_frequency. -} diff --git a/man/format_date.Rd b/man/format_date.Rd deleted file mode 100644 index 69c8b17..0000000 --- a/man/format_date.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{format_date} -\alias{format_date} -\title{format_date} -\usage{ -format_date(product_df) -} -\description{ -Combines all date reformatting functions into one overall function -} diff --git a/man/format_date_exceldate.Rd b/man/format_date_exceldate.Rd deleted file mode 100644 index 3ade5a0..0000000 --- a/man/format_date_exceldate.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{format_date_exceldate} -\alias{format_date_exceldate} -\title{format_date_exceldate} -\usage{ -format_date_exceldate(product_df) -} -\description{ -Reformates dates entered into excel in wrong format (e.g., dd-mm-yyyy) to final date format (yyyy-mm-dd) -} diff --git a/man/format_date_excelnum.Rd b/man/format_date_excelnum.Rd deleted file mode 100644 index f0afce5..0000000 --- a/man/format_date_excelnum.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{format_date_excelnum} -\alias{format_date_excelnum} -\title{format_date_excelnum} -\usage{ -format_date_excelnum(product_df) -} -\description{ -Reformates dates entered into excel in wrong format (e.g., 44042) to final date format (yyyy-mm-dd) -} diff --git a/man/get_allowed_provinces.Rd b/man/get_allowed_provinces.Rd deleted file mode 100644 index 46a54dc..0000000 --- a/man/get_allowed_provinces.Rd +++ /dev/null @@ -1,14 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_main.R -\name{get_allowed_provinces} -\alias{get_allowed_provinces} -\title{Read allowed provinces from a YAML file.} -\usage{ -get_allowed_provinces() -} -\value{ -A named character vector with all allowed provinces. -} -\description{ -Read in all provinces from a YAML file inside the provinces folder. -} diff --git a/man/get_files.Rd b/man/get_files.Rd deleted file mode 100644 index f49189e..0000000 --- a/man/get_files.Rd +++ /dev/null @@ -1,20 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_main.R -\name{get_files} -\alias{get_files} -\title{Find all files matching a search pattern in a given directory.} -\usage{ -get_files(tracker_root, pattern = "\\\\.xlsx$") -} -\arguments{ -\item{tracker_root}{The root directory to search in.} - -\item{pattern}{The search pattern to filter files.} -} -\value{ -A vector with file names. -} -\description{ -Searches recursively for files matching a search pattern inside the root dir. -Only returns the file names without the paths. -} diff --git a/man/get_synonyms.Rd b/man/get_synonyms.Rd deleted file mode 100644 index d2fdf82..0000000 --- a/man/get_synonyms.Rd +++ /dev/null @@ -1,14 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_main.R -\name{get_synonyms} -\alias{get_synonyms} -\title{Read synonyms from the synonyms YAML files.} -\usage{ -get_synonyms() -} -\value{ -A list with both patient and product data synonyms as tibble. -} -\description{ -Read in all defined synonyms from the YAML files inside the synonyms folder. -} diff --git a/man/get_tracker_year.Rd b/man/get_tracker_year.Rd deleted file mode 100644 index 3f8c11f..0000000 --- a/man/get_tracker_year.Rd +++ /dev/null @@ -1,20 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script1_get_tracker_year.R -\name{get_tracker_year} -\alias{get_tracker_year} -\title{Get the tracker year} -\usage{ -get_tracker_year(tracker_data_file, month_list) -} -\arguments{ -\item{tracker_data_file}{The path to the tracker file} - -\item{month_list}{The list of all the monthly sheets within a tracker} -} -\value{ -The year of the tracker -} -\description{ -This function parses the tracker year from either the monthly sheets -or the tracker file name if the parsing is not successful -} diff --git a/man/harmonize_input_data_columns.Rd b/man/harmonize_input_data_columns.Rd deleted file mode 100644 index 232cbd0..0000000 --- a/man/harmonize_input_data_columns.Rd +++ /dev/null @@ -1,18 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{harmonize_input_data_columns} -\alias{harmonize_input_data_columns} -\title{harmonize_input_data_columns} -\usage{ -harmonize_input_data_columns(product_df, columns_synonyms, sheet) -} -\arguments{ -\item{product_df:}{The product data to be cleaned and matched against column synonyms} - -\item{columns_synonyms:}{Long format output of read_column_synonyms to match columns} - -\item{sheet:}{The current sheet name of the tracker file} -} -\description{ -Imports the product df, cleans it and matches it against -} diff --git a/man/harmonize_patient_data_columns.Rd b/man/harmonize_patient_data_columns.Rd deleted file mode 100644 index 34b7738..0000000 --- a/man/harmonize_patient_data_columns.Rd +++ /dev/null @@ -1,20 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script1_helper_read_patient_data.R -\name{harmonize_patient_data_columns} -\alias{harmonize_patient_data_columns} -\title{Harmonize patient data column names.} -\usage{ -harmonize_patient_data_columns(patient_df, columns_synonyms) -} -\arguments{ -\item{patient_df}{data.frame holding the patient data of the month sheet of a tracker.} - -\item{columns_synonyms}{data.frame with synonyms for tracker variables.} -} -\value{ -data.frame with harmonized column names. -} -\description{ -Imports the patient df, cleans it and matches it against -column synonyms to unify column names -} diff --git a/man/init_paths.Rd b/man/init_paths.Rd deleted file mode 100644 index 52faf11..0000000 --- a/man/init_paths.Rd +++ /dev/null @@ -1,23 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_main.R -\name{init_paths} -\alias{init_paths} -\title{initialize all necessary paths} -\usage{ -init_paths(names, delete = FALSE) -} -\arguments{ -\item{names}{Folder names that will be created under the output folder.} - -\item{delete}{If TRUE, delete all files under output.} -} -\value{ -A list with tracker_root_path and output_root path -} -\description{ -Create necessary output folder for main script under tracker_root_path. -This script creates a new output folder next to the data files, and -deletes all old files in it. -The script asks the user to set the tracker_root_path, either by -selecting a folder or reading from the A4D_DATA_ROOT env var. -} diff --git a/man/link_product_patient.Rd b/man/link_product_patient.Rd deleted file mode 100644 index 5b003ef..0000000 --- a/man/link_product_patient.Rd +++ /dev/null @@ -1,25 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script3_link_product_patient.R -\name{link_product_patient} -\alias{link_product_patient} -\title{Link Product and Patient Data} -\usage{ -link_product_patient(product_file, patient_file) -} -\arguments{ -\item{product_file}{A string specifying the path to the product file.} - -\item{patient_file}{A string specifying the path to the patient file.} -} -\value{ -This function does not return a value. It prints log messages about the linking process. -} -\description{ -This function links a product file with a patient file. It reads the files, merges them based on 'file_name' and patient 'id', -finds the ids that are in the product data but not in the patient data, creates a summary of these ids, and logs the process. -} -\examples{ -\dontrun{ -link_product_patient("path/to/product_data.parquet", "path/to/patient_data.parquet") -} -} diff --git a/man/load_product_reference_data.Rd b/man/load_product_reference_data.Rd deleted file mode 100644 index 3c60057..0000000 --- a/man/load_product_reference_data.Rd +++ /dev/null @@ -1,25 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_product_data.R -\name{load_product_reference_data} -\alias{load_product_reference_data} -\title{Load Product List from Stock Summary} -\usage{ -load_product_reference_data( - stock_summary_xlsx = "reference_data/master_tracker_variables.xlsx" -) -} -\arguments{ -\item{stock_summary_xlsx}{A string that represents the path to the Excel file. Defaults to "reference_data/master_tracker_variables.xlsx".} -} -\value{ -A data frame that contains the product names. If there is an error during the process, it logs the error message. -} -\description{ -This function loads the product list from 'Stock_Summary' sheet in an Excel file. -} -\examples{ -\dontrun{ -product_list <- load_product_reference_data() -product_list <- load_product_reference_data("your_file.xlsx") -} -} diff --git a/man/log_to_json.Rd b/man/log_to_json.Rd deleted file mode 100644 index 35bfd32..0000000 --- a/man/log_to_json.Rd +++ /dev/null @@ -1,37 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/logger.R -\name{log_to_json} -\alias{log_to_json} -\title{Log-To-Json Formatter} -\usage{ -log_to_json( - message, - values = NA, - script = NA, - file = NA, - errorCode = NA, - warningCode = NA, - functionName = NA -) -} -\arguments{ -\item{message}{log message with glue syntax {values\link{'X'}}.} - -\item{values}{A named list with the glue referenced values as single strings.} - -\item{script}{script name where the log message was created.} - -\item{file}{file name where the log message was created.} - -\item{errorCode}{A unique error code for easy aggregation of error messages.} - -\item{warningCode}{A unique warning code for easy aggregation of warning messages.} - -\item{functionName}{The name of the function where the log message was created.} -} -\value{ -stringified JSON to log with logX functions -} -\description{ -Log-To-Json Formatter -} diff --git a/man/parse_dates.Rd b/man/parse_dates.Rd deleted file mode 100644 index 4cd3078..0000000 --- a/man/parse_dates.Rd +++ /dev/null @@ -1,18 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{parse_dates} -\alias{parse_dates} -\title{Try to handle invalid dates.} -\usage{ -parse_dates(date) -} -\arguments{ -\item{date}{character date value.} -} -\value{ -Either a correctly parsed date or NA. -} -\description{ -For each value of the character vector, this function uses the -lubridate parse functions to find common formats -} diff --git a/man/parse_invalid_dates.Rd b/man/parse_invalid_dates.Rd deleted file mode 100644 index 22da5b1..0000000 --- a/man/parse_invalid_dates.Rd +++ /dev/null @@ -1,17 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_dates.R -\name{parse_invalid_dates} -\alias{parse_invalid_dates} -\title{Try to handle invalid dates.} -\usage{ -parse_invalid_dates(df, colname) -} -\arguments{ -\item{df}{data frame with patient data.} - -\item{colname}{column that has invalid dates.} -} -\description{ -For each value of the character vector, this function uses the -lubridate parse functions to find common formats -} diff --git a/man/preparing_product_fields.Rd b/man/preparing_product_fields.Rd deleted file mode 100644 index 29dcc8f..0000000 --- a/man/preparing_product_fields.Rd +++ /dev/null @@ -1,28 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script3_create_table_product_data.R -\name{preparing_product_fields} -\alias{preparing_product_fields} -\title{Preparing Product Fields} -\usage{ -preparing_product_fields(merged_data) -} -\arguments{ -\item{merged_data}{A data frame that needs to be processed.} -} -\value{ -A data frame with processed fields. -} -\description{ -This function processes fields for a single csv product_data. -It checks if all fields are present in merged_data and adds missing fields with NA. -It ensures the correct data type for each column, replaces incorrect values with specified error values, -and reorders the columns according to the list of fields. -Any additional fields in the input dataframe are moved to the end. -} -\examples{ -\dontrun{ -df <- data.frame(product = c("A", "B"), product_units_notes = c("note1", "note2")) -df <- preparing_product_fields(df) -} - -} diff --git a/man/process_raw_patient_file.Rd b/man/process_raw_patient_file.Rd deleted file mode 100644 index 62ce0be..0000000 --- a/man/process_raw_patient_file.Rd +++ /dev/null @@ -1,21 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_process_patient_data.R -\name{process_raw_patient_file} -\alias{process_raw_patient_file} -\title{Process raw patient data} -\usage{ -process_raw_patient_file(paths, patient_file, patient_file_name, output_root) -} -\arguments{ -\item{paths}{list of paths to the input and output directories.} - -\item{patient_file}{name of the parquet file with the raw patient data.} - -\item{patient_file_name}{base name of the parquet file with the raw patient data.} - -\item{output_root}{root directory of the output folder.} -} -\description{ -This function reads the raw patient data from a parquet file, cleans the data, -and exports the cleaned data as a parquet file. -} diff --git a/man/process_raw_product_file.Rd b/man/process_raw_product_file.Rd deleted file mode 100644 index 4502e57..0000000 --- a/man/process_raw_product_file.Rd +++ /dev/null @@ -1,28 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_process_product_data.R -\name{process_raw_product_file} -\alias{process_raw_product_file} -\title{Process raw product data} -\usage{ -process_raw_product_file( - paths, - product_file, - product_file_name, - synonyms_product, - output_root -) -} -\arguments{ -\item{paths}{list of paths to the input and output directories.} - -\item{product_file}{name of the raw product data file.} - -\item{product_file_name}{base name of the raw product data file.} - -\item{synonyms_product}{list of synonyms for the product data.} - -\item{output_root}{The root directory of the output folder.} -} -\description{ -This function reads the raw product data from a parquet file, cleans the data, and exports the cleaned data as a parquet file. -} diff --git a/man/process_tracker_file.Rd b/man/process_tracker_file.Rd deleted file mode 100644 index c690f7d..0000000 --- a/man/process_tracker_file.Rd +++ /dev/null @@ -1,20 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script1_process_tracker_file.R -\name{process_tracker_file} -\alias{process_tracker_file} -\title{Process tracker file.} -\usage{ -process_tracker_file(paths, tracker_file, tracker_name, synonyms) -} -\arguments{ -\item{paths}{list of paths to directories.} - -\item{tracker_file}{Name of the tracker file.} - -\item{tracker_name}{Basename of the tracker file.} - -\item{synonyms}{list of synonyms for patient and product data.} -} -\description{ -Process tracker file. -} diff --git a/man/process_tracker_patient_data.Rd b/man/process_tracker_patient_data.Rd deleted file mode 100644 index f23fe91..0000000 --- a/man/process_tracker_patient_data.Rd +++ /dev/null @@ -1,26 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script1_process_patient_data.R -\name{process_tracker_patient_data} -\alias{process_tracker_patient_data} -\title{Process tracker patient data} -\usage{ -process_tracker_patient_data( - tracker_name, - tracker_data_file, - output_root, - synonyms_patient -) -} -\arguments{ -\item{tracker_name}{The name of the tracker.} - -\item{tracker_data_file}{The path to the tracker data file.} - -\item{output_root}{The root directory of the output folder.} - -\item{synonyms_patient}{A named list of synonyms for the patient data columns.} -} -\description{ -This function reads the patient data from a CSV file, adds a file name column, -and exports the data as a parquet file. -} diff --git a/man/process_tracker_product_data.Rd b/man/process_tracker_product_data.Rd deleted file mode 100644 index 701e792..0000000 --- a/man/process_tracker_product_data.Rd +++ /dev/null @@ -1,26 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script1_process_product_data.R -\name{process_tracker_product_data} -\alias{process_tracker_product_data} -\title{Process tracker product data} -\usage{ -process_tracker_product_data( - tracker_name, - tracker_data_file, - output_root, - synonyms_product -) -} -\arguments{ -\item{tracker_name}{The name of the tracker.} - -\item{tracker_data_file}{The path to the tracker data file.} - -\item{output_root}{The root directory of the output folder.} - -\item{synonyms_product}{A named list of synonyms for the product data columns.} -} -\description{ -This function reads the product data from an Excel file, -fills missing values, and exports the data as a parquet file. -} diff --git a/man/read_cleaned_patient_data.Rd b/man/read_cleaned_patient_data.Rd deleted file mode 100644 index 37f17e8..0000000 --- a/man/read_cleaned_patient_data.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script3_read_cleaned_patient_data.R -\name{read_cleaned_patient_data} -\alias{read_cleaned_patient_data} -\title{Read in all processed and cleaned patient data files} -\usage{ -read_cleaned_patient_data(input_root, patient_data_files) -} -\arguments{ -\item{input_root}{root directory containing all files.} - -\item{patient_data_files}{list of CSV files with cleaned patient data.} -} -\value{ -data frame holding all cleaned patient data with correct column types. -} -\description{ -Read in all processed and cleaned patient data files -} diff --git a/man/read_column_synonyms.Rd b/man/read_column_synonyms.Rd deleted file mode 100644 index bccf909..0000000 --- a/man/read_column_synonyms.Rd +++ /dev/null @@ -1,30 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_main.R -\name{read_column_synonyms} -\alias{read_column_synonyms} -\title{Get all synonyms for all variable names} -\usage{ -read_column_synonyms( - synonym_file, - path_prefixes = c("reference_data", "synonyms") -) -} -\arguments{ -\item{synonym_file}{A YAML file containing the synonyms} - -\item{path_prefixes}{Path prefixes for searching for the yaml file. Usually -this does not need to be set unless for testing purpouses.} -} -\value{ -A tibble containing unique column names and their synonyms. -} -\description{ -This function reads the synonyms from a YAML file -and generates a tibble containing unique column names and their synonyms. -} -\examples{ -\dontrun{ -read_column_synonyms(synonym_file = "synonyms_patient.yaml") -read_column_synonyms(synonym_file = "synonyms_product.yaml") -} -} diff --git a/man/read_column_synonyms_product.Rd b/man/read_column_synonyms_product.Rd deleted file mode 100644 index b823a6a..0000000 --- a/man/read_column_synonyms_product.Rd +++ /dev/null @@ -1,12 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{read_column_synonyms_product} -\alias{read_column_synonyms_product} -\title{read_column_synonyms_product} -\usage{ -read_column_synonyms_product(codebook_data_file) -} -\description{ -Imports the codebook, cleans, removes duplicates and transforms it -into long df format -} diff --git a/man/reading_patient_data.Rd b/man/reading_patient_data.Rd deleted file mode 100644 index 447be66..0000000 --- a/man/reading_patient_data.Rd +++ /dev/null @@ -1,20 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script1_read_patient_data.R -\name{reading_patient_data} -\alias{reading_patient_data} -\title{Reading patient data from the tracker data file} -\usage{ -reading_patient_data(tracker_data_file, columns_synonyms) -} -\arguments{ -\item{tracker_data_file}{The path to the tracker data file.} - -\item{columns_synonyms}{A named list of column synonyms.} -} -\value{ -A data frame with the patient data. -} -\description{ -This function reads the patient data from the tracker data file, -harmonizes the columns, and returns the data as a data frame. -} diff --git a/man/reading_product_data_step2.Rd b/man/reading_product_data_step2.Rd deleted file mode 100644 index 1d9683e..0000000 --- a/man/reading_product_data_step2.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_product_data.R -\name{reading_product_data_step2} -\alias{reading_product_data_step2} -\title{Process product data in script 2.} -\usage{ -reading_product_data_step2(df, columns_synonyms) -} -\arguments{ -\item{df}{Dataframe. Output of tracker file from script 1 for product data.} - -\item{columns_synonyms}{Tibble with synonyms for product data (columns: name_clean and name_to_be_matched)} -} -\value{ -Cleaned product data for one specified tracker. -} -\description{ -Processes data output from script 1. Data will be further cleaned and processed to fix data quality issues and to create a report. -} diff --git a/man/recode_unitcolumnstozero.Rd b/man/recode_unitcolumnstozero.Rd deleted file mode 100644 index b9d8d9f..0000000 --- a/man/recode_unitcolumnstozero.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{recode_unitcolumnstozero} -\alias{recode_unitcolumnstozero} -\title{recode_unitcolumnstozero} -\usage{ -recode_unitcolumnstozero(product_df) -} -\description{ -Recode NAs to 0 in all "unit columns" -} diff --git a/man/remove_original_cells.Rd b/man/remove_original_cells.Rd deleted file mode 100644 index 40d54ae..0000000 --- a/man/remove_original_cells.Rd +++ /dev/null @@ -1,32 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/wide_format_2_long_format.R -\name{remove_original_cells} -\alias{remove_original_cells} -\title{Remove Original Cells based on Relationship} -\usage{ -remove_original_cells(df, col_units_rel, col_rel_to) -} -\arguments{ -\item{df}{A data frame containing the columns \code{col_units_rel} and \code{col_rel_to}.} - -\item{col_units_rel}{The name of the column containing units (cells) potentially related to multiple other units.} - -\item{col_rel_to}{The name of the column containing relationships between units, where comma (",") indicates a relationship with multiple units.} -} -\value{ -The modified data frame where rows with comma-separated relationships in \code{col_rel_to} have corresponding values in \code{col_units_rel} set to NA and the comma-separated values removed from \code{col_rel_to}. -} -\description{ -This function removes rows from the column \code{col_units_rel} where the corresponding value in \code{col_rel_to} contains a comma (","). -It also removes those comma-separated values from the \code{col_rel_to} column itself. -} -\examples{ -# Create a sample data frame -df <- data.frame(col_units_rel = c(1, 2, 3, 4), col_rel_to = c("A", "B,C", "D", "A,B")) - -# Remove original cells based on relationship -df_modified <- remove_original_cells(df, col_units_rel = "col_units_rel", col_rel_to = "col_rel_to") - -# Print the modified data frame -print(df_modified) -} diff --git a/man/remove_rows_with_na_columns.Rd b/man/remove_rows_with_na_columns.Rd deleted file mode 100644 index 589b9f5..0000000 --- a/man/remove_rows_with_na_columns.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_product_data.R -\name{remove_rows_with_na_columns} -\alias{remove_rows_with_na_columns} -\title{Remove Rows with NA Values in Specified Columns.} -\usage{ -remove_rows_with_na_columns(df, column_names) -} -\arguments{ -\item{df}{A data frame that contains the data from which rows need to be removed.} - -\item{column_names}{A character vector specifying the column names to be checked for NA values.} -} -\value{ -A new data frame with rows removed if all the specified columns in the row contain NA values. The returned data frame will have the same structure as the input data frame, but with rows that satisfy the condition removed. -} -\description{ -This function takes a data frame and a vector of column names as input. It removes rows from the data frame where all the specified columns contain only NA values. If any of the specified columns have at least one non-NA value, the corresponding row will be retained in the output data frame. -} diff --git a/man/replace_extra_total_values_with_NA.Rd b/man/replace_extra_total_values_with_NA.Rd deleted file mode 100644 index e7f8936..0000000 --- a/man/replace_extra_total_values_with_NA.Rd +++ /dev/null @@ -1,30 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/wide_format_2_long_format.R -\name{replace_extra_total_values_with_NA} -\alias{replace_extra_total_values_with_NA} -\title{Replace Extra "Total" Values with NA} -\usage{ -replace_extra_total_values_with_NA(df, column_name) -} -\arguments{ -\item{df}{A data frame containing the column to be processed.} - -\item{column_name}{The name of the column where extra "Total" values will be replaced with NA.} -} -\value{ -The modified data frame where any value in \code{column_name} preceded by "Total" in one of the two preceding columns is replaced with NA. -} -\description{ -This function replaces values in a specified column (\code{column_name}) with NA if one of the two preceding columns contains the word "Total". -This is intended to handle cases where there might be multiple "Total" values in close proximity, keeping only the first. -} -\examples{ -# Create a sample data frame -df <- data.frame(col1 = c("A", "B", "Total"), col2 = c(1, 2, 3), col3 = c(4, 5, 6), col4 = c(7, "Total", 9)) - -# Replace extra "Total" values with NA -df_modified <- replace_extra_total_values_with_NA(df, column_name = "col4") - -# Print the modified data frame -print(df_modified) -} diff --git a/man/replace_range_with_mean.Rd b/man/replace_range_with_mean.Rd deleted file mode 100644 index 56e99d3..0000000 --- a/man/replace_range_with_mean.Rd +++ /dev/null @@ -1,17 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{replace_range_with_mean} -\alias{replace_range_with_mean} -\title{Replace a range with mean.} -\usage{ -replace_range_with_mean(x) -} -\arguments{ -\item{x}{range as text..} -} -\value{ -mean of that range -} -\description{ -A range like "0-2" will be replaced with the mean(c(0,2)). -} diff --git a/man/report_empty_intersections.Rd b/man/report_empty_intersections.Rd deleted file mode 100644 index 40b5cd9..0000000 --- a/man/report_empty_intersections.Rd +++ /dev/null @@ -1,26 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script3_create_table_product_data.R -\name{report_empty_intersections} -\alias{report_empty_intersections} -\title{Report Empty Intersections} -\usage{ -report_empty_intersections(df, row_category, col_category) -} -\arguments{ -\item{df}{A dataframe.} - -\item{row_category}{The name of the column in df to use for the rows of the cross-tabulation.} - -\item{col_category}{The name of the column in df to use for the columns of the cross-tabulation.} -} -\value{ -A warning message with the number of row_category values that have a sum of 0 in col_category, and a list of these row_category values. -} -\description{ -This function reports the names of the rows in a cross-tabulation of two columns -from a dataframe where the sum is 0. -} -\examples{ -df <- data.frame(file_name = c("file1", "file2", "file3"), table_country = c("country1", "country2", NA)) -report_empty_intersections(df, "file_name", "table_country") -} diff --git a/man/report_unknown_products.Rd b/man/report_unknown_products.Rd deleted file mode 100644 index 9319952..0000000 --- a/man/report_unknown_products.Rd +++ /dev/null @@ -1,21 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_product_data.R -\name{report_unknown_products} -\alias{report_unknown_products} -\title{Report unknown products} -\usage{ -report_unknown_products(df, Sheet, stock_list_df) -} -\arguments{ -\item{df}{A dataframe containing product data.} - -\item{Sheet}{The name of the sheet where the product data is located.} - -\item{stock_list_df}{A dataframe containing the stock list of products.} -} -\value{ -This function does not return a value. It logs a warning message if there are any unknown products, and logs an info message if there are no unknown products. -} -\description{ -This function reports unknown products. It compares the product list in a given dataframe with a stock list, and logs any products that are not found in the stock list. -} diff --git a/man/sanitize_str.Rd b/man/sanitize_str.Rd deleted file mode 100644 index 6955911..0000000 --- a/man/sanitize_str.Rd +++ /dev/null @@ -1,26 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_sanitize_str.R -\name{sanitize_str} -\alias{sanitize_str} -\title{Sanitize column name} -\usage{ -sanitize_str(text) -} -\arguments{ -\item{text}{A text to be sanitized.} -} -\value{ -The sanitized text. -} -\description{ -This function takes in a string, -sanitizes it by converting all letters to lower case, -removing all spaces and special characters except alphanumeric characters, -and then returns the sanitized string. -} -\examples{ -sanitize_str("John Doe's Column") # should return "johndoescolumn" -sanitize_str("Date 2022") # should return "date2022" -sanitize_str("My Awesome 1st Column!!") # should return "myawesome1stcolumn" - -} diff --git a/man/set_second_row_as_headers_and_remove_first_row.Rd b/man/set_second_row_as_headers_and_remove_first_row.Rd deleted file mode 100644 index 76a9fa5..0000000 --- a/man/set_second_row_as_headers_and_remove_first_row.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{set_second_row_as_headers_and_remove_first_row} -\alias{set_second_row_as_headers_and_remove_first_row} -\title{set_second_row_as_headers_and_remove_first_row} -\usage{ -set_second_row_as_headers_and_remove_first_row(df) -} -\description{ -Clean df where first row is empty -} diff --git a/man/setup_file_logger.Rd b/man/setup_file_logger.Rd deleted file mode 100644 index 2b1ff4e..0000000 --- a/man/setup_file_logger.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/logger.R -\name{setup_file_logger} -\alias{setup_file_logger} -\title{Add a basic default file logger for a newly processed file.} -\usage{ -setup_file_logger(logfile, output_root) -} -\arguments{ -\item{logfile}{The name of the log file.} - -\item{output_root}{Output root directory for the current process.} -} -\value{ -returns the loggers that where previously set for usage with with_ -} -\description{ -Add a basic default file logger for a newly processed file. -} diff --git a/man/setup_logger.Rd b/man/setup_logger.Rd deleted file mode 100644 index d338b19..0000000 --- a/man/setup_logger.Rd +++ /dev/null @@ -1,14 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/logger.R -\name{setup_logger} -\alias{setup_logger} -\title{Configure basic logger} -\usage{ -setup_logger(output_dir, log_name) -} -\arguments{ -\item{output_dir:}{Output directory of the main script} -} -\description{ -Configure basic logger -} diff --git a/man/split_bp_in_sys_and_dias.Rd b/man/split_bp_in_sys_and_dias.Rd deleted file mode 100644 index bd54877..0000000 --- a/man/split_bp_in_sys_and_dias.Rd +++ /dev/null @@ -1,17 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/script2_helper_patient_data_fix.R -\name{split_bp_in_sys_and_dias} -\alias{split_bp_in_sys_and_dias} -\title{Separate single blood pressure value into sys and dias values.} -\usage{ -split_bp_in_sys_and_dias(df) -} -\arguments{ -\item{df}{data frame with patient data.} -} -\value{ -data frame with two new columns: blood_pressure_sys_mmhg and blood_pressure_dias_mmhg. -} -\description{ -Separate single blood pressure value into sys and dias values. -} diff --git a/man/switch_columns_stock.Rd b/man/switch_columns_stock.Rd deleted file mode 100644 index 49aa806..0000000 --- a/man/switch_columns_stock.Rd +++ /dev/null @@ -1,17 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/read_product_data.R -\name{switch_columns_stock} -\alias{switch_columns_stock} -\title{Switch product_received_from and product_units_received column.} -\usage{ -switch_columns_stock(df) -} -\arguments{ -\item{df}{Dataframe. Output of tracker file from script 1 for product data.} -} -\value{ -Dataframe with columns switched (renamed) -} -\description{ -Renaming columns when string Remaining stock in wrong column. E.g. 2018_PNG for Nov and Dec. -} diff --git a/man/testit.Rd b/man/testit.Rd deleted file mode 100644 index 35249bf..0000000 --- a/man/testit.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{testit} -\alias{testit} -\title{testit} -\usage{ -testit(x) -} -\description{ -Wait for x seconds in script -} diff --git a/man/update_receivedfrom.Rd b/man/update_receivedfrom.Rd deleted file mode 100644 index 01987a4..0000000 --- a/man/update_receivedfrom.Rd +++ /dev/null @@ -1,17 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/helper_product_data.R -\name{update_receivedfrom} -\alias{update_receivedfrom} -\title{Transfers in trackers with specific format which have "START BALANCE" in product data the start balance to the correct column.} -\usage{ -update_receivedfrom(product_df) -} -\arguments{ -\item{product_df}{Input product data.} -} -\value{ -Dataframe with corrected product_received_from column extracting the starting balance from trackers where necessary. -} -\description{ -Run before clean_receivedfrom and recode_unitcolumnstozero. Necessary for specific format such as in 2019_PKH and 2020_STH. When start balance in tracker next to text "start balance" one need to extract the balance either from the "received from" column or the "units released" column. -} diff --git a/man/wide_cells_2_rows.Rd b/man/wide_cells_2_rows.Rd deleted file mode 100644 index 55d0fad..0000000 --- a/man/wide_cells_2_rows.Rd +++ /dev/null @@ -1,44 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/wide_format_2_long_format.R -\name{wide_cells_2_rows} -\alias{wide_cells_2_rows} -\title{Convert Wide-Format Cells to Long-Format Rows} -\usage{ -wide_cells_2_rows(df) -} -\arguments{ -\item{df}{A data frame in wide-format with columns named as described.} -} -\value{ -A new data frame in long-format where each row represents a single release event with columns: -\itemize{ -\item The original columns from the input data frame. -\item "Released To": The recipient unit for a specific release. -\item "Units Released": The quantity of units released to the corresponding recipient. -\item "Date": The date of the release event (unchanged from the input). -\item "Received From": unchanged from the input. -} -} -\description{ -This function converts a data frame from wide-format to long-format. It assumes specific column naming conventions: -\itemize{ -\item Columns containing "Released To" are expected to have comma-separated values representing units released to. -\item Columns containing "Units Released" are expected to store the corresponding unit quantity for each "Released To" value. -\item A column containing "Date" (or similar) should exist to hold the date information. -} -} -\examples{ -# Create a sample data frame in wide-format -df <- data.frame( - ID = c(1, 2, 3), - "Units Available" = c(10, 15, 20), - "Released To A,B" = c(NA, "5-2", "8-1"), - Date = c("2024-01-01", "2024-02-01", "2024-03-01") -) - -# Convert wide-format cells to long-format rows -df_long <- wide_cells_2_rows(df) - -# Print the long-format data frame -print(df_long) -} diff --git a/man/with_file_logger.Rd b/man/with_file_logger.Rd deleted file mode 100644 index fbe1797..0000000 --- a/man/with_file_logger.Rd +++ /dev/null @@ -1,18 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/logger.R -\name{with_file_logger} -\alias{with_file_logger} -\title{Temporary File Logger} -\usage{ -with_file_logger(new, code, output_root) -} -\arguments{ -\item{new}{New filename for the logfile} - -\item{code}{Code to be executed} - -\item{output_root}{Root output folder from where to construct the log path} -} -\description{ -Temporarily change logging to exclusively log to specific log file -} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d53bb16 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,85 @@ +[project] +name = "a4d" +version = "2.0.0" +description = "A4D Medical Tracker Data Processing Pipeline (Python)" +readme = "README.md" +requires-python = ">=3.14" +authors = [{ name = "Michael Aydinbas", email = "michael.aydinbas@gmail.com" }] +license = { text = "MIT" } + +dependencies = [ + "polars>=0.20.0", + "pydantic>=2.6.0", + "pydantic-settings>=2.2.0", + "pandera[polars]>=0.18.0", + "loguru>=0.7.0", + "openpyxl>=3.1.0", + "google-cloud-bigquery>=3.17.0", + "google-cloud-storage>=2.14.0", + "pyyaml>=6.0", + "typer>=0.9.0", + "rich>=13.7.0", + "tqdm>=4.66.0", + "python-dateutil>=2.8.0", + "fastexcel>=0.16.0", +] + + +[dependency-groups] +dev = [ + "pre-commit>=4.3.0", + "pytest>=8.4.2", + "pytest-cov>=7.0.0", + "pytest-mock>=3.15.1", + "ruff>=0.14.1", + "ty>=0.0.1a23", +] + +[project.scripts] +a4d = "a4d.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.ruff] +line-length = 100 +target-version = "py314" +# docs/archive/ holds working spec/history documents (MIGRATION_GUIDE.md, +# PYTHON_IMPROVEMENTS.md). Their fenced Python blocks are illustrative prose +# -- never imported, never executed -- so there is nothing for ruff to +# validate, and reformatting them only rewraps examples chosen for +# readability (ticket 33: this silently broke CI for four days, because +# `ruff format` rewrites them without asking and the diff reads as +# unrelated noise). +extend-exclude = ["docs/archive"] +lint.select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "N", # pep8-naming + "UP", # pyupgrade + "B", # flake8-bugbear + "A", # flake8-builtins + "C4", # flake8-comprehensions + "PT", # flake8-pytest-style +] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401"] # Allow unused imports in __init__.py + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_functions = ["test_*"] +markers = [ + "slow: marks tests as slow (deselected by default)", + "integration: marks tests as integration tests requiring real tracker files", + "e2e: marks tests as end-to-end tests (extraction + cleaning)", +] +addopts = ["--cov=src/a4d", "--cov-report=term-missing", "--cov-report=html"] +filterwarnings = [ + "ignore::RuntimeWarning:google_crc32c", + "ignore:from_arrow.. will return a Series:FutureWarning", +] diff --git a/readme.html b/readme.html deleted file mode 100644 index 4b4d388..0000000 --- a/readme.html +++ /dev/null @@ -1,489 +0,0 @@ - - - - - - - - - - - - - -readme - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - -
-

Welcome!

-

Below is a description of the folders and the files we find in -these

-
    -

  • 1_Orga : Files related to organization of the -project (e.g., timelines)

    • -

  • 2_Data: Files containing dummy data to start -setting up the scripts

    • -

  • 3_Code : Contains the scripts to extract, and -preprocess the patient and the product data.

    -
      -
    • 00_a4d_patient_tracker_extract_helper : Helper functions to -extract the data for the patients (needed to run the -01_a4d_patient_tracker_extract script)
      • -
    • 00_helper_product_data : Helper functions to extract the -data for the products
      • -
    • 01_a4d_patient_tracker_extract : Script that reads in the -different raw (Excel) trackers for each clinic and year and extracts the -data into an machine readble table.
      • -
    • 02_a4d_patient_tracker_format: Script that reads in the -output of 01_a4d_patient_tracker_extract and reformats columns -according to the codebook indications, performs checks and removes -duplicates (i.e., patients whose information is copied across months but -remains unchanged). Returns a dataframe that is ready to input in the -database and another dataframe indicating the locations of errors or -non-readable data.
      • -
    • OTHER PRODUCT SCRIPTS: ….NEED TO ADD INFORMATION
      • -
    • -

  • 4_db: Subfolders and files related to the -organization and structure of the database

    • -

  • 4ADMonthlyTrackerCodebook : Codebook containing -the information on the variables that we are extracting from the -trackers (patient and product data). Also contains tabs listing the -different labels that one variable may have together with its -standardized formulation.

    • -
-
-
-

Setup

-

We use the renv package to -manage dependencies.

-

This project was setup with renv and uses a local -.Rprofile file that activates renv. The first -time you open this project in RStudio, this will check if -renv is already installed, and, if that is not the case, -install it.

-

You are then informed about the difference between your local R -environment and the packages used for this project. Check your console -in RStudio after opening. You should see something like:

-
* One or more packages recorded in the lockfile are not installed.
-* Use `renv::status()` for more details.
-

You can use renv::status() to see which packages will be -installed. Once you ready, run

-
renv::restore()
-

and renv will install all packages with the version as -stated in the renv.lock file.

-
-

Updating the Lockfile

-

See (collaborating)[https://rstudio.github.io/renv/articles/collaborating.html] -for the full details.

-

While working on a project, you or your collaborators may need to -update or install new packages in your project. When this occurs, you’ll -also want to ensure your collaborators are then using the same -newly-installed packages. In general, the process looks like this:

-
    -
  • A user installs, or updates, one or more packages in their local -project library;
    • -
  • That user calls renv::snapshot() to update the -renv.lock lockfile;
    • -
  • That user then shares the updated version of renv.lock -with their collaborators (meaning that this file is commited and pushed -via git);
    • -
  • Other collaborators then call renv::restore() to -install the packages specified in the newly-updated lockfile.
    • -
-

If you want to add another package to this project, install the -package with renv::install() instead of -package.install().

-

Note: Not all packages are locked by -renv. For example, if you want to preview this Readme with -RStudio, RStudio will likely ask to install or update additional -packages like markdown. That is ok and intended because -these additional packages are not used by the project code so it is up -to you to install them or not.

-
-
- - - - -
- - - - - - - - - - - - - - - diff --git a/readme.md b/readme.md deleted file mode 100644 index 077cca5..0000000 --- a/readme.md +++ /dev/null @@ -1,142 +0,0 @@ -## Welcome! - -Below is a description of the folders and the files we find in these - -- **R**: Contains the scripts to extract, and preprocess the patient and the product data. - - *patient_tracker_extract_helper* : Helper functions to extract the data for the patients (needed to run the *patient_tracker_extract* script) - - *helper_product_data*: Helper functions to extract the data for the products - - *patient_tracker_extract*: Script that reads in the different raw (Excel) trackers for each clinic and year and extracts the data into an machine readable table. - - *patient_tracker_format*: Script that reads in the output of *patient_tracker_extract* and reformats columns according to the codebook indications, performs checks and removes duplicates (i.e., patients whose information is copied across months but remains unchanged). Returns a dataframe that is ready to input in the database and another dataframe indicating the locations of errors or non-readable data. -- **4ADMonthlyTrackerCodebook**: Codebook containing the information on the variables that we are extracting from the trackers (patient and product data). Also contains tabs listing the different labels that one variable may have together with its standardized formulation. - -## Setup - -We use the [renv](https://rstudio.github.io/renv/index.html) package to manage dependencies. - -This project was setup with `renv` and uses a local `.Rprofile` file that activates `renv`. -The first time you open this project in RStudio, this will check if `renv` is already installed, and, if that is not the case, install it. - -You are then informed about the difference between your local R environment and the packages used for this project. -Check your console in RStudio after opening. -You should see something like: - -```R -* One or more packages recorded in the lockfile are not installed. -* Use `renv::status()` for more details. -``` - -You can use `renv::status()` to see which packages will be installed. -Once you are ready, run - -```R -renv::restore() -``` - -and `renv` will install all packages with the version as stated in the `renv.lock` file. - -### Updating the Lockfile - -See [collaborating](https://rstudio.github.io/renv/articles/collaborating.html) for the full details. - -While working on a project, you or your collaborators may need to update or install new packages in your project. -When this occurs, you’ll also want to ensure your collaborators are then using the same newly-installed packages. -In general, the process looks like this: - -- A user installs, or updates, one or more packages in their local project library; -- That user calls `renv::snapshot()` to update the `renv.lock` lockfile; -- That user then shares the updated version of `renv.lock` with their collaborators (meaning that this file is commited and pushed via `git`); -- Other collaborators then call `renv::restore()` to install the packages specified in the newly-updated lockfile. - -If you want to add another package to this project, install the package with `renv::install()` instead of `package.install()`. - -**Note**: Not all packages are locked by `renv`. -For example, if you want to preview this Readme with RStudio, RStudio will likely ask to install or update additional packages like `markdown`. -That is ok and intended because these additional packages are not used by the project code so it is up to you to install them or not. - -### Loading the code - -Once you have installed the dependencies with `renv::restore()` you can go on and load our "package". - -However, you need one more dependency, and that is [devtools](https://devtools.r-lib.org/). -This is because `renv` does only lock required packages that are needed to run the code, not packages to develop the code (like `devtools`). - -So make sure to run - -```r -install.packages("devtools") -``` - -It is also possible that this package was already downloaded because we have this code in the `.Rprofile` file that is executed automatically when RStudio is opened: - -```r -if (interactive()) { - require("devtools", quietly = TRUE) - # automatically attaches usethis - devtools::load_all() -} -``` - -Now you have access to `devtools`, which is in fact a set of packages that are installed, like `usethis` and `roxygen2`. - -To load all functions within the `./R` folder, run - -```r -devtools::load_all() -``` - -This will make available the `a4d` package in the global environment, giving you access to all functions within it, as well as the core packages of `tidyverse` because it is listed at "Depends" in the DESCRIPTION file. - -You can now go ahead and run one of the two main scripts: -- `run_a4d_patient_data.R` -- `run_a4d_product_data.R` - -### Loading the data - -We will all have the encrypted data stored on different folders within our computers. -To account for the different file paths for every user and to speed the selection of the tracker files (where the data is stored), there is the following solution: - -- Run `usethis::edit_r_environ()` - - This should open the following file: `.Renviron` -- Add the following line: - - `A4D_DATA_ROOT = "your_path"` - - Replace `"your_path"` with the path to your A4D tracker files - - E.g. `A4D_DATA_ROOT = "D:/A4D"` -- Save the `.Renviron` file -- You are good to go and will not need to re-select the folder containing the tracker files when running `select_A4D_directory()`. - This function will now get the correct path from the `.Renviron` file. - -## Development workflow - -For a short overview, see [cheatsheets](https://devtools.r-lib.org/#cheatsheet). - -If you want to change any (code) file, add new files or delete existing files, please follow these steps: - -1. only once: `git clone` this repository -2. switch to the develop branch: `git checkout develop` or `git switch develop` -3. update develop: `git pull` -4. create a new branch: `git checkout -b -` (or create the branch in GitHub and just switch to it, no `-b` needed then) -5. do your code changes, create new R files with `usethis::use_r()` and new test files with `usethis::use_test()` -6. load and test/execute your code changes: `devtools::load_all()` -7. run the tests: `devtools::test()` - a. Fix any problems until all tests are green and make sure your changes do not break other code -8. check the package: `devtools::check()` - a. Fix any problems until all checks are green -9. document your functions: use roxygen documentation by comments starting with `#'` -10. update documentation: `devtools:document()` -11. optional: add additional documentation to the README or create a RMarkdown file with examples -12. get latest changes from develop: `git merge develop` - a. if there are any conflicts, solve them -13. create a (final) commit with all your changes: `git add ` and `git commit -m""` -14. push your changes: `git push` (for a newly created local branch, you will need to set an up-stream first, just follow the instructions on the command line) -15. Check the GitHub workflows (cicd pipelines) for your branch and fix any problems -16. Create a PR with develop as target -17. Again, check the GitHub workflows for your PR and fix any problems - -In addition to this general workflow, there are some additional steps required if you made use of external packages not yet stored in `renv.lock`: - -1. install the package: Use `renv::install()` if you want to use this package only for development, or `usethis::use_package()` to add the package to the DESCRIPTION file - a. if you want to update a package, you can also use `renv::install()`, without arguments it will update all listed packages in the lock file -2. use the package with the `package::fun()` syntax in your code -3. use `renv::snapshot()` to update the `renv.lock` -4. make sure to add the `renv.lock` file with your PR - diff --git a/reference_data/clinic_data.xlsx b/reference_data/clinic_data.xlsx index 686e2c0..6659d5b 100644 Binary files a/reference_data/clinic_data.xlsx and b/reference_data/clinic_data.xlsx differ diff --git a/reference_data/data_cleaning.yaml b/reference_data/data_cleaning.yaml index 504d5e4..789553a 100644 --- a/reference_data/data_cleaning.yaml +++ b/reference_data/data_cleaning.yaml @@ -91,7 +91,7 @@ insulin_regimen: type: basic_function - allowed_values: - "Basal-bolus (MDI)" - - "Premixed 30/70 DB" + - "Premixed 30/70 BD" - "Self-mixed BD" - "Modified conventional TID" replace_invalid: false diff --git a/reference_data/master_tracker_variables.xlsx b/reference_data/master_tracker_variables.xlsx index 484ece2..da5f036 100644 Binary files a/reference_data/master_tracker_variables.xlsx and b/reference_data/master_tracker_variables.xlsx differ diff --git a/reference_data/synonyms/synonyms_patient.yaml b/reference_data/synonyms/synonyms_patient.yaml index 3844198..cdb3527 100644 --- a/reference_data/synonyms/synonyms_patient.yaml +++ b/reference_data/synonyms/synonyms_patient.yaml @@ -74,6 +74,7 @@ complication_screening_kidney_test_date: - Kidney Function Test Date (dd-mmm-yyyy) complication_screening_kidney_test_value: - Kidney Function Test UACR (mg/mmol) +- Kidney Function Test UACR (mg/g) complication_screening_lipid_profile_cholesterol_value: - Lipid Profile Cholesterol complication_screening_lipid_profile_date: diff --git a/reference_data/validation_rules.yaml b/reference_data/validation_rules.yaml new file mode 100644 index 0000000..e79e388 --- /dev/null +++ b/reference_data/validation_rules.yaml @@ -0,0 +1,163 @@ +# Python Pipeline Validation Rules +# +# This file defines allowed values for data validation in the Python pipeline. +# It is separate from data_cleaning.yaml (used by R pipeline) to allow +# independent evolution of the two pipelines. +# +# Structure: +# column_name: +# allowed_values: [list of valid values] +# replace_invalid: true/false (whether to replace with error value) +# +# Note: Data transformations are hardcoded in src/a4d/clean/transformers.py, +# not defined in YAML. + +# Numeric range bounds consumed by src/a4d/validate/source_vs_output_patient.py +# only. Mirrors the hardcoded thresholds in src/a4d/clean/patient.py:510-562. +# Single-source-of-truth refactor blocked by patient.py being a golden file +# (see CLAUDE.local.md). If you change one, change the other. +# height bounds apply AFTER the cm->m auto-conversion at clean/patient.py:530-535. +numeric_ranges: + height: {min: 0.0, max: 2.3} + weight: {min: 0.0, max: 200.0} + bmi: {min: 10.0, max: 80.0} + age: {min: 0, max: 100} + hba1c_baseline: {min: 0.0, max: 25.0} + hba1c_updated: {min: 0.0, max: 25.0} + fbg_updated_mmol: {min: 0.0, max: 150.0} + +analog_insulin_long_acting: + allowed_values: ["N", "Y"] + replace_invalid: true + +analog_insulin_rapid_acting: + allowed_values: ["N", "Y"] + replace_invalid: true + +clinic_visit: + allowed_values: ["N", "Y"] + replace_invalid: true + +complication_screening_eye_exam_value: + allowed_values: ["Normal", "Abnormal"] + replace_invalid: true + +complication_screening_foot_exam_value: + allowed_values: ["Normal", "Abnormal"] + replace_invalid: true + +dm_complication_eye: + allowed_values: ["N", "Y"] + replace_invalid: true + +dm_complication_kidney: + allowed_values: ["N", "Y"] + replace_invalid: true + +dm_complication_others: + allowed_values: ["N", "Y"] + replace_invalid: true + +hospitalisation_cause: + allowed_values: ["DKA", "HYPO", "HYPER", "OTHER"] + replace_invalid: true + +human_insulin_intermediate_acting: + allowed_values: ["N", "Y"] + replace_invalid: true + +human_insulin_pre_mixed: + allowed_values: ["N", "Y"] + replace_invalid: true + +human_insulin_short_acting: + allowed_values: ["N", "Y"] + replace_invalid: true + +insulin_regimen: + # Note: Values are transformed by extract_regimen() in transformers.py first + allowed_values: + - "Basal-bolus (MDI)" + - "Premixed 30/70 BD" + - "Self-mixed BD" + - "Modified conventional TID" + replace_invalid: false # Don't replace - these are post-transformation values + +insulin_type: + allowed_values: ["Human Insulin", "Analog Insulin"] + replace_invalid: true + +insulin_subtype: + # Python's builder emits comma-joined values (e.g. "pre-mixed,rapid-acting") for + # multi-insulin patients, matching R's builder. R's validator treated the CSV as a + # single opaque string and rejected it, so R output "Undefined" for every multi-insulin + # patient. Python uses allow_csv_subset=true: each comma-separated token is validated + # against allowed_values independently, then rejoined in canonical case. + # R's "rapic-acting" typo is already corrected in Python's builder (clean/patient.py). + allowed_values: + - "Pre-mixed" + - "Short-acting" + - "Intermediate-acting" + - "Rapid-acting" + - "Long-acting" + replace_invalid: true + allow_csv_subset: true + +observations_category: + allowed_values: + - "Status IN" + - "Status OUT" + - "Clinic Follow Up" + - "Hospitalisation" + - "Support" + - "DM Complication" + - "Insulin Regimen" + - "Other" + replace_invalid: false + +patient_consent: + allowed_values: ["N", "Y"] + replace_invalid: true + +remote_followup: + allowed_values: ["N", "Y"] + replace_invalid: true + +status: + # Canonical values in Title Case. Validation is case-insensitive. + # If matched, returns the canonical value (e.g., "active" → "Active") + allowed_values: + - "Active" + - "Active Remote" + - "Active Monitoring" + - "Query" + - "Inactive" + - "Transferred" + - "Lost Follow Up" + - "Deceased" + - "Discontinued" + # Canonical label -> the retired spellings it absorbs. Reports need one + # label per status, so which spelling wins is declared here rather than + # left to the order allowed_values happens to list (ticket 29). + # "Active - Remote" is how 2020-2023 trackers wrote it; the Lookup List + # dropdown introduced with the 2024 template redesign defines only + # "Active Remote", which every 2024+ tracker uses. + aliases: + "Active Remote": + - "Active - Remote" + replace_invalid: true + +support_level: + allowed_values: + - "Standard" + - "Partial" + - "Partial - A" + - "Partial - B" + - "Semi-Partial" + - "SAC" + - "Monitoring" + replace_invalid: true + +t1d_diagnosis_with_dka: + allowed_values: ["N", "Y"] + replace_invalid: true diff --git a/renv.lock b/renv.lock deleted file mode 100644 index 50e6bfc..0000000 --- a/renv.lock +++ /dev/null @@ -1,6315 +0,0 @@ -{ - "R": { - "Version": "4.5.1", - "Repositories": [ - { - "Name": "CRAN", - "URL": "https://cloud.r-project.org" - } - ] - }, - "Packages": { - "DBI": { - "Package": "DBI", - "Version": "1.2.3", - "Source": "Repository", - "Title": "R Database Interface", - "Date": "2024-06-02", - "Authors@R": "c( person(\"R Special Interest Group on Databases (R-SIG-DB)\", role = \"aut\"), person(\"Hadley\", \"Wickham\", role = \"aut\"), person(\"Kirill\", \"Müller\", , \"kirill@cynkra.com\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-1416-3412\")), person(\"R Consortium\", role = \"fnd\") )", - "Description": "A database interface definition for communication between R and relational database management systems. All classes in this package are virtual and need to be extended by the various R/DBMS implementations.", - "License": "LGPL (>= 2.1)", - "URL": "https://dbi.r-dbi.org, https://github.com/r-dbi/DBI", - "BugReports": "https://github.com/r-dbi/DBI/issues", - "Depends": [ - "methods", - "R (>= 3.0.0)" - ], - "Suggests": [ - "arrow", - "blob", - "covr", - "DBItest", - "dbplyr", - "downlit", - "dplyr", - "glue", - "hms", - "knitr", - "magrittr", - "nanoarrow (>= 0.3.0.1)", - "RMariaDB", - "rmarkdown", - "rprojroot", - "RSQLite (>= 1.1-2)", - "testthat (>= 3.0.0)", - "vctrs", - "xml2" - ], - "VignetteBuilder": "knitr", - "Config/autostyle/scope": "line_breaks", - "Config/autostyle/strict": "false", - "Config/Needs/check": "r-dbi/DBItest", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.1", - "Config/Needs/website": "r-dbi/DBItest, r-dbi/dbitemplate, adbi, AzureKusto, bigrquery, DatabaseConnector, dittodb, duckdb, implyr, lazysf, odbc, pool, RAthena, IMSMWU/RClickhouse, RH2, RJDBC, RMariaDB, RMySQL, RPostgres, RPostgreSQL, RPresto, RSQLite, sergeant, sparklyr, withr", - "Config/testthat/edition": "3", - "NeedsCompilation": "no", - "Author": "R Special Interest Group on Databases (R-SIG-DB) [aut], Hadley Wickham [aut], Kirill Müller [aut, cre] (), R Consortium [fnd]", - "Maintainer": "Kirill Müller ", - "Repository": "RSPM" - }, - "DT": { - "Package": "DT", - "Version": "0.34.0", - "Source": "Repository", - "Type": "Package", - "Title": "A Wrapper of the JavaScript Library 'DataTables'", - "Authors@R": "c( person(\"Yihui\", \"Xie\", role = \"aut\"), person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Xianying\", \"Tan\", role = \"aut\"), person(\"Garrick\", \"Aden-Buie\", , \"garrick@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-7111-0077\")), person(\"JJ\", \"Allaire\", role = \"ctb\"), person(\"Maximilian\", \"Girlich\", role = \"ctb\"), person(\"Greg\", \"Freedman Ellis\", role = \"ctb\"), person(\"Johannes\", \"Rauh\", role = \"ctb\"), person(\"SpryMedia Limited\", role = c(\"ctb\", \"cph\"), comment = \"DataTables in htmlwidgets/lib\"), person(\"Brian\", \"Reavis\", role = c(\"ctb\", \"cph\"), comment = \"selectize.js in htmlwidgets/lib\"), person(\"Leon\", \"Gersen\", role = c(\"ctb\", \"cph\"), comment = \"noUiSlider in htmlwidgets/lib\"), person(\"Bartek\", \"Szopka\", role = c(\"ctb\", \"cph\"), comment = \"jquery.highlight.js in htmlwidgets/lib\"), person(\"Alex\", \"Pickering\", role = \"ctb\"), person(\"William\", \"Holmes\", role = \"ctb\"), person(\"Mikko\", \"Marttila\", role = \"ctb\"), person(\"Andres\", \"Quintero\", role = \"ctb\"), person(\"Stéphane\", \"Laurent\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Data objects in R can be rendered as HTML tables using the JavaScript library 'DataTables' (typically via R Markdown or Shiny). The 'DataTables' library has been included in this R package. The package name 'DT' is an abbreviation of 'DataTables'.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/rstudio/DT", - "BugReports": "https://github.com/rstudio/DT/issues", - "Imports": [ - "crosstalk", - "htmltools (>= 0.3.6)", - "htmlwidgets (>= 1.3)", - "jquerylib", - "jsonlite (>= 0.9.16)", - "magrittr", - "promises" - ], - "Suggests": [ - "bslib", - "future", - "httpuv", - "knitr (>= 1.8)", - "rmarkdown", - "shiny (>= 1.6)", - "testit", - "tibble" - ], - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Yihui Xie [aut], Joe Cheng [aut], Xianying Tan [aut], Garrick Aden-Buie [aut, cre] (ORCID: ), JJ Allaire [ctb], Maximilian Girlich [ctb], Greg Freedman Ellis [ctb], Johannes Rauh [ctb], SpryMedia Limited [ctb, cph] (DataTables in htmlwidgets/lib), Brian Reavis [ctb, cph] (selectize.js in htmlwidgets/lib), Leon Gersen [ctb, cph] (noUiSlider in htmlwidgets/lib), Bartek Szopka [ctb, cph] (jquery.highlight.js in htmlwidgets/lib), Alex Pickering [ctb], William Holmes [ctb], Mikko Marttila [ctb], Andres Quintero [ctb], Stéphane Laurent [ctb], Posit Software, PBC [cph, fnd]", - "Maintainer": "Garrick Aden-Buie ", - "Repository": "CRAN" - }, - "ParallelLogger": { - "Package": "ParallelLogger", - "Version": "3.5.1", - "Source": "Repository", - "Type": "Package", - "Title": "Support for Parallel Computation, Logging, and Function Automation", - "Date": "2025-10-29", - "Authors@R": "c( person(\"Martijn\", \"Schuemie\", email = \"schuemie@ohdsi.org\", role = c(\"aut\", \"cre\")), person(\"Marc\", \"Suchard\", role = c(\"aut\")), person(\"Observational Health Data Science and Informatics\", role = c(\"cph\")) )", - "Maintainer": "Martijn Schuemie ", - "Description": "Support for parallel computation with progress bar, and option to stop or proceed on errors. Also provides logging to console and disk, and the logging persists in the parallel threads. Additional functions support function call automation with delayed execution (e.g. for executing functions in parallel).", - "License": "Apache License 2.0", - "VignetteBuilder": "knitr", - "Depends": [ - "R (>= 4.0.0)" - ], - "Imports": [ - "snow", - "xml2", - "jsonlite", - "methods", - "utils", - "rstudioapi", - "memuse", - "parallel" - ], - "Suggests": [ - "sendmailR", - "testthat", - "shiny", - "DT", - "knitr", - "rmarkdown", - "tibble" - ], - "URL": "https://ohdsi.github.io/ParallelLogger/, https://github.com/OHDSI/ParallelLogger", - "BugReports": "https://github.com/OHDSI/ParallelLogger/issues", - "NeedsCompilation": "no", - "RoxygenNote": "7.3.3", - "Encoding": "UTF-8", - "Author": "Martijn Schuemie [aut, cre], Marc Suchard [aut], Observational Health Data Science and Informatics [cph]", - "Repository": "CRAN" - }, - "R.cache": { - "Package": "R.cache", - "Version": "0.17.0", - "Source": "Repository", - "Depends": [ - "R (>= 2.14.0)" - ], - "Imports": [ - "utils", - "R.methodsS3 (>= 1.8.1)", - "R.oo (>= 1.24.0)", - "R.utils (>= 2.10.1)", - "digest (>= 0.6.13)" - ], - "Title": "Fast and Light-Weight Caching (Memoization) of Objects and Results to Speed Up Computations", - "Authors@R": "c(person(\"Henrik\", \"Bengtsson\", role=c(\"aut\", \"cre\", \"cph\"), email = \"henrikb@braju.com\"))", - "Author": "Henrik Bengtsson [aut, cre, cph]", - "Maintainer": "Henrik Bengtsson ", - "Description": "Memoization can be used to speed up repetitive and computational expensive function calls. The first time a function that implements memoization is called the results are stored in a cache memory. The next time the function is called with the same set of parameters, the results are momentarily retrieved from the cache avoiding repeating the calculations. With this package, any R object can be cached in a key-value storage where the key can be an arbitrary set of R objects. The cache memory is persistent (on the file system).", - "License": "LGPL (>= 2.1)", - "LazyLoad": "TRUE", - "URL": "https://github.com/HenrikBengtsson/R.cache", - "BugReports": "https://github.com/HenrikBengtsson/R.cache/issues", - "Encoding": "UTF-8", - "NeedsCompilation": "no", - "Repository": "CRAN" - }, - "R.methodsS3": { - "Package": "R.methodsS3", - "Version": "1.8.2", - "Source": "Repository", - "Depends": [ - "R (>= 2.13.0)" - ], - "Imports": [ - "utils" - ], - "Suggests": [ - "codetools" - ], - "Title": "S3 Methods Simplified", - "Authors@R": "c(person(\"Henrik\", \"Bengtsson\", role=c(\"aut\", \"cre\", \"cph\"), email = \"henrikb@braju.com\"))", - "Author": "Henrik Bengtsson [aut, cre, cph]", - "Maintainer": "Henrik Bengtsson ", - "Description": "Methods that simplify the setup of S3 generic functions and S3 methods. Major effort has been made in making definition of methods as simple as possible with a minimum of maintenance for package developers. For example, generic functions are created automatically, if missing, and naming conflict are automatically solved, if possible. The method setMethodS3() is a good start for those who in the future may want to migrate to S4. This is a cross-platform package implemented in pure R that generates standard S3 methods.", - "License": "LGPL (>= 2.1)", - "LazyLoad": "TRUE", - "URL": "https://github.com/HenrikBengtsson/R.methodsS3", - "BugReports": "https://github.com/HenrikBengtsson/R.methodsS3/issues", - "NeedsCompilation": "no", - "Repository": "CRAN" - }, - "R.oo": { - "Package": "R.oo", - "Version": "1.27.1", - "Source": "Repository", - "Depends": [ - "R (>= 2.13.0)", - "R.methodsS3 (>= 1.8.2)" - ], - "Imports": [ - "methods", - "utils" - ], - "Suggests": [ - "tools" - ], - "Title": "R Object-Oriented Programming with or without References", - "Authors@R": "c(person(\"Henrik\", \"Bengtsson\", role=c(\"aut\", \"cre\", \"cph\"), email = \"henrikb@braju.com\"))", - "Author": "Henrik Bengtsson [aut, cre, cph]", - "Maintainer": "Henrik Bengtsson ", - "Description": "Methods and classes for object-oriented programming in R with or without references. Large effort has been made on making definition of methods as simple as possible with a minimum of maintenance for package developers. The package has been developed since 2001 and is now considered very stable. This is a cross-platform package implemented in pure R that defines standard S3 classes without any tricks.", - "License": "LGPL (>= 2.1)", - "LazyLoad": "TRUE", - "URL": "https://henrikbengtsson.github.io/R.oo/, https://github.com/HenrikBengtsson/R.oo", - "BugReports": "https://github.com/HenrikBengtsson/R.oo/issues", - "NeedsCompilation": "no", - "Repository": "CRAN" - }, - "R.utils": { - "Package": "R.utils", - "Version": "2.13.0", - "Source": "Repository", - "Depends": [ - "R (>= 2.14.0)", - "R.oo" - ], - "Imports": [ - "methods", - "utils", - "tools", - "R.methodsS3" - ], - "Suggests": [ - "datasets", - "digest (>= 0.6.10)" - ], - "Title": "Various Programming Utilities", - "Authors@R": "c(person(\"Henrik\", \"Bengtsson\", role=c(\"aut\", \"cre\", \"cph\"), email = \"henrikb@braju.com\"))", - "Author": "Henrik Bengtsson [aut, cre, cph]", - "Maintainer": "Henrik Bengtsson ", - "Description": "Utility functions useful when programming and developing R packages.", - "License": "LGPL (>= 2.1)", - "LazyLoad": "TRUE", - "URL": "https://henrikbengtsson.github.io/R.utils/, https://github.com/HenrikBengtsson/R.utils", - "BugReports": "https://github.com/HenrikBengtsson/R.utils/issues", - "NeedsCompilation": "no", - "Repository": "CRAN" - }, - "R6": { - "Package": "R6", - "Version": "2.6.1", - "Source": "Repository", - "Title": "Encapsulated Classes with Reference Semantics", - "Authors@R": "c( person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Creates classes with reference semantics, similar to R's built-in reference classes. Compared to reference classes, R6 classes are simpler and lighter-weight, and they are not built on S4 classes so they do not require the methods package. These classes allow public and private members, and they support inheritance, even when the classes are defined in different packages.", - "License": "MIT + file LICENSE", - "URL": "https://r6.r-lib.org, https://github.com/r-lib/R6", - "BugReports": "https://github.com/r-lib/R6/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Suggests": [ - "lobstr", - "testthat (>= 3.0.0)" - ], - "Config/Needs/website": "tidyverse/tidytemplate, ggplot2, microbenchmark, scales", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Winston Chang [aut, cre], Posit Software, PBC [cph, fnd]", - "Maintainer": "Winston Chang ", - "Repository": "CRAN" - }, - "RColorBrewer": { - "Package": "RColorBrewer", - "Version": "1.1-3", - "Source": "Repository", - "Date": "2022-04-03", - "Title": "ColorBrewer Palettes", - "Authors@R": "c(person(given = \"Erich\", family = \"Neuwirth\", role = c(\"aut\", \"cre\"), email = \"erich.neuwirth@univie.ac.at\"))", - "Author": "Erich Neuwirth [aut, cre]", - "Maintainer": "Erich Neuwirth ", - "Depends": [ - "R (>= 2.0.0)" - ], - "Description": "Provides color schemes for maps (and other graphics) designed by Cynthia Brewer as described at http://colorbrewer2.org.", - "License": "Apache License 2.0", - "NeedsCompilation": "no", - "Repository": "CRAN" - }, - "Rcpp": { - "Package": "Rcpp", - "Version": "1.1.0", - "Source": "Repository", - "Title": "Seamless R and C++ Integration", - "Date": "2025-07-01", - "Authors@R": "c(person(\"Dirk\", \"Eddelbuettel\", role = c(\"aut\", \"cre\"), email = \"edd@debian.org\", comment = c(ORCID = \"0000-0001-6419-907X\")), person(\"Romain\", \"Francois\", role = \"aut\", comment = c(ORCID = \"0000-0002-2444-4226\")), person(\"JJ\", \"Allaire\", role = \"aut\", comment = c(ORCID = \"0000-0003-0174-9868\")), person(\"Kevin\", \"Ushey\", role = \"aut\", comment = c(ORCID = \"0000-0003-2880-7407\")), person(\"Qiang\", \"Kou\", role = \"aut\", comment = c(ORCID = \"0000-0001-6786-5453\")), person(\"Nathan\", \"Russell\", role = \"aut\"), person(\"Iñaki\", \"Ucar\", role = \"aut\", comment = c(ORCID = \"0000-0001-6403-5550\")), person(\"Doug\", \"Bates\", role = \"aut\", comment = c(ORCID = \"0000-0001-8316-9503\")), person(\"John\", \"Chambers\", role = \"aut\"))", - "Description": "The 'Rcpp' package provides R functions as well as C++ classes which offer a seamless integration of R and C++. Many R data types and objects can be mapped back and forth to C++ equivalents which facilitates both writing of new code as well as easier integration of third-party libraries. Documentation about 'Rcpp' is provided by several vignettes included in this package, via the 'Rcpp Gallery' site at , the paper by Eddelbuettel and Francois (2011, ), the book by Eddelbuettel (2013, ) and the paper by Eddelbuettel and Balamuta (2018, ); see 'citation(\"Rcpp\")' for details.", - "Imports": [ - "methods", - "utils" - ], - "Suggests": [ - "tinytest", - "inline", - "rbenchmark", - "pkgKitten (>= 0.1.2)" - ], - "URL": "https://www.rcpp.org, https://dirk.eddelbuettel.com/code/rcpp.html, https://github.com/RcppCore/Rcpp", - "License": "GPL (>= 2)", - "BugReports": "https://github.com/RcppCore/Rcpp/issues", - "MailingList": "rcpp-devel@lists.r-forge.r-project.org", - "RoxygenNote": "6.1.1", - "Encoding": "UTF-8", - "NeedsCompilation": "yes", - "Author": "Dirk Eddelbuettel [aut, cre] (ORCID: ), Romain Francois [aut] (ORCID: ), JJ Allaire [aut] (ORCID: ), Kevin Ushey [aut] (ORCID: ), Qiang Kou [aut] (ORCID: ), Nathan Russell [aut], Iñaki Ucar [aut] (ORCID: ), Doug Bates [aut] (ORCID: ), John Chambers [aut]", - "Maintainer": "Dirk Eddelbuettel ", - "Repository": "CRAN" - }, - "S7": { - "Package": "S7", - "Version": "0.2.0", - "Source": "Repository", - "Title": "An Object Oriented System Meant to Become a Successor to S3 and S4", - "Authors@R": "c( person(\"Object-Oriented Programming Working Group\", role = \"cph\"), person(\"Davis\", \"Vaughan\", role = \"aut\"), person(\"Jim\", \"Hester\", role = \"aut\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"Tomasz\", \"Kalinowski\", role = \"aut\"), person(\"Will\", \"Landau\", role = \"aut\"), person(\"Michael\", \"Lawrence\", role = \"aut\"), person(\"Martin\", \"Maechler\", role = \"aut\", comment = c(ORCID = \"0000-0002-8685-9910\")), person(\"Luke\", \"Tierney\", role = \"aut\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4757-117X\")) )", - "Description": "A new object oriented programming system designed to be a successor to S3 and S4. It includes formal class, generic, and method specification, and a limited form of multiple dispatch. It has been designed and implemented collaboratively by the R Consortium Object-Oriented Programming Working Group, which includes representatives from R-Core, 'Bioconductor', 'Posit'/'tidyverse', and the wider R community.", - "License": "MIT + file LICENSE", - "URL": "https://rconsortium.github.io/S7/, https://github.com/RConsortium/S7", - "BugReports": "https://github.com/RConsortium/S7/issues", - "Depends": [ - "R (>= 3.5.0)" - ], - "Imports": [ - "utils" - ], - "Suggests": [ - "bench", - "callr", - "covr", - "knitr", - "methods", - "rmarkdown", - "testthat (>= 3.2.0)", - "tibble" - ], - "VignetteBuilder": "knitr", - "Config/build/compilation-database": "true", - "Config/Needs/website": "sloop", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "TRUE", - "Config/testthat/start-first": "external-generic", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "yes", - "Author": "Object-Oriented Programming Working Group [cph], Davis Vaughan [aut], Jim Hester [aut] (), Tomasz Kalinowski [aut], Will Landau [aut], Michael Lawrence [aut], Martin Maechler [aut] (), Luke Tierney [aut], Hadley Wickham [aut, cre] ()", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "arrow": { - "Package": "arrow", - "Version": "22.0.0", - "Source": "Repository", - "Title": "Integration to 'Apache' 'Arrow'", - "Authors@R": "c( person(\"Neal\", \"Richardson\", email = \"neal.p.richardson@gmail.com\", role = c(\"aut\")), person(\"Ian\", \"Cook\", email = \"ianmcook@gmail.com\", role = c(\"aut\")), person(\"Nic\", \"Crane\", email = \"thisisnic@gmail.com\", role = c(\"aut\")), person(\"Dewey\", \"Dunnington\", role = c(\"aut\"), email = \"dewey@fishandwhistle.net\", comment = c(ORCID = \"0000-0002-9415-4582\")), person(\"Romain\", \"Fran\\u00e7ois\", role = c(\"aut\"), comment = c(ORCID = \"0000-0002-2444-4226\")), person(\"Jonathan\", \"Keane\", email = \"jkeane@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Bryce\", \"Mecum\", email = \"brycemecum@gmail.com\", role = c(\"aut\")), person(\"Drago\\u0219\", \"Moldovan-Gr\\u00fcnfeld\", email = \"dragos.mold@gmail.com\", role = c(\"aut\")), person(\"Jeroen\", \"Ooms\", email = \"jeroen@berkeley.edu\", role = c(\"aut\")), person(\"Jacob\", \"Wujciak-Jens\", email = \"jacob@wujciak.de\", role = c(\"aut\")), person(\"Javier\", \"Luraschi\", email = \"javier@rstudio.com\", role = c(\"ctb\")), person(\"Karl\", \"Dunkle Werner\", email = \"karldw@users.noreply.github.com\", role = c(\"ctb\"), comment = c(ORCID = \"0000-0003-0523-7309\")), person(\"Jeffrey\", \"Wong\", email = \"jeffreyw@netflix.com\", role = c(\"ctb\")), person(\"Apache Arrow\", email = \"dev@arrow.apache.org\", role = c(\"aut\", \"cph\")) )", - "Description": "'Apache' 'Arrow' is a cross-language development platform for in-memory data. It specifies a standardized language-independent columnar memory format for flat and hierarchical data, organized for efficient analytic operations on modern hardware. This package provides an interface to the 'Arrow C++' library.", - "Depends": [ - "R (>= 4.1)" - ], - "License": "Apache License (>= 2.0)", - "URL": "https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/", - "BugReports": "https://github.com/apache/arrow/issues", - "Encoding": "UTF-8", - "Language": "en-US", - "SystemRequirements": "C++17; for AWS S3 support on Linux, libcurl and openssl (optional); cmake >= 3.26 (build-time only, and only for full source build)", - "Biarch": "true", - "Imports": [ - "assertthat", - "bit64 (>= 0.9-7)", - "glue", - "methods", - "purrr", - "R6", - "rlang (>= 1.0.0)", - "stats", - "tidyselect (>= 1.0.0)", - "utils", - "vctrs" - ], - "RoxygenNote": "7.3.3", - "Config/testthat/edition": "3", - "Config/build/bootstrap": "TRUE", - "Suggests": [ - "blob", - "curl", - "cli", - "DBI", - "dbplyr", - "decor", - "distro", - "dplyr", - "duckdb (>= 0.2.8)", - "hms", - "jsonlite", - "knitr", - "lubridate", - "pillar", - "pkgload", - "reticulate", - "rmarkdown", - "stringi", - "stringr", - "sys", - "testthat (>= 3.1.0)", - "tibble", - "tzdb", - "withr" - ], - "LinkingTo": [ - "cpp11 (>= 0.4.2)" - ], - "Collate": "'arrowExports.R' 'enums.R' 'arrow-object.R' 'type.R' 'array-data.R' 'arrow-datum.R' 'array.R' 'arrow-info.R' 'arrow-package.R' 'arrow-tabular.R' 'buffer.R' 'chunked-array.R' 'io.R' 'compression.R' 'scalar.R' 'compute.R' 'config.R' 'csv.R' 'dataset.R' 'dataset-factory.R' 'dataset-format.R' 'dataset-partition.R' 'dataset-scan.R' 'dataset-write.R' 'dictionary.R' 'dplyr-across.R' 'dplyr-arrange.R' 'dplyr-by.R' 'dplyr-collect.R' 'dplyr-count.R' 'dplyr-datetime-helpers.R' 'dplyr-distinct.R' 'dplyr-eval.R' 'dplyr-filter.R' 'dplyr-funcs-agg.R' 'dplyr-funcs-augmented.R' 'dplyr-funcs-conditional.R' 'dplyr-funcs-datetime.R' 'dplyr-funcs-doc.R' 'dplyr-funcs-math.R' 'dplyr-funcs-simple.R' 'dplyr-funcs-string.R' 'dplyr-funcs-type.R' 'expression.R' 'dplyr-funcs.R' 'dplyr-glimpse.R' 'dplyr-group-by.R' 'dplyr-join.R' 'dplyr-mutate.R' 'dplyr-select.R' 'dplyr-slice.R' 'dplyr-summarize.R' 'dplyr-union.R' 'record-batch.R' 'table.R' 'dplyr.R' 'duckdb.R' 'extension.R' 'feather.R' 'field.R' 'filesystem.R' 'flight.R' 'install-arrow.R' 'ipc-stream.R' 'json.R' 'memory-pool.R' 'message.R' 'metadata.R' 'parquet.R' 'python.R' 'query-engine.R' 'record-batch-reader.R' 'record-batch-writer.R' 'reexports-bit64.R' 'reexports-tidyselect.R' 'schema.R' 'udf.R' 'util.R'", - "NeedsCompilation": "yes", - "Author": "Neal Richardson [aut], Ian Cook [aut], Nic Crane [aut], Dewey Dunnington [aut] (ORCID: ), Romain François [aut] (ORCID: ), Jonathan Keane [aut, cre], Bryce Mecum [aut], Dragoș Moldovan-Grünfeld [aut], Jeroen Ooms [aut], Jacob Wujciak-Jens [aut], Javier Luraschi [ctb], Karl Dunkle Werner [ctb] (ORCID: ), Jeffrey Wong [ctb], Apache Arrow [aut, cph]", - "Maintainer": "Jonathan Keane ", - "Repository": "CRAN" - }, - "askpass": { - "Package": "askpass", - "Version": "1.2.1", - "Source": "Repository", - "Type": "Package", - "Title": "Password Entry Utilities for R, Git, and SSH", - "Authors@R": "person(\"Jeroen\", \"Ooms\", role = c(\"aut\", \"cre\"), email = \"jeroenooms@gmail.com\", comment = c(ORCID = \"0000-0002-4035-0289\"))", - "Description": "Cross-platform utilities for prompting the user for credentials or a passphrase, for example to authenticate with a server or read a protected key. Includes native programs for MacOS and Windows, hence no 'tcltk' is required. Password entry can be invoked in two different ways: directly from R via the askpass() function, or indirectly as password-entry back-end for 'ssh-agent' or 'git-credential' via the SSH_ASKPASS and GIT_ASKPASS environment variables. Thereby the user can be prompted for credentials or a passphrase if needed when R calls out to git or ssh.", - "License": "MIT + file LICENSE", - "URL": "https://r-lib.r-universe.dev/askpass", - "BugReports": "https://github.com/r-lib/askpass/issues", - "Encoding": "UTF-8", - "Imports": [ - "sys (>= 2.1)" - ], - "RoxygenNote": "7.2.3", - "Suggests": [ - "testthat" - ], - "Language": "en-US", - "NeedsCompilation": "yes", - "Author": "Jeroen Ooms [aut, cre] ()", - "Maintainer": "Jeroen Ooms ", - "Repository": "CRAN" - }, - "assertthat": { - "Package": "assertthat", - "Version": "0.2.1", - "Source": "Repository", - "Title": "Easy Pre and Post Assertions", - "Authors@R": "person(\"Hadley\", \"Wickham\", , \"hadley@rstudio.com\", c(\"aut\", \"cre\"))", - "Description": "An extension to stopifnot() that makes it easy to declare the pre and post conditions that you code should satisfy, while also producing friendly error messages so that your users know what's gone wrong.", - "License": "GPL-3", - "Imports": [ - "tools" - ], - "Suggests": [ - "testthat", - "covr" - ], - "RoxygenNote": "6.0.1", - "Collate": "'assert-that.r' 'on-failure.r' 'assertions-file.r' 'assertions-scalar.R' 'assertions.r' 'base.r' 'base-comparison.r' 'base-is.r' 'base-logical.r' 'base-misc.r' 'utils.r' 'validate-that.R'", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre]", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "backports": { - "Package": "backports", - "Version": "1.5.0", - "Source": "Repository", - "Type": "Package", - "Title": "Reimplementations of Functions Introduced Since R-3.0.0", - "Authors@R": "c( person(\"Michel\", \"Lang\", NULL, \"michellang@gmail.com\", role = c(\"cre\", \"aut\"), comment = c(ORCID = \"0000-0001-9754-0393\")), person(\"Duncan\", \"Murdoch\", NULL, \"murdoch.duncan@gmail.com\", role = c(\"aut\")), person(\"R Core Team\", role = \"aut\"))", - "Maintainer": "Michel Lang ", - "Description": "Functions introduced or changed since R v3.0.0 are re-implemented in this package. The backports are conditionally exported in order to let R resolve the function name to either the implemented backport, or the respective base version, if available. Package developers can make use of new functions or arguments by selectively importing specific backports to support older installations.", - "URL": "https://github.com/r-lib/backports", - "BugReports": "https://github.com/r-lib/backports/issues", - "License": "GPL-2 | GPL-3", - "NeedsCompilation": "yes", - "ByteCompile": "yes", - "Depends": [ - "R (>= 3.0.0)" - ], - "Encoding": "UTF-8", - "RoxygenNote": "7.3.1", - "Author": "Michel Lang [cre, aut] (), Duncan Murdoch [aut], R Core Team [aut]", - "Repository": "RSPM" - }, - "base64enc": { - "Package": "base64enc", - "Version": "0.1-3", - "Source": "Repository", - "Title": "Tools for base64 encoding", - "Author": "Simon Urbanek ", - "Maintainer": "Simon Urbanek ", - "Depends": [ - "R (>= 2.9.0)" - ], - "Enhances": [ - "png" - ], - "Description": "This package provides tools for handling base64 encoding. It is more flexible than the orphaned base64 package.", - "License": "GPL-2 | GPL-3", - "URL": "http://www.rforge.net/base64enc", - "NeedsCompilation": "yes", - "Repository": "CRAN" - }, - "bit": { - "Package": "bit", - "Version": "4.6.0", - "Source": "Repository", - "Title": "Classes and Methods for Fast Memory-Efficient Boolean Selections", - "Authors@R": "c( person(\"Michael\", \"Chirico\", email = \"MichaelChirico4@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Jens\", \"Oehlschlägel\", role = \"aut\"), person(\"Brian\", \"Ripley\", role = \"ctb\") )", - "Depends": [ - "R (>= 3.4.0)" - ], - "Suggests": [ - "testthat (>= 3.0.0)", - "roxygen2", - "knitr", - "markdown", - "rmarkdown", - "microbenchmark", - "bit64 (>= 4.0.0)", - "ff (>= 4.0.0)" - ], - "Description": "Provided are classes for boolean and skewed boolean vectors, fast boolean methods, fast unique and non-unique integer sorting, fast set operations on sorted and unsorted sets of integers, and foundations for ff (range index, compression, chunked processing).", - "License": "GPL-2 | GPL-3", - "LazyLoad": "yes", - "ByteCompile": "yes", - "Encoding": "UTF-8", - "URL": "https://github.com/r-lib/bit", - "VignetteBuilder": "knitr, rmarkdown", - "RoxygenNote": "7.3.2", - "Config/testthat/edition": "3", - "NeedsCompilation": "yes", - "Author": "Michael Chirico [aut, cre], Jens Oehlschlägel [aut], Brian Ripley [ctb]", - "Maintainer": "Michael Chirico ", - "Repository": "CRAN" - }, - "bit64": { - "Package": "bit64", - "Version": "4.6.0-1", - "Source": "Repository", - "Title": "A S3 Class for Vectors of 64bit Integers", - "Authors@R": "c( person(\"Michael\", \"Chirico\", email = \"michaelchirico4@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Jens\", \"Oehlschlägel\", role = \"aut\"), person(\"Leonardo\", \"Silvestri\", role = \"ctb\"), person(\"Ofek\", \"Shilon\", role = \"ctb\") )", - "Depends": [ - "R (>= 3.4.0)", - "bit (>= 4.0.0)" - ], - "Description": "Package 'bit64' provides serializable S3 atomic 64bit (signed) integers. These are useful for handling database keys and exact counting in +-2^63. WARNING: do not use them as replacement for 32bit integers, integer64 are not supported for subscripting by R-core and they have different semantics when combined with double, e.g. integer64 + double => integer64. Class integer64 can be used in vectors, matrices, arrays and data.frames. Methods are available for coercion from and to logicals, integers, doubles, characters and factors as well as many elementwise and summary functions. Many fast algorithmic operations such as 'match' and 'order' support inter- active data exploration and manipulation and optionally leverage caching.", - "License": "GPL-2 | GPL-3", - "LazyLoad": "yes", - "ByteCompile": "yes", - "URL": "https://github.com/r-lib/bit64", - "Encoding": "UTF-8", - "Imports": [ - "graphics", - "methods", - "stats", - "utils" - ], - "Suggests": [ - "testthat (>= 3.0.3)", - "withr" - ], - "Config/testthat/edition": "3", - "Config/needs/development": "testthat", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "yes", - "Author": "Michael Chirico [aut, cre], Jens Oehlschlägel [aut], Leonardo Silvestri [ctb], Ofek Shilon [ctb]", - "Maintainer": "Michael Chirico ", - "Repository": "CRAN" - }, - "blob": { - "Package": "blob", - "Version": "1.2.4", - "Source": "Repository", - "Title": "A Simple S3 Class for Representing Vectors of Binary Data ('BLOBS')", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", role = \"aut\"), person(\"Kirill\", \"Müller\", , \"kirill@cynkra.com\", role = \"cre\"), person(\"RStudio\", role = c(\"cph\", \"fnd\")) )", - "Description": "R's raw vector is useful for storing a single binary object. What if you want to put a vector of them in a data frame? The 'blob' package provides the blob object, a list of raw vectors, suitable for use as a column in data frame.", - "License": "MIT + file LICENSE", - "URL": "https://blob.tidyverse.org, https://github.com/tidyverse/blob", - "BugReports": "https://github.com/tidyverse/blob/issues", - "Imports": [ - "methods", - "rlang", - "vctrs (>= 0.2.1)" - ], - "Suggests": [ - "covr", - "crayon", - "pillar (>= 1.2.1)", - "testthat" - ], - "Config/autostyle/scope": "line_breaks", - "Config/autostyle/strict": "false", - "Config/Needs/website": "tidyverse/tidytemplate", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.3", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut], Kirill Müller [cre], RStudio [cph, fnd]", - "Maintainer": "Kirill Müller ", - "Repository": "RSPM" - }, - "brew": { - "Package": "brew", - "Version": "1.0-10", - "Source": "Repository", - "Type": "Package", - "Title": "Templating Framework for Report Generation", - "Authors@R": "c( person(\"Jeffrey\", \"Horner\", role = c(\"aut\", \"cph\")), person(\"Greg\", \"Hunt\", , \"greg@firmansyah.com\", role = c(\"aut\", \"cre\", \"cph\")) )", - "Description": "Implements a templating framework for mixing text and R code for report generation. brew template syntax is similar to PHP, Ruby's erb module, Java Server Pages, and Python's psp module.", - "License": "GPL (>= 2)", - "URL": "https://github.com/gregfrog/brew", - "BugReports": "https://github.com/gregfrog/brew/issues", - "Suggests": [ - "testthat (>= 3.0.0)" - ], - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "Repository": "RSPM", - "NeedsCompilation": "no", - "Author": "Jeffrey Horner [aut, cph], Greg Hunt [aut, cre, cph]", - "Maintainer": "Greg Hunt " - }, - "brio": { - "Package": "brio", - "Version": "1.1.5", - "Source": "Repository", - "Title": "Basic R Input Output", - "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Functions to handle basic input output, these functions always read and write UTF-8 (8-bit Unicode Transformation Format) files and provide more explicit control over line endings.", - "License": "MIT + file LICENSE", - "URL": "https://brio.r-lib.org, https://github.com/r-lib/brio", - "BugReports": "https://github.com/r-lib/brio/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Suggests": [ - "covr", - "testthat (>= 3.0.0)" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.3", - "NeedsCompilation": "yes", - "Author": "Jim Hester [aut] (), Gábor Csárdi [aut, cre], Posit Software, PBC [cph, fnd]", - "Maintainer": "Gábor Csárdi ", - "Repository": "RSPM" - }, - "broom": { - "Package": "broom", - "Version": "1.0.10", - "Source": "Repository", - "Type": "Package", - "Title": "Convert Statistical Objects into Tidy Tibbles", - "Authors@R": "c( person(\"David\", \"Robinson\", , \"admiral.david@gmail.com\", role = \"aut\"), person(\"Alex\", \"Hayes\", , \"alexpghayes@gmail.com\", role = \"aut\", comment = c(ORCID = \"0000-0002-4985-5160\")), person(\"Simon\", \"Couch\", , \"simon.couch@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0001-5676-5107\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")), person(\"Indrajeet\", \"Patil\", , \"patilindrajeet.science@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0003-1995-6531\")), person(\"Derek\", \"Chiu\", , \"dchiu@bccrc.ca\", role = \"ctb\"), person(\"Matthieu\", \"Gomez\", , \"mattg@princeton.edu\", role = \"ctb\"), person(\"Boris\", \"Demeshev\", , \"boris.demeshev@gmail.com\", role = \"ctb\"), person(\"Dieter\", \"Menne\", , \"dieter.menne@menne-biomed.de\", role = \"ctb\"), person(\"Benjamin\", \"Nutter\", , \"nutter@battelle.org\", role = \"ctb\"), person(\"Luke\", \"Johnston\", , \"luke.johnston@mail.utoronto.ca\", role = \"ctb\"), person(\"Ben\", \"Bolker\", , \"bolker@mcmaster.ca\", role = \"ctb\"), person(\"Francois\", \"Briatte\", , \"f.briatte@gmail.com\", role = \"ctb\"), person(\"Jeffrey\", \"Arnold\", , \"jeffrey.arnold@gmail.com\", role = \"ctb\"), person(\"Jonah\", \"Gabry\", , \"jsg2201@columbia.edu\", role = \"ctb\"), person(\"Luciano\", \"Selzer\", , \"luciano.selzer@gmail.com\", role = \"ctb\"), person(\"Gavin\", \"Simpson\", , \"ucfagls@gmail.com\", role = \"ctb\"), person(\"Jens\", \"Preussner\", , \"jens.preussner@mpi-bn.mpg.de\", role = \"ctb\"), person(\"Jay\", \"Hesselberth\", , \"jay.hesselberth@gmail.com\", role = \"ctb\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"ctb\"), person(\"Matthew\", \"Lincoln\", , \"matthew.d.lincoln@gmail.com\", role = \"ctb\"), person(\"Alessandro\", \"Gasparini\", , \"ag475@leicester.ac.uk\", role = \"ctb\"), person(\"Lukasz\", \"Komsta\", , \"lukasz.komsta@umlub.pl\", role = \"ctb\"), person(\"Frederick\", \"Novometsky\", role = \"ctb\"), person(\"Wilson\", \"Freitas\", role = \"ctb\"), person(\"Michelle\", \"Evans\", role = \"ctb\"), person(\"Jason Cory\", \"Brunson\", , \"cornelioid@gmail.com\", role = \"ctb\"), person(\"Simon\", \"Jackson\", , \"drsimonjackson@gmail.com\", role = \"ctb\"), person(\"Ben\", \"Whalley\", , \"ben.whalley@plymouth.ac.uk\", role = \"ctb\"), person(\"Karissa\", \"Whiting\", , \"karissa.whiting@gmail.com\", role = \"ctb\"), person(\"Yves\", \"Rosseel\", , \"yrosseel@gmail.com\", role = \"ctb\"), person(\"Michael\", \"Kuehn\", , \"mkuehn10@gmail.com\", role = \"ctb\"), person(\"Jorge\", \"Cimentada\", , \"cimentadaj@gmail.com\", role = \"ctb\"), person(\"Erle\", \"Holgersen\", , \"erle.holgersen@gmail.com\", role = \"ctb\"), person(\"Karl\", \"Dunkle Werner\", role = \"ctb\", comment = c(ORCID = \"0000-0003-0523-7309\")), person(\"Ethan\", \"Christensen\", , \"christensen.ej@gmail.com\", role = \"ctb\"), person(\"Steven\", \"Pav\", , \"shabbychef@gmail.com\", role = \"ctb\"), person(\"Paul\", \"PJ\", , \"pjpaul.stephens@gmail.com\", role = \"ctb\"), person(\"Ben\", \"Schneider\", , \"benjamin.julius.schneider@gmail.com\", role = \"ctb\"), person(\"Patrick\", \"Kennedy\", , \"pkqstr@protonmail.com\", role = \"ctb\"), person(\"Lily\", \"Medina\", , \"lilymiru@gmail.com\", role = \"ctb\"), person(\"Brian\", \"Fannin\", , \"captain@pirategrunt.com\", role = \"ctb\"), person(\"Jason\", \"Muhlenkamp\", , \"jason.muhlenkamp@gmail.com\", role = \"ctb\"), person(\"Matt\", \"Lehman\", role = \"ctb\"), person(\"Bill\", \"Denney\", , \"wdenney@humanpredictions.com\", role = \"ctb\", comment = c(ORCID = \"0000-0002-5759-428X\")), person(\"Nic\", \"Crane\", role = \"ctb\"), person(\"Andrew\", \"Bates\", role = \"ctb\"), person(\"Vincent\", \"Arel-Bundock\", , \"vincent.arel-bundock@umontreal.ca\", role = \"ctb\", comment = c(ORCID = \"0000-0003-2042-7063\")), person(\"Hideaki\", \"Hayashi\", role = \"ctb\"), person(\"Luis\", \"Tobalina\", role = \"ctb\"), person(\"Annie\", \"Wang\", , \"anniewang.uc@gmail.com\", role = \"ctb\"), person(\"Wei Yang\", \"Tham\", , \"weiyang.tham@gmail.com\", role = \"ctb\"), person(\"Clara\", \"Wang\", , \"clara.wang.94@gmail.com\", role = \"ctb\"), person(\"Abby\", \"Smith\", , \"als1@u.northwestern.edu\", role = \"ctb\", comment = c(ORCID = \"0000-0002-3207-0375\")), person(\"Jasper\", \"Cooper\", , \"jaspercooper@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0002-8639-3188\")), person(\"E Auden\", \"Krauska\", , \"krauskae@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0002-1466-5850\")), person(\"Alex\", \"Wang\", , \"x249wang@uwaterloo.ca\", role = \"ctb\"), person(\"Malcolm\", \"Barrett\", , \"malcolmbarrett@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0003-0299-5825\")), person(\"Charles\", \"Gray\", , \"charlestigray@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0002-9978-011X\")), person(\"Jared\", \"Wilber\", role = \"ctb\"), person(\"Vilmantas\", \"Gegzna\", , \"GegznaV@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0002-9500-5167\")), person(\"Eduard\", \"Szoecs\", , \"eduardszoecs@gmail.com\", role = \"ctb\"), person(\"Frederik\", \"Aust\", , \"frederik.aust@uni-koeln.de\", role = \"ctb\", comment = c(ORCID = \"0000-0003-4900-788X\")), person(\"Angus\", \"Moore\", , \"angusmoore9@gmail.com\", role = \"ctb\"), person(\"Nick\", \"Williams\", , \"ntwilliams.personal@gmail.com\", role = \"ctb\"), person(\"Marius\", \"Barth\", , \"marius.barth.uni.koeln@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0002-3421-6665\")), person(\"Bruna\", \"Wundervald\", , \"brunadaviesw@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0001-8163-220X\")), person(\"Joyce\", \"Cahoon\", , \"joyceyu48@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0001-7217-4702\")), person(\"Grant\", \"McDermott\", , \"grantmcd@uoregon.edu\", role = \"ctb\", comment = c(ORCID = \"0000-0001-7883-8573\")), person(\"Kevin\", \"Zarca\", , \"kevin.zarca@gmail.com\", role = \"ctb\"), person(\"Shiro\", \"Kuriwaki\", , \"shirokuriwaki@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0002-5687-2647\")), person(\"Lukas\", \"Wallrich\", , \"lukas.wallrich@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0003-2121-5177\")), person(\"James\", \"Martherus\", , \"james@martherus.com\", role = \"ctb\", comment = c(ORCID = \"0000-0002-8285-3300\")), person(\"Chuliang\", \"Xiao\", , \"cxiao@umich.edu\", role = \"ctb\", comment = c(ORCID = \"0000-0002-8466-9398\")), person(\"Joseph\", \"Larmarange\", , \"joseph@larmarange.net\", role = \"ctb\"), person(\"Max\", \"Kuhn\", , \"max@posit.co\", role = \"ctb\"), person(\"Michal\", \"Bojanowski\", , \"michal2992@gmail.com\", role = \"ctb\"), person(\"Hakon\", \"Malmedal\", , \"hmalmedal@gmail.com\", role = \"ctb\"), person(\"Clara\", \"Wang\", role = \"ctb\"), person(\"Sergio\", \"Oller\", , \"sergioller@gmail.com\", role = \"ctb\"), person(\"Luke\", \"Sonnet\", , \"luke.sonnet@gmail.com\", role = \"ctb\"), person(\"Jim\", \"Hester\", , \"jim.hester@posit.co\", role = \"ctb\"), person(\"Ben\", \"Schneider\", , \"benjamin.julius.schneider@gmail.com\", role = \"ctb\"), person(\"Bernie\", \"Gray\", , \"bfgray3@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0001-9190-6032\")), person(\"Mara\", \"Averick\", , \"mara@posit.co\", role = \"ctb\"), person(\"Aaron\", \"Jacobs\", , \"atheriel@gmail.com\", role = \"ctb\"), person(\"Andreas\", \"Bender\", , \"bender.at.R@gmail.com\", role = \"ctb\"), person(\"Sven\", \"Templer\", , \"sven.templer@gmail.com\", role = \"ctb\"), person(\"Paul-Christian\", \"Buerkner\", , \"paul.buerkner@gmail.com\", role = \"ctb\"), person(\"Matthew\", \"Kay\", , \"mjskay@umich.edu\", role = \"ctb\"), person(\"Erwan\", \"Le Pennec\", , \"lepennec@gmail.com\", role = \"ctb\"), person(\"Johan\", \"Junkka\", , \"johan.junkka@umu.se\", role = \"ctb\"), person(\"Hao\", \"Zhu\", , \"haozhu233@gmail.com\", role = \"ctb\"), person(\"Benjamin\", \"Soltoff\", , \"soltoffbc@uchicago.edu\", role = \"ctb\"), person(\"Zoe\", \"Wilkinson Saldana\", , \"zoewsaldana@gmail.com\", role = \"ctb\"), person(\"Tyler\", \"Littlefield\", , \"tylurp1@gmail.com\", role = \"ctb\"), person(\"Charles T.\", \"Gray\", , \"charlestigray@gmail.com\", role = \"ctb\"), person(\"Shabbh E.\", \"Banks\", role = \"ctb\"), person(\"Serina\", \"Robinson\", , \"robi0916@umn.edu\", role = \"ctb\"), person(\"Roger\", \"Bivand\", , \"Roger.Bivand@nhh.no\", role = \"ctb\"), person(\"Riinu\", \"Ots\", , \"riinuots@gmail.com\", role = \"ctb\"), person(\"Nicholas\", \"Williams\", , \"ntwilliams.personal@gmail.com\", role = \"ctb\"), person(\"Nina\", \"Jakobsen\", role = \"ctb\"), person(\"Michael\", \"Weylandt\", , \"michael.weylandt@gmail.com\", role = \"ctb\"), person(\"Lisa\", \"Lendway\", , \"llendway@macalester.edu\", role = \"ctb\"), person(\"Karl\", \"Hailperin\", , \"khailper@gmail.com\", role = \"ctb\"), person(\"Josue\", \"Rodriguez\", , \"jerrodriguez@ucdavis.edu\", role = \"ctb\"), person(\"Jenny\", \"Bryan\", , \"jenny@posit.co\", role = \"ctb\"), person(\"Chris\", \"Jarvis\", , \"Christopher1.jarvis@gmail.com\", role = \"ctb\"), person(\"Greg\", \"Macfarlane\", , \"gregmacfarlane@gmail.com\", role = \"ctb\"), person(\"Brian\", \"Mannakee\", , \"bmannakee@gmail.com\", role = \"ctb\"), person(\"Drew\", \"Tyre\", , \"atyre2@unl.edu\", role = \"ctb\"), person(\"Shreyas\", \"Singh\", , \"shreyas.singh.298@gmail.com\", role = \"ctb\"), person(\"Laurens\", \"Geffert\", , \"laurensgeffert@gmail.com\", role = \"ctb\"), person(\"Hong\", \"Ooi\", , \"hongooi@microsoft.com\", role = \"ctb\"), person(\"Henrik\", \"Bengtsson\", , \"henrikb@braju.com\", role = \"ctb\"), person(\"Eduard\", \"Szocs\", , \"eduardszoecs@gmail.com\", role = \"ctb\"), person(\"David\", \"Hugh-Jones\", , \"davidhughjones@gmail.com\", role = \"ctb\"), person(\"Matthieu\", \"Stigler\", , \"Matthieu.Stigler@gmail.com\", role = \"ctb\"), person(\"Hugo\", \"Tavares\", , \"hm533@cam.ac.uk\", role = \"ctb\", comment = c(ORCID = \"0000-0001-9373-2726\")), person(\"R. Willem\", \"Vervoort\", , \"Willemvervoort@gmail.com\", role = \"ctb\"), person(\"Brenton M.\", \"Wiernik\", , \"brenton@wiernik.org\", role = \"ctb\"), person(\"Josh\", \"Yamamoto\", , \"joshuayamamoto5@gmail.com\", role = \"ctb\"), person(\"Jasme\", \"Lee\", role = \"ctb\"), person(\"Taren\", \"Sanders\", , \"taren.sanders@acu.edu.au\", role = \"ctb\", comment = c(ORCID = \"0000-0002-4504-6008\")), person(\"Ilaria\", \"Prosdocimi\", , \"prosdocimi.ilaria@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0001-8565-094X\")), person(\"Daniel D.\", \"Sjoberg\", , \"danield.sjoberg@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0003-0862-2018\")), person(\"Alex\", \"Reinhart\", , \"areinhar@stat.cmu.edu\", role = \"ctb\", comment = c(ORCID = \"0000-0002-6658-514X\")) )", - "Description": "Summarizes key information about statistical objects in tidy tibbles. This makes it easy to report results, create plots and consistently work with large numbers of models at once. Broom provides three verbs that each provide different types of information about a model. tidy() summarizes information about model components such as coefficients of a regression. glance() reports information about an entire model, such as goodness of fit measures like AIC and BIC. augment() adds information about individual observations to a dataset, such as fitted values or influence measures.", - "License": "MIT + file LICENSE", - "URL": "https://broom.tidymodels.org/, https://github.com/tidymodels/broom", - "BugReports": "https://github.com/tidymodels/broom/issues", - "Depends": [ - "R (>= 4.1)" - ], - "Imports": [ - "backports", - "cli", - "dplyr (>= 1.0.0)", - "generics (>= 0.0.2)", - "glue", - "lifecycle", - "purrr", - "rlang (>= 1.1.0)", - "stringr", - "tibble (>= 3.0.0)", - "tidyr (>= 1.0.0)" - ], - "Suggests": [ - "AER", - "AUC", - "bbmle", - "betareg (>= 3.2-1)", - "biglm", - "binGroup", - "boot", - "btergm (>= 1.10.6)", - "car (>= 3.1-2)", - "carData", - "caret", - "cluster", - "cmprsk", - "coda", - "covr", - "drc", - "e1071", - "emmeans", - "epiR (>= 2.0.85)", - "ergm (>= 3.10.4)", - "fixest (>= 0.9.0)", - "gam (>= 1.15)", - "gee", - "geepack", - "ggplot2", - "glmnet", - "glmnetUtils", - "gmm", - "Hmisc", - "interp", - "irlba", - "joineRML", - "Kendall", - "knitr", - "ks", - "Lahman", - "lavaan (>= 0.6.18)", - "leaps", - "lfe", - "lm.beta", - "lme4", - "lmodel2", - "lmtest (>= 0.9.38)", - "lsmeans", - "maps", - "margins", - "MASS", - "mclust", - "mediation", - "metafor", - "mfx", - "mgcv", - "mlogit", - "modeldata", - "modeltests (>= 0.1.6)", - "muhaz", - "multcomp", - "network", - "nnet", - "ordinal", - "plm", - "poLCA", - "psych", - "quantreg", - "rmarkdown", - "robust", - "robustbase", - "rsample", - "sandwich", - "spatialreg", - "spdep (>= 1.1)", - "speedglm", - "spelling", - "stats4", - "survey", - "survival (>= 3.6-4)", - "systemfit", - "testthat (>= 3.0.0)", - "tseries", - "vars", - "zoo" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/usethis/last-upkeep": "2025-04-25", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.3", - "Collate": "'aaa-documentation-helper.R' 'null-and-default.R' 'aer.R' 'auc.R' 'base.R' 'bbmle.R' 'betareg.R' 'biglm.R' 'bingroup.R' 'boot.R' 'broom-package.R' 'broom.R' 'btergm.R' 'car.R' 'caret.R' 'cluster.R' 'cmprsk.R' 'data-frame.R' 'deprecated-0-7-0.R' 'drc.R' 'emmeans.R' 'epiR.R' 'ergm.R' 'fixest.R' 'gam.R' 'geepack.R' 'glmnet-cv-glmnet.R' 'glmnet-glmnet.R' 'gmm.R' 'hmisc.R' 'import-standalone-obj-type.R' 'import-standalone-types-check.R' 'joinerml.R' 'kendall.R' 'ks.R' 'lavaan.R' 'leaps.R' 'lfe.R' 'list-irlba.R' 'list-optim.R' 'list-svd.R' 'list-xyz.R' 'list.R' 'lm-beta.R' 'lmodel2.R' 'lmtest.R' 'maps.R' 'margins.R' 'mass-fitdistr.R' 'mass-negbin.R' 'mass-polr.R' 'mass-ridgelm.R' 'stats-lm.R' 'mass-rlm.R' 'mclust.R' 'mediation.R' 'metafor.R' 'mfx.R' 'mgcv.R' 'mlogit.R' 'muhaz.R' 'multcomp.R' 'nnet.R' 'nobs.R' 'ordinal-clm.R' 'ordinal-clmm.R' 'plm.R' 'polca.R' 'psych.R' 'stats-nls.R' 'quantreg-nlrq.R' 'quantreg-rq.R' 'quantreg-rqs.R' 'robust-glmrob.R' 'robust-lmrob.R' 'robustbase-glmrob.R' 'robustbase-lmrob.R' 'sp.R' 'spdep.R' 'speedglm-speedglm.R' 'speedglm-speedlm.R' 'stats-anova.R' 'stats-arima.R' 'stats-decompose.R' 'stats-factanal.R' 'stats-glm.R' 'stats-htest.R' 'stats-kmeans.R' 'stats-loess.R' 'stats-mlm.R' 'stats-prcomp.R' 'stats-smooth.spline.R' 'stats-summary-lm.R' 'stats-time-series.R' 'survey.R' 'survival-aareg.R' 'survival-cch.R' 'survival-coxph.R' 'survival-pyears.R' 'survival-survdiff.R' 'survival-survexp.R' 'survival-survfit.R' 'survival-survreg.R' 'systemfit.R' 'tseries.R' 'utilities.R' 'vars.R' 'zoo.R' 'zzz.R'", - "NeedsCompilation": "no", - "Author": "David Robinson [aut], Alex Hayes [aut] (ORCID: ), Simon Couch [aut, cre] (ORCID: ), Posit Software, PBC [cph, fnd] (ROR: ), Indrajeet Patil [ctb] (ORCID: ), Derek Chiu [ctb], Matthieu Gomez [ctb], Boris Demeshev [ctb], Dieter Menne [ctb], Benjamin Nutter [ctb], Luke Johnston [ctb], Ben Bolker [ctb], Francois Briatte [ctb], Jeffrey Arnold [ctb], Jonah Gabry [ctb], Luciano Selzer [ctb], Gavin Simpson [ctb], Jens Preussner [ctb], Jay Hesselberth [ctb], Hadley Wickham [ctb], Matthew Lincoln [ctb], Alessandro Gasparini [ctb], Lukasz Komsta [ctb], Frederick Novometsky [ctb], Wilson Freitas [ctb], Michelle Evans [ctb], Jason Cory Brunson [ctb], Simon Jackson [ctb], Ben Whalley [ctb], Karissa Whiting [ctb], Yves Rosseel [ctb], Michael Kuehn [ctb], Jorge Cimentada [ctb], Erle Holgersen [ctb], Karl Dunkle Werner [ctb] (ORCID: ), Ethan Christensen [ctb], Steven Pav [ctb], Paul PJ [ctb], Ben Schneider [ctb], Patrick Kennedy [ctb], Lily Medina [ctb], Brian Fannin [ctb], Jason Muhlenkamp [ctb], Matt Lehman [ctb], Bill Denney [ctb] (ORCID: ), Nic Crane [ctb], Andrew Bates [ctb], Vincent Arel-Bundock [ctb] (ORCID: ), Hideaki Hayashi [ctb], Luis Tobalina [ctb], Annie Wang [ctb], Wei Yang Tham [ctb], Clara Wang [ctb], Abby Smith [ctb] (ORCID: ), Jasper Cooper [ctb] (ORCID: ), E Auden Krauska [ctb] (ORCID: ), Alex Wang [ctb], Malcolm Barrett [ctb] (ORCID: ), Charles Gray [ctb] (ORCID: ), Jared Wilber [ctb], Vilmantas Gegzna [ctb] (ORCID: ), Eduard Szoecs [ctb], Frederik Aust [ctb] (ORCID: ), Angus Moore [ctb], Nick Williams [ctb], Marius Barth [ctb] (ORCID: ), Bruna Wundervald [ctb] (ORCID: ), Joyce Cahoon [ctb] (ORCID: ), Grant McDermott [ctb] (ORCID: ), Kevin Zarca [ctb], Shiro Kuriwaki [ctb] (ORCID: ), Lukas Wallrich [ctb] (ORCID: ), James Martherus [ctb] (ORCID: ), Chuliang Xiao [ctb] (ORCID: ), Joseph Larmarange [ctb], Max Kuhn [ctb], Michal Bojanowski [ctb], Hakon Malmedal [ctb], Clara Wang [ctb], Sergio Oller [ctb], Luke Sonnet [ctb], Jim Hester [ctb], Ben Schneider [ctb], Bernie Gray [ctb] (ORCID: ), Mara Averick [ctb], Aaron Jacobs [ctb], Andreas Bender [ctb], Sven Templer [ctb], Paul-Christian Buerkner [ctb], Matthew Kay [ctb], Erwan Le Pennec [ctb], Johan Junkka [ctb], Hao Zhu [ctb], Benjamin Soltoff [ctb], Zoe Wilkinson Saldana [ctb], Tyler Littlefield [ctb], Charles T. Gray [ctb], Shabbh E. Banks [ctb], Serina Robinson [ctb], Roger Bivand [ctb], Riinu Ots [ctb], Nicholas Williams [ctb], Nina Jakobsen [ctb], Michael Weylandt [ctb], Lisa Lendway [ctb], Karl Hailperin [ctb], Josue Rodriguez [ctb], Jenny Bryan [ctb], Chris Jarvis [ctb], Greg Macfarlane [ctb], Brian Mannakee [ctb], Drew Tyre [ctb], Shreyas Singh [ctb], Laurens Geffert [ctb], Hong Ooi [ctb], Henrik Bengtsson [ctb], Eduard Szocs [ctb], David Hugh-Jones [ctb], Matthieu Stigler [ctb], Hugo Tavares [ctb] (ORCID: ), R. Willem Vervoort [ctb], Brenton M. Wiernik [ctb], Josh Yamamoto [ctb], Jasme Lee [ctb], Taren Sanders [ctb] (ORCID: ), Ilaria Prosdocimi [ctb] (ORCID: ), Daniel D. Sjoberg [ctb] (ORCID: ), Alex Reinhart [ctb] (ORCID: )", - "Maintainer": "Simon Couch ", - "Repository": "CRAN" - }, - "bslib": { - "Package": "bslib", - "Version": "0.9.0", - "Source": "Repository", - "Title": "Custom 'Bootstrap' 'Sass' Themes for 'shiny' and 'rmarkdown'", - "Authors@R": "c( person(\"Carson\", \"Sievert\", , \"carson@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Garrick\", \"Aden-Buie\", , \"garrick@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0002-7111-0077\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(, \"Bootstrap contributors\", role = \"ctb\", comment = \"Bootstrap library\"), person(, \"Twitter, Inc\", role = \"cph\", comment = \"Bootstrap library\"), person(\"Javi\", \"Aguilar\", role = c(\"ctb\", \"cph\"), comment = \"Bootstrap colorpicker library\"), person(\"Thomas\", \"Park\", role = c(\"ctb\", \"cph\"), comment = \"Bootswatch library\"), person(, \"PayPal\", role = c(\"ctb\", \"cph\"), comment = \"Bootstrap accessibility plugin\") )", - "Description": "Simplifies custom 'CSS' styling of both 'shiny' and 'rmarkdown' via 'Bootstrap' 'Sass'. Supports 'Bootstrap' 3, 4 and 5 as well as their various 'Bootswatch' themes. An interactive widget is also provided for previewing themes in real time.", - "License": "MIT + file LICENSE", - "URL": "https://rstudio.github.io/bslib/, https://github.com/rstudio/bslib", - "BugReports": "https://github.com/rstudio/bslib/issues", - "Depends": [ - "R (>= 2.10)" - ], - "Imports": [ - "base64enc", - "cachem", - "fastmap (>= 1.1.1)", - "grDevices", - "htmltools (>= 0.5.8)", - "jquerylib (>= 0.1.3)", - "jsonlite", - "lifecycle", - "memoise (>= 2.0.1)", - "mime", - "rlang", - "sass (>= 0.4.9)" - ], - "Suggests": [ - "bsicons", - "curl", - "fontawesome", - "future", - "ggplot2", - "knitr", - "magrittr", - "rappdirs", - "rmarkdown (>= 2.7)", - "shiny (> 1.8.1)", - "testthat", - "thematic", - "tools", - "utils", - "withr", - "yaml" - ], - "Config/Needs/deploy": "BH, chiflights22, colourpicker, commonmark, cpp11, cpsievert/chiflights22, cpsievert/histoslider, dplyr, DT, ggplot2, ggridges, gt, hexbin, histoslider, htmlwidgets, lattice, leaflet, lubridate, markdown, modelr, plotly, reactable, reshape2, rprojroot, rsconnect, rstudio/shiny, scales, styler, tibble", - "Config/Needs/routine": "chromote, desc, renv", - "Config/Needs/website": "brio, crosstalk, dplyr, DT, ggplot2, glue, htmlwidgets, leaflet, lorem, palmerpenguins, plotly, purrr, rprojroot, rstudio/htmltools, scales, stringr, tidyr, webshot2", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "true", - "Config/testthat/start-first": "zzzz-bs-sass, fonts, zzz-precompile, theme-*, rmd-*", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "Collate": "'accordion.R' 'breakpoints.R' 'bs-current-theme.R' 'bs-dependencies.R' 'bs-global.R' 'bs-remove.R' 'bs-theme-layers.R' 'bs-theme-preset-bootswatch.R' 'bs-theme-preset-brand.R' 'bs-theme-preset-builtin.R' 'bs-theme-preset.R' 'utils.R' 'bs-theme-preview.R' 'bs-theme-update.R' 'bs-theme.R' 'bslib-package.R' 'buttons.R' 'card.R' 'deprecated.R' 'files.R' 'fill.R' 'imports.R' 'input-dark-mode.R' 'input-switch.R' 'layout.R' 'nav-items.R' 'nav-update.R' 'navbar_options.R' 'navs-legacy.R' 'navs.R' 'onLoad.R' 'page.R' 'popover.R' 'precompiled.R' 'print.R' 'shiny-devmode.R' 'sidebar.R' 'staticimports.R' 'tooltip.R' 'utils-deps.R' 'utils-shiny.R' 'utils-tags.R' 'value-box.R' 'version-default.R' 'versions.R'", - "NeedsCompilation": "no", - "Author": "Carson Sievert [aut, cre] (), Joe Cheng [aut], Garrick Aden-Buie [aut] (), Posit Software, PBC [cph, fnd], Bootstrap contributors [ctb] (Bootstrap library), Twitter, Inc [cph] (Bootstrap library), Javi Aguilar [ctb, cph] (Bootstrap colorpicker library), Thomas Park [ctb, cph] (Bootswatch library), PayPal [ctb, cph] (Bootstrap accessibility plugin)", - "Maintainer": "Carson Sievert ", - "Repository": "CRAN" - }, - "cachem": { - "Package": "cachem", - "Version": "1.1.0", - "Source": "Repository", - "Title": "Cache R Objects with Automatic Pruning", - "Description": "Key-value stores with automatic pruning. Caches can limit either their total size or the age of the oldest object (or both), automatically pruning objects to maintain the constraints.", - "Authors@R": "c( person(\"Winston\", \"Chang\", , \"winston@posit.co\", c(\"aut\", \"cre\")), person(family = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")))", - "License": "MIT + file LICENSE", - "Encoding": "UTF-8", - "ByteCompile": "true", - "URL": "https://cachem.r-lib.org/, https://github.com/r-lib/cachem", - "Imports": [ - "rlang", - "fastmap (>= 1.2.0)" - ], - "Suggests": [ - "testthat" - ], - "RoxygenNote": "7.2.3", - "Config/Needs/routine": "lobstr", - "Config/Needs/website": "pkgdown", - "NeedsCompilation": "yes", - "Author": "Winston Chang [aut, cre], Posit Software, PBC [cph, fnd]", - "Maintainer": "Winston Chang ", - "Repository": "RSPM" - }, - "callr": { - "Package": "callr", - "Version": "3.7.6", - "Source": "Repository", - "Title": "Call R from R", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\", \"cph\"), comment = c(ORCID = \"0000-0001-7098-9676\")), person(\"Winston\", \"Chang\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(\"Ascent Digital Services\", role = c(\"cph\", \"fnd\")) )", - "Description": "It is sometimes useful to perform a computation in a separate R process, without affecting the current R process at all. This packages does exactly that.", - "License": "MIT + file LICENSE", - "URL": "https://callr.r-lib.org, https://github.com/r-lib/callr", - "BugReports": "https://github.com/r-lib/callr/issues", - "Depends": [ - "R (>= 3.4)" - ], - "Imports": [ - "processx (>= 3.6.1)", - "R6", - "utils" - ], - "Suggests": [ - "asciicast (>= 2.3.1)", - "cli (>= 1.1.0)", - "mockery", - "ps", - "rprojroot", - "spelling", - "testthat (>= 3.2.0)", - "withr (>= 2.3.0)" - ], - "Config/Needs/website": "r-lib/asciicast, glue, htmlwidgets, igraph, tibble, tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.1.9000", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [aut, cre, cph] (), Winston Chang [aut], Posit Software, PBC [cph, fnd], Ascent Digital Services [cph, fnd]", - "Maintainer": "Gábor Csárdi ", - "Repository": "RSPM" - }, - "cellranger": { - "Package": "cellranger", - "Version": "1.1.0", - "Source": "Repository", - "Title": "Translate Spreadsheet Cell Ranges to Rows and Columns", - "Authors@R": "c( person(\"Jennifer\", \"Bryan\", , \"jenny@stat.ubc.ca\", c(\"cre\", \"aut\")), person(\"Hadley\", \"Wickham\", , \"hadley@rstudio.com\", \"ctb\") )", - "Description": "Helper functions to work with spreadsheets and the \"A1:D10\" style of cell range specification.", - "Depends": [ - "R (>= 3.0.0)" - ], - "License": "MIT + file LICENSE", - "LazyData": "true", - "URL": "https://github.com/rsheets/cellranger", - "BugReports": "https://github.com/rsheets/cellranger/issues", - "Suggests": [ - "covr", - "testthat (>= 1.0.0)", - "knitr", - "rmarkdown" - ], - "RoxygenNote": "5.0.1.9000", - "VignetteBuilder": "knitr", - "Imports": [ - "rematch", - "tibble" - ], - "NeedsCompilation": "no", - "Author": "Jennifer Bryan [cre, aut], Hadley Wickham [ctb]", - "Maintainer": "Jennifer Bryan ", - "Repository": "CRAN" - }, - "cli": { - "Package": "cli", - "Version": "3.6.5", - "Source": "Repository", - "Title": "Helpers for Developing Command Line Interfaces", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"gabor@posit.co\", role = c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", role = \"ctb\"), person(\"Kirill\", \"Müller\", role = \"ctb\"), person(\"Salim\", \"Brüggemann\", , \"salim-b@pm.me\", role = \"ctb\", comment = c(ORCID = \"0000-0002-5329-5987\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "A suite of tools to build attractive command line interfaces ('CLIs'), from semantic elements: headings, lists, alerts, paragraphs, etc. Supports custom themes via a 'CSS'-like language. It also contains a number of lower level 'CLI' elements: rules, boxes, trees, and 'Unicode' symbols with 'ASCII' alternatives. It support ANSI colors and text styles as well.", - "License": "MIT + file LICENSE", - "URL": "https://cli.r-lib.org, https://github.com/r-lib/cli", - "BugReports": "https://github.com/r-lib/cli/issues", - "Depends": [ - "R (>= 3.4)" - ], - "Imports": [ - "utils" - ], - "Suggests": [ - "callr", - "covr", - "crayon", - "digest", - "glue (>= 1.6.0)", - "grDevices", - "htmltools", - "htmlwidgets", - "knitr", - "methods", - "processx", - "ps (>= 1.3.4.9000)", - "rlang (>= 1.0.2.9003)", - "rmarkdown", - "rprojroot", - "rstudioapi", - "testthat (>= 3.2.0)", - "tibble", - "whoami", - "withr" - ], - "Config/Needs/website": "r-lib/asciicast, bench, brio, cpp11, decor, desc, fansi, prettyunits, sessioninfo, tidyverse/tidytemplate, usethis, vctrs", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "yes", - "Author": "Gábor Csárdi [aut, cre], Hadley Wickham [ctb], Kirill Müller [ctb], Salim Brüggemann [ctb] (), Posit Software, PBC [cph, fnd]", - "Maintainer": "Gábor Csárdi ", - "Repository": "CRAN" - }, - "clipr": { - "Package": "clipr", - "Version": "0.8.0", - "Source": "Repository", - "Type": "Package", - "Title": "Read and Write from the System Clipboard", - "Authors@R": "c( person(\"Matthew\", \"Lincoln\", , \"matthew.d.lincoln@gmail.com\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4387-3384\")), person(\"Louis\", \"Maddox\", role = \"ctb\"), person(\"Steve\", \"Simpson\", role = \"ctb\"), person(\"Jennifer\", \"Bryan\", role = \"ctb\") )", - "Description": "Simple utility functions to read from and write to the Windows, OS X, and X11 clipboards.", - "License": "GPL-3", - "URL": "https://github.com/mdlincoln/clipr, http://matthewlincoln.net/clipr/", - "BugReports": "https://github.com/mdlincoln/clipr/issues", - "Imports": [ - "utils" - ], - "Suggests": [ - "covr", - "knitr", - "rmarkdown", - "rstudioapi (>= 0.5)", - "testthat (>= 2.0.0)" - ], - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.1.2", - "SystemRequirements": "xclip (https://github.com/astrand/xclip) or xsel (http://www.vergenet.net/~conrad/software/xsel/) for accessing the X11 clipboard, or wl-clipboard (https://github.com/bugaevc/wl-clipboard) for systems using Wayland.", - "NeedsCompilation": "no", - "Author": "Matthew Lincoln [aut, cre] (), Louis Maddox [ctb], Steve Simpson [ctb], Jennifer Bryan [ctb]", - "Maintainer": "Matthew Lincoln ", - "Repository": "RSPM" - }, - "commonmark": { - "Package": "commonmark", - "Version": "2.0.0", - "Source": "Repository", - "Type": "Package", - "Title": "High Performance CommonMark and Github Markdown Rendering in R", - "Authors@R": "c( person(\"Jeroen\", \"Ooms\", ,\"jeroenooms@gmail.com\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4035-0289\")), person(\"John MacFarlane\", role = \"cph\", comment = \"Author of cmark\"))", - "Description": "The CommonMark specification defines a rationalized version of markdown syntax. This package uses the 'cmark' reference implementation for converting markdown text into various formats including html, latex and groff man. In addition it exposes the markdown parse tree in xml format. Also includes opt-in support for GFM extensions including tables, autolinks, and strikethrough text.", - "License": "BSD_2_clause + file LICENSE", - "URL": "https://docs.ropensci.org/commonmark/ https://ropensci.r-universe.dev/commonmark", - "BugReports": "https://github.com/r-lib/commonmark/issues", - "Suggests": [ - "curl", - "testthat", - "xml2" - ], - "RoxygenNote": "7.3.2", - "Language": "en-US", - "Encoding": "UTF-8", - "NeedsCompilation": "yes", - "Author": "Jeroen Ooms [aut, cre] (ORCID: ), John MacFarlane [cph] (Author of cmark)", - "Maintainer": "Jeroen Ooms ", - "Repository": "CRAN" - }, - "config": { - "Package": "config", - "Version": "0.3.2", - "Source": "Repository", - "Type": "Package", - "Title": "Manage Environment Specific Configuration Values", - "Authors@R": "c( person(\"JJ\", \"Allaire\", role = c(\"aut\"), email = \"jj@rstudio.com\"), person(\"Andrie\", \"de Vries\", role = \"cre\", email = \"apdevries@gmail.com\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Imports": [ - "yaml (>= 2.1.19)" - ], - "Suggests": [ - "testthat", - "knitr", - "rmarkdown", - "covr", - "spelling", - "withr" - ], - "Description": "Manage configuration values across multiple environments (e.g. development, test, production). Read values using a function that determines the current environment and returns the appropriate value.", - "License": "GPL-3", - "URL": "https://rstudio.github.io/config/, https://github.com/rstudio/config", - "BugReports": "https://github.com/rstudio/config/issues", - "RoxygenNote": "7.2.3", - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "Language": "en-US", - "Config/testthat/edition": "3", - "NeedsCompilation": "no", - "Author": "JJ Allaire [aut], Andrie de Vries [cre], Posit Software, PBC [cph, fnd]", - "Maintainer": "Andrie de Vries ", - "Repository": "CRAN" - }, - "conflicted": { - "Package": "conflicted", - "Version": "1.2.0", - "Source": "Repository", - "Title": "An Alternative Conflict Resolution Strategy", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@rstudio.com\", role = c(\"aut\", \"cre\")), person(\"RStudio\", role = c(\"cph\", \"fnd\")) )", - "Description": "R's default conflict management system gives the most recently loaded package precedence. This can make it hard to detect conflicts, particularly when they arise because a package update creates ambiguity that did not previously exist. 'conflicted' takes a different approach, making every conflict an error and forcing you to choose which function to use.", - "License": "MIT + file LICENSE", - "URL": "https://conflicted.r-lib.org/, https://github.com/r-lib/conflicted", - "BugReports": "https://github.com/r-lib/conflicted/issues", - "Depends": [ - "R (>= 3.2)" - ], - "Imports": [ - "cli (>= 3.4.0)", - "memoise", - "rlang (>= 1.0.0)" - ], - "Suggests": [ - "callr", - "covr", - "dplyr", - "Matrix", - "methods", - "pkgload", - "testthat (>= 3.0.0)", - "withr" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.3", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre], RStudio [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "RSPM" - }, - "cpp11": { - "Package": "cpp11", - "Version": "0.5.2", - "Source": "Repository", - "Title": "A C++11 Interface for R's C Interface", - "Authors@R": "c( person(\"Davis\", \"Vaughan\", email = \"davis@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4777-038X\")), person(\"Jim\",\"Hester\", role = \"aut\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"Romain\", \"François\", role = \"aut\", comment = c(ORCID = \"0000-0002-2444-4226\")), person(\"Benjamin\", \"Kietzman\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Provides a header only, C++11 interface to R's C interface. Compared to other approaches 'cpp11' strives to be safe against long jumps from the C API as well as C++ exceptions, conform to normal R function semantics and supports interaction with 'ALTREP' vectors.", - "License": "MIT + file LICENSE", - "URL": "https://cpp11.r-lib.org, https://github.com/r-lib/cpp11", - "BugReports": "https://github.com/r-lib/cpp11/issues", - "Depends": [ - "R (>= 4.0.0)" - ], - "Suggests": [ - "bench", - "brio", - "callr", - "cli", - "covr", - "decor", - "desc", - "ggplot2", - "glue", - "knitr", - "lobstr", - "mockery", - "progress", - "rmarkdown", - "scales", - "Rcpp", - "testthat (>= 3.2.0)", - "tibble", - "utils", - "vctrs", - "withr" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/Needs/cpp11/cpp_register": "brio, cli, decor, desc, glue, tibble, vctrs", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Davis Vaughan [aut, cre] (), Jim Hester [aut] (), Romain François [aut] (), Benjamin Kietzman [ctb], Posit Software, PBC [cph, fnd]", - "Maintainer": "Davis Vaughan ", - "Repository": "CRAN" - }, - "crayon": { - "Package": "crayon", - "Version": "1.5.3", - "Source": "Repository", - "Title": "Colored Terminal Output", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Brodie\", \"Gaslam\", , \"brodie.gaslam@yahoo.com\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "The crayon package is now superseded. Please use the 'cli' package for new projects. Colored terminal output on terminals that support 'ANSI' color and highlight codes. It also works in 'Emacs' 'ESS'. 'ANSI' color support is automatically detected. Colors and highlighting can be combined and nested. New styles can also be created easily. This package was inspired by the 'chalk' 'JavaScript' project.", - "License": "MIT + file LICENSE", - "URL": "https://r-lib.github.io/crayon/, https://github.com/r-lib/crayon", - "BugReports": "https://github.com/r-lib/crayon/issues", - "Imports": [ - "grDevices", - "methods", - "utils" - ], - "Suggests": [ - "mockery", - "rstudioapi", - "testthat", - "withr" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.1", - "Collate": "'aaa-rstudio-detect.R' 'aaaa-rematch2.R' 'aab-num-ansi-colors.R' 'aac-num-ansi-colors.R' 'ansi-256.R' 'ansi-palette.R' 'combine.R' 'string.R' 'utils.R' 'crayon-package.R' 'disposable.R' 'enc-utils.R' 'has_ansi.R' 'has_color.R' 'link.R' 'styles.R' 'machinery.R' 'parts.R' 'print.R' 'style-var.R' 'show.R' 'string_operations.R'", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [aut, cre], Brodie Gaslam [ctb], Posit Software, PBC [cph, fnd]", - "Maintainer": "Gábor Csárdi ", - "Repository": "RSPM" - }, - "credentials": { - "Package": "credentials", - "Version": "2.0.3", - "Source": "Repository", - "Type": "Package", - "Title": "Tools for Managing SSH and Git Credentials", - "Authors@R": "person(\"Jeroen\", \"Ooms\", role = c(\"aut\", \"cre\"), email = \"jeroenooms@gmail.com\", comment = c(ORCID = \"0000-0002-4035-0289\"))", - "Description": "Setup and retrieve HTTPS and SSH credentials for use with 'git' and other services. For HTTPS remotes the package interfaces the 'git-credential' utility which 'git' uses to store HTTP usernames and passwords. For SSH remotes we provide convenient functions to find or generate appropriate SSH keys. The package both helps the user to setup a local git installation, and also provides a back-end for git/ssh client libraries to authenticate with existing user credentials.", - "License": "MIT + file LICENSE", - "SystemRequirements": "git (optional)", - "Encoding": "UTF-8", - "Imports": [ - "openssl (>= 1.3)", - "sys (>= 2.1)", - "curl", - "jsonlite", - "askpass" - ], - "Suggests": [ - "testthat", - "knitr", - "rmarkdown" - ], - "RoxygenNote": "7.2.1", - "VignetteBuilder": "knitr", - "Language": "en-US", - "URL": "https://docs.ropensci.org/credentials/ https://r-lib.r-universe.dev/credentials", - "BugReports": "https://github.com/r-lib/credentials/issues", - "NeedsCompilation": "no", - "Author": "Jeroen Ooms [aut, cre] (ORCID: )", - "Maintainer": "Jeroen Ooms ", - "Repository": "CRAN" - }, - "crosstalk": { - "Package": "crosstalk", - "Version": "1.2.2", - "Source": "Repository", - "Type": "Package", - "Title": "Inter-Widget Interactivity for HTML Widgets", - "Authors@R": "c( person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Carson\", \"Sievert\", , \"carson@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(, \"jQuery Foundation\", role = \"cph\", comment = \"jQuery library and jQuery UI library\"), person(, \"jQuery contributors\", role = c(\"ctb\", \"cph\"), comment = \"jQuery library; authors listed in inst/www/shared/jquery-AUTHORS.txt\"), person(\"Mark\", \"Otto\", role = \"ctb\", comment = \"Bootstrap library\"), person(\"Jacob\", \"Thornton\", role = \"ctb\", comment = \"Bootstrap library\"), person(, \"Bootstrap contributors\", role = \"ctb\", comment = \"Bootstrap library\"), person(, \"Twitter, Inc\", role = \"cph\", comment = \"Bootstrap library\"), person(\"Brian\", \"Reavis\", role = c(\"ctb\", \"cph\"), comment = \"selectize.js library\"), person(\"Kristopher Michael\", \"Kowal\", role = c(\"ctb\", \"cph\"), comment = \"es5-shim library\"), person(, \"es5-shim contributors\", role = c(\"ctb\", \"cph\"), comment = \"es5-shim library\"), person(\"Denis\", \"Ineshin\", role = c(\"ctb\", \"cph\"), comment = \"ion.rangeSlider library\"), person(\"Sami\", \"Samhuri\", role = c(\"ctb\", \"cph\"), comment = \"Javascript strftime library\") )", - "Description": "Provides building blocks for allowing HTML widgets to communicate with each other, with Shiny or without (i.e. static .html files). Currently supports linked brushing and filtering.", - "License": "MIT + file LICENSE", - "URL": "https://rstudio.github.io/crosstalk/, https://github.com/rstudio/crosstalk", - "BugReports": "https://github.com/rstudio/crosstalk/issues", - "Imports": [ - "htmltools (>= 0.3.6)", - "jsonlite", - "lazyeval", - "R6" - ], - "Suggests": [ - "bslib", - "ggplot2", - "sass", - "shiny", - "testthat (>= 2.1.0)" - ], - "Config/Needs/website": "jcheng5/d3scatter, DT, leaflet, rmarkdown", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Joe Cheng [aut], Carson Sievert [aut, cre] (ORCID: ), Posit Software, PBC [cph, fnd], jQuery Foundation [cph] (jQuery library and jQuery UI library), jQuery contributors [ctb, cph] (jQuery library; authors listed in inst/www/shared/jquery-AUTHORS.txt), Mark Otto [ctb] (Bootstrap library), Jacob Thornton [ctb] (Bootstrap library), Bootstrap contributors [ctb] (Bootstrap library), Twitter, Inc [cph] (Bootstrap library), Brian Reavis [ctb, cph] (selectize.js library), Kristopher Michael Kowal [ctb, cph] (es5-shim library), es5-shim contributors [ctb, cph] (es5-shim library), Denis Ineshin [ctb, cph] (ion.rangeSlider library), Sami Samhuri [ctb, cph] (Javascript strftime library)", - "Maintainer": "Carson Sievert ", - "Repository": "CRAN" - }, - "curl": { - "Package": "curl", - "Version": "7.0.0", - "Source": "Repository", - "Type": "Package", - "Title": "A Modern and Flexible Web Client for R", - "Authors@R": "c( person(\"Jeroen\", \"Ooms\", role = c(\"aut\", \"cre\"), email = \"jeroenooms@gmail.com\", comment = c(ORCID = \"0000-0002-4035-0289\")), person(\"Hadley\", \"Wickham\", role = \"ctb\"), person(\"Posit Software, PBC\", role = \"cph\"))", - "Description": "Bindings to 'libcurl' for performing fully configurable HTTP/FTP requests where responses can be processed in memory, on disk, or streaming via the callback or connection interfaces. Some knowledge of 'libcurl' is recommended; for a more-user-friendly web client see the 'httr2' package which builds on this package with http specific tools and logic.", - "License": "MIT + file LICENSE", - "SystemRequirements": "libcurl (>= 7.73): libcurl-devel (rpm) or libcurl4-openssl-dev (deb)", - "URL": "https://jeroen.r-universe.dev/curl", - "BugReports": "https://github.com/jeroen/curl/issues", - "Suggests": [ - "spelling", - "testthat (>= 1.0.0)", - "knitr", - "jsonlite", - "later", - "rmarkdown", - "httpuv (>= 1.4.4)", - "webutils" - ], - "VignetteBuilder": "knitr", - "Depends": [ - "R (>= 3.0.0)" - ], - "RoxygenNote": "7.3.2", - "Encoding": "UTF-8", - "Language": "en-US", - "NeedsCompilation": "yes", - "Author": "Jeroen Ooms [aut, cre] (ORCID: ), Hadley Wickham [ctb], Posit Software, PBC [cph]", - "Maintainer": "Jeroen Ooms ", - "Repository": "CRAN" - }, - "data.table": { - "Package": "data.table", - "Version": "1.17.8", - "Source": "Repository", - "Title": "Extension of `data.frame`", - "Depends": [ - "R (>= 3.3.0)" - ], - "Imports": [ - "methods" - ], - "Suggests": [ - "bit64 (>= 4.0.0)", - "bit (>= 4.0.4)", - "R.utils", - "xts", - "zoo (>= 1.8-1)", - "yaml", - "knitr", - "markdown" - ], - "Description": "Fast aggregation of large data (e.g. 100GB in RAM), fast ordered joins, fast add/modify/delete of columns by group using no copies at all, list columns, friendly and fast character-separated-value read/write. Offers a natural and flexible syntax, for faster development.", - "License": "MPL-2.0 | file LICENSE", - "URL": "https://r-datatable.com, https://Rdatatable.gitlab.io/data.table, https://github.com/Rdatatable/data.table", - "BugReports": "https://github.com/Rdatatable/data.table/issues", - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "ByteCompile": "TRUE", - "Authors@R": "c( person(\"Tyson\",\"Barrett\", role=c(\"aut\",\"cre\"), email=\"t.barrett88@gmail.com\", comment = c(ORCID=\"0000-0002-2137-1391\")), person(\"Matt\",\"Dowle\", role=\"aut\", email=\"mattjdowle@gmail.com\"), person(\"Arun\",\"Srinivasan\", role=\"aut\", email=\"asrini@pm.me\"), person(\"Jan\",\"Gorecki\", role=\"aut\"), person(\"Michael\",\"Chirico\", role=\"aut\", comment = c(ORCID=\"0000-0003-0787-087X\")), person(\"Toby\",\"Hocking\", role=\"aut\", comment = c(ORCID=\"0000-0002-3146-0865\")), person(\"Benjamin\",\"Schwendinger\",role=\"aut\", comment = c(ORCID=\"0000-0003-3315-8114\")), person(\"Ivan\", \"Krylov\", role=\"aut\", email=\"ikrylov@disroot.org\", comment = c(ORCID=\"0000-0002-0172-3812\")), person(\"Pasha\",\"Stetsenko\", role=\"ctb\"), person(\"Tom\",\"Short\", role=\"ctb\"), person(\"Steve\",\"Lianoglou\", role=\"ctb\"), person(\"Eduard\",\"Antonyan\", role=\"ctb\"), person(\"Markus\",\"Bonsch\", role=\"ctb\"), person(\"Hugh\",\"Parsonage\", role=\"ctb\"), person(\"Scott\",\"Ritchie\", role=\"ctb\"), person(\"Kun\",\"Ren\", role=\"ctb\"), person(\"Xianying\",\"Tan\", role=\"ctb\"), person(\"Rick\",\"Saporta\", role=\"ctb\"), person(\"Otto\",\"Seiskari\", role=\"ctb\"), person(\"Xianghui\",\"Dong\", role=\"ctb\"), person(\"Michel\",\"Lang\", role=\"ctb\"), person(\"Watal\",\"Iwasaki\", role=\"ctb\"), person(\"Seth\",\"Wenchel\", role=\"ctb\"), person(\"Karl\",\"Broman\", role=\"ctb\"), person(\"Tobias\",\"Schmidt\", role=\"ctb\"), person(\"David\",\"Arenburg\", role=\"ctb\"), person(\"Ethan\",\"Smith\", role=\"ctb\"), person(\"Francois\",\"Cocquemas\", role=\"ctb\"), person(\"Matthieu\",\"Gomez\", role=\"ctb\"), person(\"Philippe\",\"Chataignon\", role=\"ctb\"), person(\"Nello\",\"Blaser\", role=\"ctb\"), person(\"Dmitry\",\"Selivanov\", role=\"ctb\"), person(\"Andrey\",\"Riabushenko\", role=\"ctb\"), person(\"Cheng\",\"Lee\", role=\"ctb\"), person(\"Declan\",\"Groves\", role=\"ctb\"), person(\"Daniel\",\"Possenriede\", role=\"ctb\"), person(\"Felipe\",\"Parages\", role=\"ctb\"), person(\"Denes\",\"Toth\", role=\"ctb\"), person(\"Mus\",\"Yaramaz-David\", role=\"ctb\"), person(\"Ayappan\",\"Perumal\", role=\"ctb\"), person(\"James\",\"Sams\", role=\"ctb\"), person(\"Martin\",\"Morgan\", role=\"ctb\"), person(\"Michael\",\"Quinn\", role=\"ctb\"), person(\"@javrucebo\",\"\", role=\"ctb\"), person(\"@marc-outins\",\"\", role=\"ctb\"), person(\"Roy\",\"Storey\", role=\"ctb\"), person(\"Manish\",\"Saraswat\", role=\"ctb\"), person(\"Morgan\",\"Jacob\", role=\"ctb\"), person(\"Michael\",\"Schubmehl\", role=\"ctb\"), person(\"Davis\",\"Vaughan\", role=\"ctb\"), person(\"Leonardo\",\"Silvestri\", role=\"ctb\"), person(\"Jim\",\"Hester\", role=\"ctb\"), person(\"Anthony\",\"Damico\", role=\"ctb\"), person(\"Sebastian\",\"Freundt\", role=\"ctb\"), person(\"David\",\"Simons\", role=\"ctb\"), person(\"Elliott\",\"Sales de Andrade\", role=\"ctb\"), person(\"Cole\",\"Miller\", role=\"ctb\"), person(\"Jens Peder\",\"Meldgaard\", role=\"ctb\"), person(\"Vaclav\",\"Tlapak\", role=\"ctb\"), person(\"Kevin\",\"Ushey\", role=\"ctb\"), person(\"Dirk\",\"Eddelbuettel\", role=\"ctb\"), person(\"Tony\",\"Fischetti\", role=\"ctb\"), person(\"Ofek\",\"Shilon\", role=\"ctb\"), person(\"Vadim\",\"Khotilovich\", role=\"ctb\"), person(\"Hadley\",\"Wickham\", role=\"ctb\"), person(\"Bennet\",\"Becker\", role=\"ctb\"), person(\"Kyle\",\"Haynes\", role=\"ctb\"), person(\"Boniface Christian\",\"Kamgang\", role=\"ctb\"), person(\"Olivier\",\"Delmarcell\", role=\"ctb\"), person(\"Josh\",\"O'Brien\", role=\"ctb\"), person(\"Dereck\",\"de Mezquita\", role=\"ctb\"), person(\"Michael\",\"Czekanski\", role=\"ctb\"), person(\"Dmitry\", \"Shemetov\", role=\"ctb\"), person(\"Nitish\", \"Jha\", role=\"ctb\"), person(\"Joshua\", \"Wu\", role=\"ctb\"), person(\"Iago\", \"Giné-Vázquez\", role=\"ctb\"), person(\"Anirban\", \"Chetia\", role=\"ctb\"), person(\"Doris\", \"Amoakohene\", role=\"ctb\"), person(\"Angel\", \"Feliz\", role=\"ctb\"), person(\"Michael\",\"Young\", role=\"ctb\"), person(\"Mark\", \"Seeto\", role=\"ctb\"), person(\"Philippe\", \"Grosjean\", role=\"ctb\"), person(\"Vincent\", \"Runge\", role=\"ctb\"), person(\"Christian\", \"Wia\", role=\"ctb\"), person(\"Elise\", \"Maigné\", role=\"ctb\"), person(\"Vincent\", \"Rocher\", role=\"ctb\"), person(\"Vijay\", \"Lulla\", role=\"ctb\"), person(\"Aljaž\", \"Sluga\", role=\"ctb\"), person(\"Bill\", \"Evans\", role=\"ctb\") )", - "NeedsCompilation": "yes", - "Author": "Tyson Barrett [aut, cre] (ORCID: ), Matt Dowle [aut], Arun Srinivasan [aut], Jan Gorecki [aut], Michael Chirico [aut] (ORCID: ), Toby Hocking [aut] (ORCID: ), Benjamin Schwendinger [aut] (ORCID: ), Ivan Krylov [aut] (ORCID: ), Pasha Stetsenko [ctb], Tom Short [ctb], Steve Lianoglou [ctb], Eduard Antonyan [ctb], Markus Bonsch [ctb], Hugh Parsonage [ctb], Scott Ritchie [ctb], Kun Ren [ctb], Xianying Tan [ctb], Rick Saporta [ctb], Otto Seiskari [ctb], Xianghui Dong [ctb], Michel Lang [ctb], Watal Iwasaki [ctb], Seth Wenchel [ctb], Karl Broman [ctb], Tobias Schmidt [ctb], David Arenburg [ctb], Ethan Smith [ctb], Francois Cocquemas [ctb], Matthieu Gomez [ctb], Philippe Chataignon [ctb], Nello Blaser [ctb], Dmitry Selivanov [ctb], Andrey Riabushenko [ctb], Cheng Lee [ctb], Declan Groves [ctb], Daniel Possenriede [ctb], Felipe Parages [ctb], Denes Toth [ctb], Mus Yaramaz-David [ctb], Ayappan Perumal [ctb], James Sams [ctb], Martin Morgan [ctb], Michael Quinn [ctb], @javrucebo [ctb], @marc-outins [ctb], Roy Storey [ctb], Manish Saraswat [ctb], Morgan Jacob [ctb], Michael Schubmehl [ctb], Davis Vaughan [ctb], Leonardo Silvestri [ctb], Jim Hester [ctb], Anthony Damico [ctb], Sebastian Freundt [ctb], David Simons [ctb], Elliott Sales de Andrade [ctb], Cole Miller [ctb], Jens Peder Meldgaard [ctb], Vaclav Tlapak [ctb], Kevin Ushey [ctb], Dirk Eddelbuettel [ctb], Tony Fischetti [ctb], Ofek Shilon [ctb], Vadim Khotilovich [ctb], Hadley Wickham [ctb], Bennet Becker [ctb], Kyle Haynes [ctb], Boniface Christian Kamgang [ctb], Olivier Delmarcell [ctb], Josh O'Brien [ctb], Dereck de Mezquita [ctb], Michael Czekanski [ctb], Dmitry Shemetov [ctb], Nitish Jha [ctb], Joshua Wu [ctb], Iago Giné-Vázquez [ctb], Anirban Chetia [ctb], Doris Amoakohene [ctb], Angel Feliz [ctb], Michael Young [ctb], Mark Seeto [ctb], Philippe Grosjean [ctb], Vincent Runge [ctb], Christian Wia [ctb], Elise Maigné [ctb], Vincent Rocher [ctb], Vijay Lulla [ctb], Aljaž Sluga [ctb], Bill Evans [ctb]", - "Maintainer": "Tyson Barrett ", - "Repository": "CRAN" - }, - "dbplyr": { - "Package": "dbplyr", - "Version": "2.5.1", - "Source": "Repository", - "Type": "Package", - "Title": "A 'dplyr' Back End for Databases", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Maximilian\", \"Girlich\", role = \"aut\"), person(\"Edgar\", \"Ruiz\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "A 'dplyr' back end for databases that allows you to work with remote database tables as if they are in-memory data frames. Basic features works with any database that has a 'DBI' back end; more advanced features require 'SQL' translation to be provided by the package author.", - "License": "MIT + file LICENSE", - "URL": "https://dbplyr.tidyverse.org/, https://github.com/tidyverse/dbplyr", - "BugReports": "https://github.com/tidyverse/dbplyr/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "blob (>= 1.2.0)", - "cli (>= 3.6.1)", - "DBI (>= 1.1.3)", - "dplyr (>= 1.1.2)", - "glue (>= 1.6.2)", - "lifecycle (>= 1.0.3)", - "magrittr", - "methods", - "pillar (>= 1.9.0)", - "purrr (>= 1.0.1)", - "R6 (>= 2.2.2)", - "rlang (>= 1.1.1)", - "tibble (>= 3.2.1)", - "tidyr (>= 1.3.0)", - "tidyselect (>= 1.2.1)", - "utils", - "vctrs (>= 0.6.3)", - "withr (>= 2.5.0)" - ], - "Suggests": [ - "bit64", - "covr", - "knitr", - "Lahman", - "nycflights13", - "odbc (>= 1.4.2)", - "RMariaDB (>= 1.2.2)", - "rmarkdown", - "RPostgres (>= 1.4.5)", - "RPostgreSQL", - "RSQLite (>= 2.3.8)", - "testthat (>= 3.1.10)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "TRUE", - "Encoding": "UTF-8", - "Language": "en-gb", - "RoxygenNote": "7.3.3", - "Collate": "'db-sql.R' 'utils-check.R' 'import-standalone-types-check.R' 'import-standalone-obj-type.R' 'utils.R' 'sql.R' 'escape.R' 'translate-sql-cut.R' 'translate-sql-quantile.R' 'translate-sql-string.R' 'translate-sql-paste.R' 'translate-sql-helpers.R' 'translate-sql-window.R' 'translate-sql-conditional.R' 'backend-.R' 'backend-access.R' 'backend-hana.R' 'backend-hive.R' 'backend-impala.R' 'verb-copy-to.R' 'backend-mssql.R' 'backend-mysql.R' 'backend-odbc.R' 'backend-oracle.R' 'backend-postgres.R' 'backend-postgres-old.R' 'backend-redshift.R' 'backend-snowflake.R' 'backend-spark-sql.R' 'backend-sqlite.R' 'backend-teradata.R' 'build-sql.R' 'data-cache.R' 'data-lahman.R' 'data-nycflights13.R' 'db-escape.R' 'db-io.R' 'db.R' 'dbplyr.R' 'explain.R' 'ident.R' 'import-standalone-s3-register.R' 'join-by-compat.R' 'join-cols-compat.R' 'lazy-join-query.R' 'lazy-ops.R' 'lazy-query.R' 'lazy-select-query.R' 'lazy-set-op-query.R' 'memdb.R' 'optimise-utils.R' 'pillar.R' 'progress.R' 'sql-build.R' 'query-join.R' 'query-select.R' 'query-semi-join.R' 'query-set-op.R' 'query.R' 'reexport.R' 'remote.R' 'rows.R' 'schema.R' 'simulate.R' 'sql-clause.R' 'sql-expr.R' 'src-sql.R' 'src_dbi.R' 'table-name.R' 'tbl-lazy.R' 'tbl-sql.R' 'test-frame.R' 'testthat.R' 'tidyeval-across.R' 'tidyeval.R' 'translate-sql.R' 'utils-format.R' 'verb-arrange.R' 'verb-compute.R' 'verb-count.R' 'verb-distinct.R' 'verb-do-query.R' 'verb-do.R' 'verb-expand.R' 'verb-fill.R' 'verb-filter.R' 'verb-group_by.R' 'verb-head.R' 'verb-joins.R' 'verb-mutate.R' 'verb-pivot-longer.R' 'verb-pivot-wider.R' 'verb-pull.R' 'verb-select.R' 'verb-set-ops.R' 'verb-slice.R' 'verb-summarise.R' 'verb-uncount.R' 'verb-window.R' 'zzz.R'", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre], Maximilian Girlich [aut], Edgar Ruiz [aut], Posit Software, PBC [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "desc": { - "Package": "desc", - "Version": "1.4.3", - "Source": "Repository", - "Title": "Manipulate DESCRIPTION Files", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Kirill\", \"Müller\", role = \"aut\"), person(\"Jim\", \"Hester\", , \"james.f.hester@gmail.com\", role = \"aut\"), person(\"Maëlle\", \"Salmon\", role = \"ctb\", comment = c(ORCID = \"0000-0002-2815-0399\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Maintainer": "Gábor Csárdi ", - "Description": "Tools to read, write, create, and manipulate DESCRIPTION files. It is intended for packages that create or manipulate other packages.", - "License": "MIT + file LICENSE", - "URL": "https://desc.r-lib.org/, https://github.com/r-lib/desc", - "BugReports": "https://github.com/r-lib/desc/issues", - "Depends": [ - "R (>= 3.4)" - ], - "Imports": [ - "cli", - "R6", - "utils" - ], - "Suggests": [ - "callr", - "covr", - "gh", - "spelling", - "testthat", - "whoami", - "withr" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.2.3", - "Collate": "'assertions.R' 'authors-at-r.R' 'built.R' 'classes.R' 'collate.R' 'constants.R' 'deps.R' 'desc-package.R' 'description.R' 'encoding.R' 'find-package-root.R' 'latex.R' 'non-oo-api.R' 'package-archives.R' 'read.R' 'remotes.R' 'str.R' 'syntax_checks.R' 'urls.R' 'utils.R' 'validate.R' 'version.R'", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [aut, cre], Kirill Müller [aut], Jim Hester [aut], Maëlle Salmon [ctb] (), Posit Software, PBC [cph, fnd]", - "Repository": "RSPM" - }, - "devtools": { - "Package": "devtools", - "Version": "2.4.6", - "Source": "Repository", - "Title": "Tools to Make Developing R Packages Easier", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", role = \"aut\"), person(\"Jim\", \"Hester\", role = \"aut\"), person(\"Winston\", \"Chang\", role = \"aut\"), person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Collection of package development tools.", - "License": "MIT + file LICENSE", - "URL": "https://devtools.r-lib.org/, https://github.com/r-lib/devtools", - "BugReports": "https://github.com/r-lib/devtools/issues", - "Depends": [ - "R (>= 4.1)", - "usethis (>= 3.2.1)" - ], - "Imports": [ - "cli (>= 3.6.5)", - "desc (>= 1.4.3)", - "ellipsis (>= 0.3.2)", - "fs (>= 1.6.6)", - "lifecycle (>= 1.0.4)", - "memoise (>= 2.0.1)", - "miniUI (>= 0.1.2)", - "pkgbuild (>= 1.4.8)", - "pkgdown (>= 2.1.3)", - "pkgload (>= 1.4.1)", - "profvis (>= 0.4.0)", - "rcmdcheck (>= 1.4.0)", - "remotes (>= 2.5.0)", - "rlang (>= 1.1.6)", - "roxygen2 (>= 7.3.3)", - "rversions (>= 2.1.2)", - "sessioninfo (>= 1.2.3)", - "stats", - "testthat (>= 3.2.3)", - "tools", - "urlchecker (>= 1.0.1)", - "utils", - "withr (>= 3.0.2)" - ], - "Suggests": [ - "BiocManager (>= 1.30.18)", - "callr (>= 3.7.1)", - "covr (>= 3.5.1)", - "curl (>= 4.3.2)", - "digest (>= 0.6.29)", - "DT (>= 0.23)", - "foghorn (>= 1.4.2)", - "gh (>= 1.3.0)", - "gmailr (>= 1.0.1)", - "httr (>= 1.4.3)", - "knitr (>= 1.39)", - "lintr (>= 3.0.0)", - "MASS", - "mockery (>= 0.4.3)", - "pingr (>= 2.0.1)", - "rhub (>= 1.1.1)", - "rmarkdown (>= 2.14)", - "rstudioapi (>= 0.13)", - "spelling (>= 2.2)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.3", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut], Jim Hester [aut], Winston Chang [aut], Jennifer Bryan [aut, cre] (ORCID: ), Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Jennifer Bryan ", - "Repository": "CRAN" - }, - "diffobj": { - "Package": "diffobj", - "Version": "0.3.6", - "Source": "Repository", - "Type": "Package", - "Title": "Diffs for R Objects", - "Description": "Generate a colorized diff of two R objects for an intuitive visualization of their differences.", - "Authors@R": "c( person( \"Brodie\", \"Gaslam\", email=\"brodie.gaslam@yahoo.com\", role=c(\"aut\", \"cre\")), person( \"Michael B.\", \"Allen\", email=\"ioplex@gmail.com\", role=c(\"ctb\", \"cph\"), comment=\"Original C implementation of Myers Diff Algorithm\"))", - "Depends": [ - "R (>= 3.1.0)" - ], - "License": "GPL-2 | GPL-3", - "URL": "https://github.com/brodieG/diffobj", - "BugReports": "https://github.com/brodieG/diffobj/issues", - "RoxygenNote": "7.2.3", - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "Suggests": [ - "knitr", - "rmarkdown" - ], - "Collate": "'capt.R' 'options.R' 'pager.R' 'check.R' 'finalizer.R' 'misc.R' 'html.R' 'styles.R' 's4.R' 'core.R' 'diff.R' 'get.R' 'guides.R' 'hunks.R' 'layout.R' 'myerssimple.R' 'rdiff.R' 'rds.R' 'set.R' 'subset.R' 'summmary.R' 'system.R' 'text.R' 'tochar.R' 'trim.R' 'word.R'", - "Imports": [ - "crayon (>= 1.3.2)", - "tools", - "methods", - "utils", - "stats" - ], - "NeedsCompilation": "yes", - "Author": "Brodie Gaslam [aut, cre], Michael B. Allen [ctb, cph] (Original C implementation of Myers Diff Algorithm)", - "Maintainer": "Brodie Gaslam ", - "Repository": "CRAN" - }, - "digest": { - "Package": "digest", - "Version": "0.6.38", - "Source": "Repository", - "Authors@R": "c(person(\"Dirk\", \"Eddelbuettel\", role = c(\"aut\", \"cre\"), email = \"edd@debian.org\", comment = c(ORCID = \"0000-0001-6419-907X\")), person(\"Antoine\", \"Lucas\", role=\"ctb\", comment = c(ORCID = \"0000-0002-8059-9767\")), person(\"Jarek\", \"Tuszynski\", role=\"ctb\"), person(\"Henrik\", \"Bengtsson\", role=\"ctb\", comment = c(ORCID = \"0000-0002-7579-5165\")), person(\"Simon\", \"Urbanek\", role=\"ctb\", comment = c(ORCID = \"0000-0003-2297-1732\")), person(\"Mario\", \"Frasca\", role=\"ctb\"), person(\"Bryan\", \"Lewis\", role=\"ctb\"), person(\"Murray\", \"Stokely\", role=\"ctb\"), person(\"Hannes\", \"Muehleisen\", role=\"ctb\", comment = c(ORCID = \"0000-0001-8552-0029\")), person(\"Duncan\", \"Murdoch\", role=\"ctb\"), person(\"Jim\", \"Hester\", role=\"ctb\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"Wush\", \"Wu\", role=\"ctb\", comment = c(ORCID = \"0000-0001-5180-0567\")), person(\"Qiang\", \"Kou\", role=\"ctb\", comment = c(ORCID = \"0000-0001-6786-5453\")), person(\"Thierry\", \"Onkelinx\", role=\"ctb\", comment = c(ORCID = \"0000-0001-8804-4216\")), person(\"Michel\", \"Lang\", role=\"ctb\", comment = c(ORCID = \"0000-0001-9754-0393\")), person(\"Viliam\", \"Simko\", role=\"ctb\"), person(\"Kurt\", \"Hornik\", role=\"ctb\", comment = c(ORCID = \"0000-0003-4198-9911\")), person(\"Radford\", \"Neal\", role=\"ctb\", comment = c(ORCID = \"0000-0002-2473-3407\")), person(\"Kendon\", \"Bell\", role=\"ctb\", comment = c(ORCID = \"0000-0002-9093-8312\")), person(\"Matthew\", \"de Queljoe\", role=\"ctb\"), person(\"Dmitry\", \"Selivanov\", role=\"ctb\", comment = c(ORCID = \"0000-0003-0492-6647\")), person(\"Ion\", \"Suruceanu\", role=\"ctb\", comment = c(ORCID = \"0009-0005-6446-4909\")), person(\"Bill\", \"Denney\", role=\"ctb\", comment = c(ORCID = \"0000-0002-5759-428X\")), person(\"Dirk\", \"Schumacher\", role=\"ctb\"), person(\"András\", \"Svraka\", role=\"ctb\", comment = c(ORCID = \"0009-0008-8480-1329\")), person(\"Sergey\", \"Fedorov\", role=\"ctb\", comment = c(ORCID = \"0000-0002-5970-7233\")), person(\"Will\", \"Landau\", role=\"ctb\", comment = c(ORCID = \"0000-0003-1878-3253\")), person(\"Floris\", \"Vanderhaeghe\", role=\"ctb\", comment = c(ORCID = \"0000-0002-6378-6229\")), person(\"Kevin\", \"Tappe\", role=\"ctb\"), person(\"Harris\", \"McGehee\", role=\"ctb\"), person(\"Tim\", \"Mastny\", role=\"ctb\"), person(\"Aaron\", \"Peikert\", role=\"ctb\", comment = c(ORCID = \"0000-0001-7813-818X\")), person(\"Mark\", \"van der Loo\", role=\"ctb\", comment = c(ORCID = \"0000-0002-9807-4686\")), person(\"Chris\", \"Muir\", role=\"ctb\", comment = c(ORCID = \"0000-0003-2555-3878\")), person(\"Moritz\", \"Beller\", role=\"ctb\", comment = c(ORCID = \"0000-0003-4852-0526\")), person(\"Sebastian\", \"Campbell\", role=\"ctb\", comment = c(ORCID = \"0009-0000-5948-4503\")), person(\"Winston\", \"Chang\", role=\"ctb\", comment = c(ORCID = \"0000-0002-1576-2126\")), person(\"Dean\", \"Attali\", role=\"ctb\", comment = c(ORCID = \"0000-0002-5645-3493\")), person(\"Michael\", \"Chirico\", role=\"ctb\", comment = c(ORCID = \"0000-0003-0787-087X\")), person(\"Kevin\", \"Ushey\", role=\"ctb\", comment = c(ORCID = \"0000-0003-2880-7407\")), person(\"Carl\", \"Pearson\", role=\"ctb\", comment = c(ORCID = \"0000-0003-0701-7860\")))", - "Date": "2025-11-09", - "Title": "Create Compact Hash Digests of R Objects", - "Description": "Implementation of a function 'digest()' for the creation of hash digests of arbitrary R objects (using the 'md5', 'sha-1', 'sha-256', 'crc32', 'xxhash', 'murmurhash', 'spookyhash', 'blake3', 'crc32c', 'xxh3_64', and 'xxh3_128' algorithms) permitting easy comparison of R language objects, as well as functions such as 'hmac()' to create hash-based message authentication code. Please note that this package is not meant to be deployed for cryptographic purposes for which more comprehensive (and widely tested) libraries such as 'OpenSSL' should be used.", - "URL": "https://github.com/eddelbuettel/digest, https://eddelbuettel.github.io/digest/, https://dirk.eddelbuettel.com/code/digest.html", - "BugReports": "https://github.com/eddelbuettel/digest/issues", - "Depends": [ - "R (>= 3.3.0)" - ], - "Imports": [ - "utils" - ], - "License": "GPL (>= 2)", - "Suggests": [ - "tinytest", - "simplermarkdown", - "rbenchmark" - ], - "VignetteBuilder": "simplermarkdown", - "Encoding": "UTF-8", - "NeedsCompilation": "yes", - "Author": "Dirk Eddelbuettel [aut, cre] (ORCID: ), Antoine Lucas [ctb] (ORCID: ), Jarek Tuszynski [ctb], Henrik Bengtsson [ctb] (ORCID: ), Simon Urbanek [ctb] (ORCID: ), Mario Frasca [ctb], Bryan Lewis [ctb], Murray Stokely [ctb], Hannes Muehleisen [ctb] (ORCID: ), Duncan Murdoch [ctb], Jim Hester [ctb] (ORCID: ), Wush Wu [ctb] (ORCID: ), Qiang Kou [ctb] (ORCID: ), Thierry Onkelinx [ctb] (ORCID: ), Michel Lang [ctb] (ORCID: ), Viliam Simko [ctb], Kurt Hornik [ctb] (ORCID: ), Radford Neal [ctb] (ORCID: ), Kendon Bell [ctb] (ORCID: ), Matthew de Queljoe [ctb], Dmitry Selivanov [ctb] (ORCID: ), Ion Suruceanu [ctb] (ORCID: ), Bill Denney [ctb] (ORCID: ), Dirk Schumacher [ctb], András Svraka [ctb] (ORCID: ), Sergey Fedorov [ctb] (ORCID: ), Will Landau [ctb] (ORCID: ), Floris Vanderhaeghe [ctb] (ORCID: ), Kevin Tappe [ctb], Harris McGehee [ctb], Tim Mastny [ctb], Aaron Peikert [ctb] (ORCID: ), Mark van der Loo [ctb] (ORCID: ), Chris Muir [ctb] (ORCID: ), Moritz Beller [ctb] (ORCID: ), Sebastian Campbell [ctb] (ORCID: ), Winston Chang [ctb] (ORCID: ), Dean Attali [ctb] (ORCID: ), Michael Chirico [ctb] (ORCID: ), Kevin Ushey [ctb] (ORCID: ), Carl Pearson [ctb] (ORCID: )", - "Maintainer": "Dirk Eddelbuettel ", - "Repository": "CRAN" - }, - "downlit": { - "Package": "downlit", - "Version": "0.4.4", - "Source": "Repository", - "Title": "Syntax Highlighting and Automatic Linking", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Syntax highlighting of R code, specifically designed for the needs of 'RMarkdown' packages like 'pkgdown', 'hugodown', and 'bookdown'. It includes linking of function calls to their documentation on the web, and automatic translation of ANSI escapes in output to the equivalent HTML.", - "License": "MIT + file LICENSE", - "URL": "https://downlit.r-lib.org/, https://github.com/r-lib/downlit", - "BugReports": "https://github.com/r-lib/downlit/issues", - "Depends": [ - "R (>= 4.0.0)" - ], - "Imports": [ - "brio", - "desc", - "digest", - "evaluate", - "fansi", - "memoise", - "rlang", - "vctrs", - "withr", - "yaml" - ], - "Suggests": [ - "covr", - "htmltools", - "jsonlite", - "MASS", - "MassSpecWavelet", - "pkgload", - "rmarkdown", - "testthat (>= 3.0.0)", - "xml2" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.1", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre], Posit Software, PBC [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "RSPM" - }, - "dplyr": { - "Package": "dplyr", - "Version": "1.1.4", - "Source": "Repository", - "Type": "Package", - "Title": "A Grammar of Data Manipulation", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Romain\", \"François\", role = \"aut\", comment = c(ORCID = \"0000-0002-2444-4226\")), person(\"Lionel\", \"Henry\", role = \"aut\"), person(\"Kirill\", \"Müller\", role = \"aut\", comment = c(ORCID = \"0000-0002-1416-3412\")), person(\"Davis\", \"Vaughan\", , \"davis@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4777-038X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "A fast, consistent tool for working with data frame like objects, both in memory and out of memory.", - "License": "MIT + file LICENSE", - "URL": "https://dplyr.tidyverse.org, https://github.com/tidyverse/dplyr", - "BugReports": "https://github.com/tidyverse/dplyr/issues", - "Depends": [ - "R (>= 3.5.0)" - ], - "Imports": [ - "cli (>= 3.4.0)", - "generics", - "glue (>= 1.3.2)", - "lifecycle (>= 1.0.3)", - "magrittr (>= 1.5)", - "methods", - "pillar (>= 1.9.0)", - "R6", - "rlang (>= 1.1.0)", - "tibble (>= 3.2.0)", - "tidyselect (>= 1.2.0)", - "utils", - "vctrs (>= 0.6.4)" - ], - "Suggests": [ - "bench", - "broom", - "callr", - "covr", - "DBI", - "dbplyr (>= 2.2.1)", - "ggplot2", - "knitr", - "Lahman", - "lobstr", - "microbenchmark", - "nycflights13", - "purrr", - "rmarkdown", - "RMySQL", - "RPostgreSQL", - "RSQLite", - "stringi (>= 1.7.6)", - "testthat (>= 3.1.5)", - "tidyr (>= 1.3.0)", - "withr" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse, shiny, pkgdown, tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "LazyData": "true", - "RoxygenNote": "7.2.3", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut, cre] (), Romain François [aut] (), Lionel Henry [aut], Kirill Müller [aut] (), Davis Vaughan [aut] (), Posit Software, PBC [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "RSPM" - }, - "dtplyr": { - "Package": "dtplyr", - "Version": "1.3.2", - "Source": "Repository", - "Title": "Data Table Back-End for 'dplyr'", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"cre\", \"aut\")), person(\"Maximilian\", \"Girlich\", role = \"aut\"), person(\"Mark\", \"Fairbanks\", role = \"aut\"), person(\"Ryan\", \"Dickerson\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Provides a data.table backend for 'dplyr'. The goal of 'dtplyr' is to allow you to write 'dplyr' code that is automatically translated to the equivalent, but usually much faster, data.table code.", - "License": "MIT + file LICENSE", - "URL": "https://dtplyr.tidyverse.org, https://github.com/tidyverse/dtplyr", - "BugReports": "https://github.com/tidyverse/dtplyr/issues", - "Depends": [ - "R (>= 4.0)" - ], - "Imports": [ - "cli (>= 3.4.0)", - "data.table (>= 1.13.0)", - "dplyr (>= 1.1.0)", - "glue", - "lifecycle", - "rlang (>= 1.0.4)", - "tibble", - "tidyselect (>= 1.2.0)", - "vctrs (>= 0.4.1)" - ], - "Suggests": [ - "bench", - "covr", - "knitr", - "rmarkdown", - "testthat (>= 3.1.2)", - "tidyr (>= 1.1.0)", - "waldo (>= 0.3.1)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2.9000", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [cre, aut], Maximilian Girlich [aut], Mark Fairbanks [aut], Ryan Dickerson [aut], Posit Software, PBC [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "ellipsis": { - "Package": "ellipsis", - "Version": "0.3.2", - "Source": "Repository", - "Title": "Tools for Working with ...", - "Description": "The ellipsis is a powerful tool for extending functions. Unfortunately this power comes at a cost: misspelled arguments will be silently ignored. The ellipsis package provides a collection of functions to catch problems and alert the user.", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@rstudio.com\", role = c(\"aut\", \"cre\")), person(\"RStudio\", role = \"cph\") )", - "License": "MIT + file LICENSE", - "Encoding": "UTF-8", - "RoxygenNote": "7.1.1", - "URL": "https://ellipsis.r-lib.org, https://github.com/r-lib/ellipsis", - "BugReports": "https://github.com/r-lib/ellipsis/issues", - "Depends": [ - "R (>= 3.2)" - ], - "Imports": [ - "rlang (>= 0.3.0)" - ], - "Suggests": [ - "covr", - "testthat" - ], - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut, cre], RStudio [cph]", - "Maintainer": "Hadley Wickham ", - "Repository": "RSPM" - }, - "evaluate": { - "Package": "evaluate", - "Version": "1.0.5", - "Source": "Repository", - "Type": "Package", - "Title": "Parsing and Evaluation Tools that Provide More Details than the Default", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Yihui\", \"Xie\", role = \"aut\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Michael\", \"Lawrence\", role = \"ctb\"), person(\"Thomas\", \"Kluyver\", role = \"ctb\"), person(\"Jeroen\", \"Ooms\", role = \"ctb\"), person(\"Barret\", \"Schloerke\", role = \"ctb\"), person(\"Adam\", \"Ryczkowski\", role = \"ctb\"), person(\"Hiroaki\", \"Yutani\", role = \"ctb\"), person(\"Michel\", \"Lang\", role = \"ctb\"), person(\"Karolis\", \"Koncevičius\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Parsing and evaluation tools that make it easy to recreate the command line behaviour of R.", - "License": "MIT + file LICENSE", - "URL": "https://evaluate.r-lib.org/, https://github.com/r-lib/evaluate", - "BugReports": "https://github.com/r-lib/evaluate/issues", - "Depends": [ - "R (>= 3.6.0)" - ], - "Suggests": [ - "callr", - "covr", - "ggplot2 (>= 3.3.6)", - "lattice", - "methods", - "pkgload", - "ragg (>= 1.4.0)", - "rlang (>= 1.1.5)", - "knitr", - "testthat (>= 3.0.0)", - "withr" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre], Yihui Xie [aut] (ORCID: ), Michael Lawrence [ctb], Thomas Kluyver [ctb], Jeroen Ooms [ctb], Barret Schloerke [ctb], Adam Ryczkowski [ctb], Hiroaki Yutani [ctb], Michel Lang [ctb], Karolis Koncevičius [ctb], Posit Software, PBC [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "fansi": { - "Package": "fansi", - "Version": "1.0.6", - "Source": "Repository", - "Title": "ANSI Control Sequence Aware String Functions", - "Description": "Counterparts to R string manipulation functions that account for the effects of ANSI text formatting control sequences.", - "Authors@R": "c( person(\"Brodie\", \"Gaslam\", email=\"brodie.gaslam@yahoo.com\", role=c(\"aut\", \"cre\")), person(\"Elliott\", \"Sales De Andrade\", role=\"ctb\"), person(family=\"R Core Team\", email=\"R-core@r-project.org\", role=\"cph\", comment=\"UTF8 byte length calcs from src/util.c\" ))", - "Depends": [ - "R (>= 3.1.0)" - ], - "License": "GPL-2 | GPL-3", - "URL": "https://github.com/brodieG/fansi", - "BugReports": "https://github.com/brodieG/fansi/issues", - "VignetteBuilder": "knitr", - "Suggests": [ - "unitizer", - "knitr", - "rmarkdown" - ], - "Imports": [ - "grDevices", - "utils" - ], - "RoxygenNote": "7.2.3", - "Encoding": "UTF-8", - "Collate": "'constants.R' 'fansi-package.R' 'internal.R' 'load.R' 'misc.R' 'nchar.R' 'strwrap.R' 'strtrim.R' 'strsplit.R' 'substr2.R' 'trimws.R' 'tohtml.R' 'unhandled.R' 'normalize.R' 'sgr.R'", - "NeedsCompilation": "yes", - "Author": "Brodie Gaslam [aut, cre], Elliott Sales De Andrade [ctb], R Core Team [cph] (UTF8 byte length calcs from src/util.c)", - "Maintainer": "Brodie Gaslam ", - "Repository": "RSPM" - }, - "farver": { - "Package": "farver", - "Version": "2.1.2", - "Source": "Repository", - "Type": "Package", - "Title": "High Performance Colour Space Manipulation", - "Authors@R": "c( person(\"Thomas Lin\", \"Pedersen\", , \"thomas.pedersen@posit.co\", role = c(\"cre\", \"aut\"), comment = c(ORCID = \"0000-0002-5147-4711\")), person(\"Berendea\", \"Nicolae\", role = \"aut\", comment = \"Author of the ColorSpace C++ library\"), person(\"Romain\", \"François\", , \"romain@purrple.cat\", role = \"aut\", comment = c(ORCID = \"0000-0002-2444-4226\")), person(\"Posit, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "The encoding of colour can be handled in many different ways, using different colour spaces. As different colour spaces have different uses, efficient conversion between these representations are important. The 'farver' package provides a set of functions that gives access to very fast colour space conversion and comparisons implemented in C++, and offers speed improvements over the 'convertColor' function in the 'grDevices' package.", - "License": "MIT + file LICENSE", - "URL": "https://farver.data-imaginist.com, https://github.com/thomasp85/farver", - "BugReports": "https://github.com/thomasp85/farver/issues", - "Suggests": [ - "covr", - "testthat (>= 3.0.0)" - ], - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.1", - "NeedsCompilation": "yes", - "Author": "Thomas Lin Pedersen [cre, aut] (), Berendea Nicolae [aut] (Author of the ColorSpace C++ library), Romain François [aut] (), Posit, PBC [cph, fnd]", - "Maintainer": "Thomas Lin Pedersen ", - "Repository": "RSPM" - }, - "fastmap": { - "Package": "fastmap", - "Version": "1.2.0", - "Source": "Repository", - "Title": "Fast Data Structures", - "Authors@R": "c( person(\"Winston\", \"Chang\", email = \"winston@posit.co\", role = c(\"aut\", \"cre\")), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(given = \"Tessil\", role = \"cph\", comment = \"hopscotch_map library\") )", - "Description": "Fast implementation of data structures, including a key-value store, stack, and queue. Environments are commonly used as key-value stores in R, but every time a new key is used, it is added to R's global symbol table, causing a small amount of memory leakage. This can be problematic in cases where many different keys are used. Fastmap avoids this memory leak issue by implementing the map using data structures in C++.", - "License": "MIT + file LICENSE", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.3", - "Suggests": [ - "testthat (>= 2.1.1)" - ], - "URL": "https://r-lib.github.io/fastmap/, https://github.com/r-lib/fastmap", - "BugReports": "https://github.com/r-lib/fastmap/issues", - "NeedsCompilation": "yes", - "Author": "Winston Chang [aut, cre], Posit Software, PBC [cph, fnd], Tessil [cph] (hopscotch_map library)", - "Maintainer": "Winston Chang ", - "Repository": "RSPM" - }, - "fontawesome": { - "Package": "fontawesome", - "Version": "0.5.3", - "Source": "Repository", - "Type": "Package", - "Title": "Easily Work with 'Font Awesome' Icons", - "Description": "Easily and flexibly insert 'Font Awesome' icons into 'R Markdown' documents and 'Shiny' apps. These icons can be inserted into HTML content through inline 'SVG' tags or 'i' tags. There is also a utility function for exporting 'Font Awesome' icons as 'PNG' images for those situations where raster graphics are needed.", - "Authors@R": "c( person(\"Richard\", \"Iannone\", , \"rich@posit.co\", c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-3925-190X\")), person(\"Christophe\", \"Dervieux\", , \"cderv@posit.co\", role = \"ctb\", comment = c(ORCID = \"0000-0003-4474-2498\")), person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = \"ctb\"), person(\"Dave\", \"Gandy\", role = c(\"ctb\", \"cph\"), comment = \"Font-Awesome font\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "License": "MIT + file LICENSE", - "URL": "https://github.com/rstudio/fontawesome, https://rstudio.github.io/fontawesome/", - "BugReports": "https://github.com/rstudio/fontawesome/issues", - "Encoding": "UTF-8", - "ByteCompile": "true", - "RoxygenNote": "7.3.2", - "Depends": [ - "R (>= 3.3.0)" - ], - "Imports": [ - "rlang (>= 1.0.6)", - "htmltools (>= 0.5.1.1)" - ], - "Suggests": [ - "covr", - "dplyr (>= 1.0.8)", - "gt (>= 0.9.0)", - "knitr (>= 1.31)", - "testthat (>= 3.0.0)", - "rsvg" - ], - "Config/testthat/edition": "3", - "NeedsCompilation": "no", - "Author": "Richard Iannone [aut, cre] (), Christophe Dervieux [ctb] (), Winston Chang [ctb], Dave Gandy [ctb, cph] (Font-Awesome font), Posit Software, PBC [cph, fnd]", - "Maintainer": "Richard Iannone ", - "Repository": "CRAN" - }, - "forcats": { - "Package": "forcats", - "Version": "1.0.1", - "Source": "Repository", - "Title": "Tools for Working with Categorical Variables (Factors)", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Helpers for reordering factor levels (including moving specified levels to front, ordering by first appearance, reversing, and randomly shuffling), and tools for modifying factor levels (including collapsing rare levels into other, 'anonymising', and manually 'recoding').", - "License": "MIT + file LICENSE", - "URL": "https://forcats.tidyverse.org/, https://github.com/tidyverse/forcats", - "BugReports": "https://github.com/tidyverse/forcats/issues", - "Depends": [ - "R (>= 4.1)" - ], - "Imports": [ - "cli (>= 3.4.0)", - "glue", - "lifecycle", - "magrittr", - "rlang (>= 1.0.0)", - "tibble" - ], - "Suggests": [ - "covr", - "dplyr", - "ggplot2", - "knitr", - "readr", - "rmarkdown", - "testthat (>= 3.0.0)", - "withr" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "LazyData": "true", - "RoxygenNote": "7.3.3", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre], Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "fs": { - "Package": "fs", - "Version": "1.6.6", - "Source": "Repository", - "Title": "Cross-Platform File System Operations Based on 'libuv'", - "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"libuv project contributors\", role = \"cph\", comment = \"libuv library\"), person(\"Joyent, Inc. and other Node contributors\", role = \"cph\", comment = \"libuv library\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "A cross-platform interface to file system operations, built on top of the 'libuv' C library.", - "License": "MIT + file LICENSE", - "URL": "https://fs.r-lib.org, https://github.com/r-lib/fs", - "BugReports": "https://github.com/r-lib/fs/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "methods" - ], - "Suggests": [ - "covr", - "crayon", - "knitr", - "pillar (>= 1.0.0)", - "rmarkdown", - "spelling", - "testthat (>= 3.0.0)", - "tibble (>= 1.1.0)", - "vctrs (>= 0.3.0)", - "withr" - ], - "VignetteBuilder": "knitr", - "ByteCompile": "true", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Copyright": "file COPYRIGHTS", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.2.3", - "SystemRequirements": "GNU make", - "NeedsCompilation": "yes", - "Author": "Jim Hester [aut], Hadley Wickham [aut], Gábor Csárdi [aut, cre], libuv project contributors [cph] (libuv library), Joyent, Inc. and other Node contributors [cph] (libuv library), Posit Software, PBC [cph, fnd]", - "Maintainer": "Gábor Csárdi ", - "Repository": "CRAN" - }, - "gargle": { - "Package": "gargle", - "Version": "1.6.0", - "Source": "Repository", - "Title": "Utilities for Working with Google APIs", - "Authors@R": "c( person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Craig\", \"Citro\", , \"craigcitro@google.com\", role = \"aut\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Google Inc\", role = \"cph\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Provides utilities for working with Google APIs . This includes functions and classes for handling common credential types and for preparing, executing, and processing HTTP requests.", - "License": "MIT + file LICENSE", - "URL": "https://gargle.r-lib.org, https://github.com/r-lib/gargle", - "BugReports": "https://github.com/r-lib/gargle/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "cli (>= 3.0.1)", - "fs (>= 1.3.1)", - "glue (>= 1.3.0)", - "httr (>= 1.4.5)", - "jsonlite", - "lifecycle (>= 0.2.0)", - "openssl", - "rappdirs", - "rlang (>= 1.1.0)", - "stats", - "utils", - "withr" - ], - "Suggests": [ - "aws.ec2metadata", - "aws.signature", - "covr", - "httpuv", - "knitr", - "rmarkdown", - "sodium", - "spelling", - "testthat (>= 3.1.7)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.2.9000", - "NeedsCompilation": "no", - "Author": "Jennifer Bryan [aut, cre] (ORCID: ), Craig Citro [aut], Hadley Wickham [aut] (ORCID: ), Google Inc [cph], Posit Software, PBC [cph, fnd]", - "Maintainer": "Jennifer Bryan ", - "Repository": "CRAN" - }, - "generics": { - "Package": "generics", - "Version": "0.1.4", - "Source": "Repository", - "Title": "Common S3 Generics not Provided by Base R Methods Related to Model Fitting", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Max\", \"Kuhn\", , \"max@posit.co\", role = \"aut\"), person(\"Davis\", \"Vaughan\", , \"davis@posit.co\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"https://ror.org/03wc8by49\")) )", - "Description": "In order to reduce potential package dependencies and conflicts, generics provides a number of commonly used S3 generics.", - "License": "MIT + file LICENSE", - "URL": "https://generics.r-lib.org, https://github.com/r-lib/generics", - "BugReports": "https://github.com/r-lib/generics/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "methods" - ], - "Suggests": [ - "covr", - "pkgload", - "testthat (>= 3.0.0)", - "tibble", - "withr" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre] (ORCID: ), Max Kuhn [aut], Davis Vaughan [aut], Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "gert": { - "Package": "gert", - "Version": "2.1.5", - "Source": "Repository", - "Type": "Package", - "Title": "Simple Git Client for R", - "Authors@R": "c( person(\"Jeroen\", \"Ooms\", role = c(\"aut\", \"cre\"), email = \"jeroenooms@gmail.com\", comment = c(ORCID = \"0000-0002-4035-0289\")), person(\"Jennifer\", \"Bryan\", role = \"ctb\", email = \"jenny@posit.co\", comment = c(ORCID = \"0000-0002-6983-2759\")))", - "Description": "Simple git client for R based on 'libgit2' with support for SSH and HTTPS remotes. All functions in 'gert' use basic R data types (such as vectors and data-frames) for their arguments and return values. User credentials are shared with command line 'git' through the git-credential store and ssh keys stored on disk or ssh-agent.", - "License": "MIT + file LICENSE", - "URL": "https://docs.ropensci.org/gert/, https://ropensci.r-universe.dev/gert", - "BugReports": "https://github.com/r-lib/gert/issues", - "Imports": [ - "askpass", - "credentials (>= 1.2.1)", - "openssl (>= 2.0.3)", - "rstudioapi (>= 0.11)", - "sys", - "zip (>= 2.1.0)" - ], - "Suggests": [ - "spelling", - "knitr", - "rmarkdown", - "testthat" - ], - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2.9000", - "SystemRequirements": "libgit2 (>= 1.0): libgit2-devel (rpm) or libgit2-dev (deb)", - "Language": "en-US", - "NeedsCompilation": "yes", - "Author": "Jeroen Ooms [aut, cre] (), Jennifer Bryan [ctb] ()", - "Maintainer": "Jeroen Ooms ", - "Repository": "CRAN" - }, - "ggplot2": { - "Package": "ggplot2", - "Version": "4.0.0", - "Source": "Repository", - "Title": "Create Elegant Data Visualisations Using the Grammar of Graphics", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Winston\", \"Chang\", role = \"aut\", comment = c(ORCID = \"0000-0002-1576-2126\")), person(\"Lionel\", \"Henry\", role = \"aut\"), person(\"Thomas Lin\", \"Pedersen\", , \"thomas.pedersen@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-5147-4711\")), person(\"Kohske\", \"Takahashi\", role = \"aut\"), person(\"Claus\", \"Wilke\", role = \"aut\", comment = c(ORCID = \"0000-0002-7470-9261\")), person(\"Kara\", \"Woo\", role = \"aut\", comment = c(ORCID = \"0000-0002-5125-4188\")), person(\"Hiroaki\", \"Yutani\", role = \"aut\", comment = c(ORCID = \"0000-0002-3385-7233\")), person(\"Dewey\", \"Dunnington\", role = \"aut\", comment = c(ORCID = \"0000-0002-9415-4582\")), person(\"Teun\", \"van den Brand\", role = \"aut\", comment = c(ORCID = \"0000-0002-9335-7468\")), person(\"Posit, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "A system for 'declaratively' creating graphics, based on \"The Grammar of Graphics\". You provide the data, tell 'ggplot2' how to map variables to aesthetics, what graphical primitives to use, and it takes care of the details.", - "License": "MIT + file LICENSE", - "URL": "https://ggplot2.tidyverse.org, https://github.com/tidyverse/ggplot2", - "BugReports": "https://github.com/tidyverse/ggplot2/issues", - "Depends": [ - "R (>= 4.1)" - ], - "Imports": [ - "cli", - "grDevices", - "grid", - "gtable (>= 0.3.6)", - "isoband", - "lifecycle (> 1.0.1)", - "rlang (>= 1.1.0)", - "S7", - "scales (>= 1.4.0)", - "stats", - "vctrs (>= 0.6.0)", - "withr (>= 2.5.0)" - ], - "Suggests": [ - "broom", - "covr", - "dplyr", - "ggplot2movies", - "hexbin", - "Hmisc", - "knitr", - "mapproj", - "maps", - "MASS", - "mgcv", - "multcomp", - "munsell", - "nlme", - "profvis", - "quantreg", - "ragg (>= 1.2.6)", - "RColorBrewer", - "rmarkdown", - "roxygen2", - "rpart", - "sf (>= 0.7-3)", - "svglite (>= 2.1.2)", - "testthat (>= 3.1.5)", - "tibble", - "vdiffr (>= 1.0.6)", - "xml2" - ], - "Enhances": [ - "sp" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "ggtext, tidyr, forcats, tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/usethis/last-upkeep": "2025-04-23", - "Encoding": "UTF-8", - "LazyData": "true", - "RoxygenNote": "7.3.2", - "Collate": "'ggproto.R' 'ggplot-global.R' 'aaa-.R' 'aes-colour-fill-alpha.R' 'aes-evaluation.R' 'aes-group-order.R' 'aes-linetype-size-shape.R' 'aes-position.R' 'all-classes.R' 'compat-plyr.R' 'utilities.R' 'aes.R' 'annotation-borders.R' 'utilities-checks.R' 'legend-draw.R' 'geom-.R' 'annotation-custom.R' 'annotation-logticks.R' 'scale-type.R' 'layer.R' 'make-constructor.R' 'geom-polygon.R' 'geom-map.R' 'annotation-map.R' 'geom-raster.R' 'annotation-raster.R' 'annotation.R' 'autolayer.R' 'autoplot.R' 'axis-secondary.R' 'backports.R' 'bench.R' 'bin.R' 'coord-.R' 'coord-cartesian-.R' 'coord-fixed.R' 'coord-flip.R' 'coord-map.R' 'coord-munch.R' 'coord-polar.R' 'coord-quickmap.R' 'coord-radial.R' 'coord-sf.R' 'coord-transform.R' 'data.R' 'docs_layer.R' 'facet-.R' 'facet-grid-.R' 'facet-null.R' 'facet-wrap.R' 'fortify-map.R' 'fortify-models.R' 'fortify-spatial.R' 'fortify.R' 'stat-.R' 'geom-abline.R' 'geom-rect.R' 'geom-bar.R' 'geom-tile.R' 'geom-bin2d.R' 'geom-blank.R' 'geom-boxplot.R' 'geom-col.R' 'geom-path.R' 'geom-contour.R' 'geom-point.R' 'geom-count.R' 'geom-crossbar.R' 'geom-segment.R' 'geom-curve.R' 'geom-defaults.R' 'geom-ribbon.R' 'geom-density.R' 'geom-density2d.R' 'geom-dotplot.R' 'geom-errorbar.R' 'geom-freqpoly.R' 'geom-function.R' 'geom-hex.R' 'geom-histogram.R' 'geom-hline.R' 'geom-jitter.R' 'geom-label.R' 'geom-linerange.R' 'geom-pointrange.R' 'geom-quantile.R' 'geom-rug.R' 'geom-sf.R' 'geom-smooth.R' 'geom-spoke.R' 'geom-text.R' 'geom-violin.R' 'geom-vline.R' 'ggplot2-package.R' 'grob-absolute.R' 'grob-dotstack.R' 'grob-null.R' 'grouping.R' 'properties.R' 'margins.R' 'theme-elements.R' 'guide-.R' 'guide-axis.R' 'guide-axis-logticks.R' 'guide-axis-stack.R' 'guide-axis-theta.R' 'guide-legend.R' 'guide-bins.R' 'guide-colorbar.R' 'guide-colorsteps.R' 'guide-custom.R' 'guide-none.R' 'guide-old.R' 'guides-.R' 'guides-grid.R' 'hexbin.R' 'import-standalone-obj-type.R' 'import-standalone-types-check.R' 'labeller.R' 'labels.R' 'layer-sf.R' 'layout.R' 'limits.R' 'performance.R' 'plot-build.R' 'plot-construction.R' 'plot-last.R' 'plot.R' 'position-.R' 'position-collide.R' 'position-dodge.R' 'position-dodge2.R' 'position-identity.R' 'position-jitter.R' 'position-jitterdodge.R' 'position-nudge.R' 'position-stack.R' 'quick-plot.R' 'reshape-add-margins.R' 'save.R' 'scale-.R' 'scale-alpha.R' 'scale-binned.R' 'scale-brewer.R' 'scale-colour.R' 'scale-continuous.R' 'scale-date.R' 'scale-discrete-.R' 'scale-expansion.R' 'scale-gradient.R' 'scale-grey.R' 'scale-hue.R' 'scale-identity.R' 'scale-linetype.R' 'scale-linewidth.R' 'scale-manual.R' 'scale-shape.R' 'scale-size.R' 'scale-steps.R' 'scale-view.R' 'scale-viridis.R' 'scales-.R' 'stat-align.R' 'stat-bin.R' 'stat-summary-2d.R' 'stat-bin2d.R' 'stat-bindot.R' 'stat-binhex.R' 'stat-boxplot.R' 'stat-connect.R' 'stat-contour.R' 'stat-count.R' 'stat-density-2d.R' 'stat-density.R' 'stat-ecdf.R' 'stat-ellipse.R' 'stat-function.R' 'stat-identity.R' 'stat-manual.R' 'stat-qq-line.R' 'stat-qq.R' 'stat-quantilemethods.R' 'stat-sf-coordinates.R' 'stat-sf.R' 'stat-smooth-methods.R' 'stat-smooth.R' 'stat-sum.R' 'stat-summary-bin.R' 'stat-summary-hex.R' 'stat-summary.R' 'stat-unique.R' 'stat-ydensity.R' 'summarise-plot.R' 'summary.R' 'theme.R' 'theme-defaults.R' 'theme-current.R' 'theme-sub.R' 'utilities-break.R' 'utilities-grid.R' 'utilities-help.R' 'utilities-patterns.R' 'utilities-resolution.R' 'utilities-tidy-eval.R' 'zxx.R' 'zzz.R'", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut] (ORCID: ), Winston Chang [aut] (ORCID: ), Lionel Henry [aut], Thomas Lin Pedersen [aut, cre] (ORCID: ), Kohske Takahashi [aut], Claus Wilke [aut] (ORCID: ), Kara Woo [aut] (ORCID: ), Hiroaki Yutani [aut] (ORCID: ), Dewey Dunnington [aut] (ORCID: ), Teun van den Brand [aut] (ORCID: ), Posit, PBC [cph, fnd] (ROR: )", - "Maintainer": "Thomas Lin Pedersen ", - "Repository": "CRAN" - }, - "gh": { - "Package": "gh", - "Version": "1.5.0", - "Source": "Repository", - "Title": "'GitHub' 'API'", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"cre\", \"ctb\")), person(\"Jennifer\", \"Bryan\", role = \"aut\"), person(\"Hadley\", \"Wickham\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Minimal client to access the 'GitHub' 'API'.", - "License": "MIT + file LICENSE", - "URL": "https://gh.r-lib.org/, https://github.com/r-lib/gh#readme", - "BugReports": "https://github.com/r-lib/gh/issues", - "Depends": [ - "R (>= 4.1)" - ], - "Imports": [ - "cli (>= 3.0.1)", - "gitcreds", - "glue", - "httr2 (>= 1.0.6)", - "ini", - "jsonlite", - "lifecycle", - "rlang (>= 1.0.0)" - ], - "Suggests": [ - "connectcreds", - "covr", - "knitr", - "rmarkdown", - "rprojroot", - "spelling", - "testthat (>= 3.0.0)", - "withr" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/usethis/last-upkeep": "2025-04-29", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.2.9000", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [cre, ctb], Jennifer Bryan [aut], Hadley Wickham [aut], Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Gábor Csárdi ", - "Repository": "CRAN" - }, - "gitcreds": { - "Package": "gitcreds", - "Version": "0.1.2", - "Source": "Repository", - "Title": "Query 'git' Credentials from 'R'", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"RStudio\", role = c(\"cph\", \"fnd\")) )", - "Description": "Query, set, delete credentials from the 'git' credential store. Manage 'GitHub' tokens and other 'git' credentials. This package is to be used by other packages that need to authenticate to 'GitHub' and/or other 'git' repositories.", - "License": "MIT + file LICENSE", - "URL": "https://gitcreds.r-lib.org/, https://github.com/r-lib/gitcreds", - "BugReports": "https://github.com/r-lib/gitcreds/issues", - "Depends": [ - "R (>= 3.4)" - ], - "Suggests": [ - "codetools", - "covr", - "knitr", - "mockery", - "oskeyring", - "rmarkdown", - "testthat (>= 3.0.0)", - "withr" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.1.9000", - "SystemRequirements": "git", - "Config/testthat/edition": "3", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [aut, cre], RStudio [cph, fnd]", - "Maintainer": "Gábor Csárdi ", - "Repository": "RSPM" - }, - "glue": { - "Package": "glue", - "Version": "1.8.0", - "Source": "Repository", - "Title": "Interpreted String Literals", - "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "An implementation of interpreted string literals, inspired by Python's Literal String Interpolation and Docstrings and Julia's Triple-Quoted String Literals .", - "License": "MIT + file LICENSE", - "URL": "https://glue.tidyverse.org/, https://github.com/tidyverse/glue", - "BugReports": "https://github.com/tidyverse/glue/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "methods" - ], - "Suggests": [ - "crayon", - "DBI (>= 1.2.0)", - "dplyr", - "knitr", - "magrittr", - "rlang", - "rmarkdown", - "RSQLite", - "testthat (>= 3.2.0)", - "vctrs (>= 0.3.0)", - "waldo (>= 0.5.3)", - "withr" - ], - "VignetteBuilder": "knitr", - "ByteCompile": "true", - "Config/Needs/website": "bench, forcats, ggbeeswarm, ggplot2, R.utils, rprintf, tidyr, tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "yes", - "Author": "Jim Hester [aut] (), Jennifer Bryan [aut, cre] (), Posit Software, PBC [cph, fnd]", - "Maintainer": "Jennifer Bryan ", - "Repository": "CRAN" - }, - "googledrive": { - "Package": "googledrive", - "Version": "2.1.2", - "Source": "Repository", - "Title": "An Interface to Google Drive", - "Authors@R": "c( person(\"Lucy\", \"D'Agostino McGowan\", , role = \"aut\"), person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Manage Google Drive files from R.", - "License": "MIT + file LICENSE", - "URL": "https://googledrive.tidyverse.org, https://github.com/tidyverse/googledrive", - "BugReports": "https://github.com/tidyverse/googledrive/issues", - "Depends": [ - "R (>= 4.1)" - ], - "Imports": [ - "cli (>= 3.0.0)", - "gargle (>= 1.6.0)", - "glue (>= 1.4.2)", - "httr", - "jsonlite", - "lifecycle", - "magrittr", - "pillar (>= 1.9.0)", - "purrr (>= 1.0.1)", - "rlang (>= 1.0.2)", - "tibble (>= 2.0.0)", - "utils", - "uuid", - "vctrs (>= 0.3.0)", - "withr" - ], - "Suggests": [ - "curl", - "dplyr (>= 1.0.0)", - "knitr", - "rmarkdown", - "spelling", - "testthat (>= 3.1.5)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse, tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.3", - "NeedsCompilation": "no", - "Author": "Lucy D'Agostino McGowan [aut], Jennifer Bryan [aut, cre] (ORCID: ), Posit Software, PBC [cph, fnd]", - "Maintainer": "Jennifer Bryan ", - "Repository": "CRAN" - }, - "googlesheets4": { - "Package": "googlesheets4", - "Version": "1.1.2", - "Source": "Repository", - "Title": "Access Google Sheets using the Sheets API V4", - "Authors@R": "c( person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"cre\", \"aut\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Interact with Google Sheets through the Sheets API v4 . \"API\" is an acronym for \"application programming interface\"; the Sheets API allows users to interact with Google Sheets programmatically, instead of via a web browser. The \"v4\" refers to the fact that the Sheets API is currently at version 4. This package can read and write both the metadata and the cell data in a Sheet.", - "License": "MIT + file LICENSE", - "URL": "https://googlesheets4.tidyverse.org, https://github.com/tidyverse/googlesheets4", - "BugReports": "https://github.com/tidyverse/googlesheets4/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "cellranger", - "cli (>= 3.0.0)", - "curl", - "gargle (>= 1.6.0)", - "glue (>= 1.3.0)", - "googledrive (>= 2.1.0)", - "httr", - "ids", - "lifecycle", - "magrittr", - "methods", - "purrr", - "rematch2", - "rlang (>= 1.0.2)", - "tibble (>= 2.1.1)", - "utils", - "vctrs (>= 0.2.3)", - "withr" - ], - "Suggests": [ - "readr", - "rmarkdown", - "spelling", - "testthat (>= 3.1.7)" - ], - "ByteCompile": "true", - "Config/Needs/website": "tidyverse, tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.2.9000", - "NeedsCompilation": "no", - "Author": "Jennifer Bryan [cre, aut] (ORCID: ), Posit Software, PBC [cph, fnd]", - "Maintainer": "Jennifer Bryan ", - "Repository": "CRAN" - }, - "gtable": { - "Package": "gtable", - "Version": "0.3.6", - "Source": "Repository", - "Title": "Arrange 'Grobs' in Tables", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Thomas Lin\", \"Pedersen\", , \"thomas.pedersen@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Tools to make it easier to work with \"tables\" of 'grobs'. The 'gtable' package defines a 'gtable' grob class that specifies a grid along with a list of grobs and their placement in the grid. Further the package makes it easy to manipulate and combine 'gtable' objects so that complex compositions can be built up sequentially.", - "License": "MIT + file LICENSE", - "URL": "https://gtable.r-lib.org, https://github.com/r-lib/gtable", - "BugReports": "https://github.com/r-lib/gtable/issues", - "Depends": [ - "R (>= 4.0)" - ], - "Imports": [ - "cli", - "glue", - "grid", - "lifecycle", - "rlang (>= 1.1.0)", - "stats" - ], - "Suggests": [ - "covr", - "ggplot2", - "knitr", - "profvis", - "rmarkdown", - "testthat (>= 3.0.0)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/usethis/last-upkeep": "2024-10-25", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut], Thomas Lin Pedersen [aut, cre], Posit Software, PBC [cph, fnd]", - "Maintainer": "Thomas Lin Pedersen ", - "Repository": "CRAN" - }, - "haven": { - "Package": "haven", - "Version": "2.5.5", - "Source": "Repository", - "Title": "Import and Export 'SPSS', 'Stata' and 'SAS' Files", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Evan\", \"Miller\", role = c(\"aut\", \"cph\"), comment = \"Author of included ReadStat code\"), person(\"Danny\", \"Smith\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Import foreign statistical formats into R via the embedded 'ReadStat' C library, .", - "License": "MIT + file LICENSE", - "URL": "https://haven.tidyverse.org, https://github.com/tidyverse/haven, https://github.com/WizardMac/ReadStat", - "BugReports": "https://github.com/tidyverse/haven/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "cli (>= 3.0.0)", - "forcats (>= 0.2.0)", - "hms", - "lifecycle", - "methods", - "readr (>= 0.1.0)", - "rlang (>= 0.4.0)", - "tibble", - "tidyselect", - "vctrs (>= 0.3.0)" - ], - "Suggests": [ - "covr", - "crayon", - "fs", - "knitr", - "pillar (>= 1.4.0)", - "rmarkdown", - "testthat (>= 3.0.0)", - "utf8" - ], - "LinkingTo": [ - "cpp11" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "SystemRequirements": "GNU make, zlib: zlib1g-dev (deb), zlib-devel (rpm)", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut, cre], Evan Miller [aut, cph] (Author of included ReadStat code), Danny Smith [aut], Posit Software, PBC [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "here": { - "Package": "here", - "Version": "1.0.2", - "Source": "Repository", - "Title": "A Simpler Way to Find Your Files", - "Date": "2025-09-06", - "Authors@R": "c(person(given = \"Kirill\", family = \"M\\u00fcller\", role = c(\"aut\", \"cre\"), email = \"kirill@cynkra.com\", comment = c(ORCID = \"0000-0002-1416-3412\")), person(given = \"Jennifer\", family = \"Bryan\", role = \"ctb\", email = \"jenny@rstudio.com\", comment = c(ORCID = \"0000-0002-6983-2759\")))", - "Description": "Constructs paths to your project's files. Declare the relative path of a file within your project with 'i_am()'. Use the 'here()' function as a drop-in replacement for 'file.path()', it will always locate the files relative to your project root.", - "License": "MIT + file LICENSE", - "URL": "https://here.r-lib.org/, https://github.com/r-lib/here", - "BugReports": "https://github.com/r-lib/here/issues", - "Imports": [ - "rprojroot (>= 2.1.0)" - ], - "Suggests": [ - "conflicted", - "covr", - "fs", - "knitr", - "palmerpenguins", - "plyr", - "readr", - "rlang", - "rmarkdown", - "testthat", - "uuid", - "withr" - ], - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.3.9000", - "Config/testthat/edition": "3", - "Config/Needs/website": "tidyverse/tidytemplate", - "NeedsCompilation": "no", - "Author": "Kirill Müller [aut, cre] (ORCID: ), Jennifer Bryan [ctb] (ORCID: )", - "Maintainer": "Kirill Müller ", - "Repository": "CRAN" - }, - "highr": { - "Package": "highr", - "Version": "0.11", - "Source": "Repository", - "Type": "Package", - "Title": "Syntax Highlighting for R Source Code", - "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Yixuan\", \"Qiu\", role = \"aut\"), person(\"Christopher\", \"Gandrud\", role = \"ctb\"), person(\"Qiang\", \"Li\", role = \"ctb\") )", - "Description": "Provides syntax highlighting for R source code. Currently it supports LaTeX and HTML output. Source code of other languages is supported via Andre Simon's highlight package ().", - "Depends": [ - "R (>= 3.3.0)" - ], - "Imports": [ - "xfun (>= 0.18)" - ], - "Suggests": [ - "knitr", - "markdown", - "testit" - ], - "License": "GPL", - "URL": "https://github.com/yihui/highr", - "BugReports": "https://github.com/yihui/highr/issues", - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.1", - "NeedsCompilation": "no", - "Author": "Yihui Xie [aut, cre] (), Yixuan Qiu [aut], Christopher Gandrud [ctb], Qiang Li [ctb]", - "Maintainer": "Yihui Xie ", - "Repository": "RSPM" - }, - "hms": { - "Package": "hms", - "Version": "1.1.4", - "Source": "Repository", - "Title": "Pretty Time of Day", - "Date": "2025-10-11", - "Authors@R": "c( person(\"Kirill\", \"Müller\", , \"kirill@cynkra.com\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-1416-3412\")), person(\"R Consortium\", role = \"fnd\"), person(\"Posit Software, PBC\", role = \"fnd\", comment = c(ROR = \"03wc8by49\")) )", - "Description": "Implements an S3 class for storing and formatting time-of-day values, based on the 'difftime' class.", - "License": "MIT + file LICENSE", - "URL": "https://hms.tidyverse.org/, https://github.com/tidyverse/hms", - "BugReports": "https://github.com/tidyverse/hms/issues", - "Imports": [ - "cli", - "lifecycle", - "methods", - "pkgconfig", - "rlang (>= 1.0.2)", - "vctrs (>= 0.3.8)" - ], - "Suggests": [ - "crayon", - "lubridate", - "pillar (>= 1.1.0)", - "testthat (>= 3.0.0)" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.3.9000", - "NeedsCompilation": "no", - "Author": "Kirill Müller [aut, cre] (ORCID: ), R Consortium [fnd], Posit Software, PBC [fnd] (ROR: )", - "Maintainer": "Kirill Müller ", - "Repository": "CRAN" - }, - "htmltools": { - "Package": "htmltools", - "Version": "0.5.8.1", - "Source": "Repository", - "Type": "Package", - "Title": "Tools for HTML", - "Authors@R": "c( person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Carson\", \"Sievert\", , \"carson@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Barret\", \"Schloerke\", , \"barret@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0001-9986-114X\")), person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0002-1576-2126\")), person(\"Yihui\", \"Xie\", , \"yihui@posit.co\", role = \"aut\"), person(\"Jeff\", \"Allen\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Tools for HTML generation and output.", - "License": "GPL (>= 2)", - "URL": "https://github.com/rstudio/htmltools, https://rstudio.github.io/htmltools/", - "BugReports": "https://github.com/rstudio/htmltools/issues", - "Depends": [ - "R (>= 2.14.1)" - ], - "Imports": [ - "base64enc", - "digest", - "fastmap (>= 1.1.0)", - "grDevices", - "rlang (>= 1.0.0)", - "utils" - ], - "Suggests": [ - "Cairo", - "markdown", - "ragg", - "shiny", - "testthat", - "withr" - ], - "Enhances": [ - "knitr" - ], - "Config/Needs/check": "knitr", - "Config/Needs/website": "rstudio/quillt, bench", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.1", - "Collate": "'colors.R' 'fill.R' 'html_dependency.R' 'html_escape.R' 'html_print.R' 'htmltools-package.R' 'images.R' 'known_tags.R' 'selector.R' 'staticimports.R' 'tag_query.R' 'utils.R' 'tags.R' 'template.R'", - "NeedsCompilation": "yes", - "Author": "Joe Cheng [aut], Carson Sievert [aut, cre] (), Barret Schloerke [aut] (), Winston Chang [aut] (), Yihui Xie [aut], Jeff Allen [aut], Posit Software, PBC [cph, fnd]", - "Maintainer": "Carson Sievert ", - "Repository": "RSPM" - }, - "htmlwidgets": { - "Package": "htmlwidgets", - "Version": "1.6.4", - "Source": "Repository", - "Type": "Package", - "Title": "HTML Widgets for R", - "Authors@R": "c( person(\"Ramnath\", \"Vaidyanathan\", role = c(\"aut\", \"cph\")), person(\"Yihui\", \"Xie\", role = \"aut\"), person(\"JJ\", \"Allaire\", role = \"aut\"), person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Carson\", \"Sievert\", , \"carson@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Kenton\", \"Russell\", role = c(\"aut\", \"cph\")), person(\"Ellis\", \"Hughes\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "A framework for creating HTML widgets that render in various contexts including the R console, 'R Markdown' documents, and 'Shiny' web applications.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/ramnathv/htmlwidgets", - "BugReports": "https://github.com/ramnathv/htmlwidgets/issues", - "Imports": [ - "grDevices", - "htmltools (>= 0.5.7)", - "jsonlite (>= 0.9.16)", - "knitr (>= 1.8)", - "rmarkdown", - "yaml" - ], - "Suggests": [ - "testthat" - ], - "Enhances": [ - "shiny (>= 1.1)" - ], - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.3", - "NeedsCompilation": "no", - "Author": "Ramnath Vaidyanathan [aut, cph], Yihui Xie [aut], JJ Allaire [aut], Joe Cheng [aut], Carson Sievert [aut, cre] (), Kenton Russell [aut, cph], Ellis Hughes [ctb], Posit Software, PBC [cph, fnd]", - "Maintainer": "Carson Sievert ", - "Repository": "RSPM" - }, - "httpuv": { - "Package": "httpuv", - "Version": "1.6.16", - "Source": "Repository", - "Type": "Package", - "Title": "HTTP and WebSocket Server Library", - "Authors@R": "c( person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit, PBC\", \"fnd\", role = \"cph\"), person(\"Hector\", \"Corrada Bravo\", role = \"ctb\"), person(\"Jeroen\", \"Ooms\", role = \"ctb\"), person(\"Andrzej\", \"Krzemienski\", role = \"cph\", comment = \"optional.hpp\"), person(\"libuv project contributors\", role = \"cph\", comment = \"libuv library, see src/libuv/AUTHORS file\"), person(\"Joyent, Inc. and other Node contributors\", role = \"cph\", comment = \"libuv library, see src/libuv/AUTHORS file; and http-parser library, see src/http-parser/AUTHORS file\"), person(\"Niels\", \"Provos\", role = \"cph\", comment = \"libuv subcomponent: tree.h\"), person(\"Internet Systems Consortium, Inc.\", role = \"cph\", comment = \"libuv subcomponent: inet_pton and inet_ntop, contained in src/libuv/src/inet.c\"), person(\"Alexander\", \"Chemeris\", role = \"cph\", comment = \"libuv subcomponent: stdint-msvc2008.h (from msinttypes)\"), person(\"Google, Inc.\", role = \"cph\", comment = \"libuv subcomponent: pthread-fixes.c\"), person(\"Sony Mobile Communcations AB\", role = \"cph\", comment = \"libuv subcomponent: pthread-fixes.c\"), person(\"Berkeley Software Design Inc.\", role = \"cph\", comment = \"libuv subcomponent: android-ifaddrs.h, android-ifaddrs.c\"), person(\"Kenneth\", \"MacKay\", role = \"cph\", comment = \"libuv subcomponent: android-ifaddrs.h, android-ifaddrs.c\"), person(\"Emergya (Cloud4all, FP7/2007-2013, grant agreement no 289016)\", role = \"cph\", comment = \"libuv subcomponent: android-ifaddrs.h, android-ifaddrs.c\"), person(\"Steve\", \"Reid\", role = \"aut\", comment = \"SHA-1 implementation\"), person(\"James\", \"Brown\", role = \"aut\", comment = \"SHA-1 implementation\"), person(\"Bob\", \"Trower\", role = \"aut\", comment = \"base64 implementation\"), person(\"Alexander\", \"Peslyak\", role = \"aut\", comment = \"MD5 implementation\"), person(\"Trantor Standard Systems\", role = \"cph\", comment = \"base64 implementation\"), person(\"Igor\", \"Sysoev\", role = \"cph\", comment = \"http-parser\") )", - "Description": "Provides low-level socket and protocol support for handling HTTP and WebSocket requests directly from within R. It is primarily intended as a building block for other packages, rather than making it particularly easy to create complete web applications using httpuv alone. httpuv is built on top of the libuv and http-parser C libraries, both of which were developed by Joyent, Inc. (See LICENSE file for libuv and http-parser license information.)", - "License": "GPL (>= 2) | file LICENSE", - "URL": "https://github.com/rstudio/httpuv", - "BugReports": "https://github.com/rstudio/httpuv/issues", - "Depends": [ - "R (>= 2.15.1)" - ], - "Imports": [ - "later (>= 0.8.0)", - "promises", - "R6", - "Rcpp (>= 1.0.7)", - "utils" - ], - "Suggests": [ - "callr", - "curl", - "jsonlite", - "testthat", - "websocket" - ], - "LinkingTo": [ - "later", - "Rcpp" - ], - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "SystemRequirements": "GNU make, zlib", - "Collate": "'RcppExports.R' 'httpuv.R' 'random_port.R' 'server.R' 'staticServer.R' 'static_paths.R' 'utils.R'", - "NeedsCompilation": "yes", - "Author": "Joe Cheng [aut], Winston Chang [aut, cre], Posit, PBC fnd [cph], Hector Corrada Bravo [ctb], Jeroen Ooms [ctb], Andrzej Krzemienski [cph] (optional.hpp), libuv project contributors [cph] (libuv library, see src/libuv/AUTHORS file), Joyent, Inc. and other Node contributors [cph] (libuv library, see src/libuv/AUTHORS file; and http-parser library, see src/http-parser/AUTHORS file), Niels Provos [cph] (libuv subcomponent: tree.h), Internet Systems Consortium, Inc. [cph] (libuv subcomponent: inet_pton and inet_ntop, contained in src/libuv/src/inet.c), Alexander Chemeris [cph] (libuv subcomponent: stdint-msvc2008.h (from msinttypes)), Google, Inc. [cph] (libuv subcomponent: pthread-fixes.c), Sony Mobile Communcations AB [cph] (libuv subcomponent: pthread-fixes.c), Berkeley Software Design Inc. [cph] (libuv subcomponent: android-ifaddrs.h, android-ifaddrs.c), Kenneth MacKay [cph] (libuv subcomponent: android-ifaddrs.h, android-ifaddrs.c), Emergya (Cloud4all, FP7/2007-2013, grant agreement no 289016) [cph] (libuv subcomponent: android-ifaddrs.h, android-ifaddrs.c), Steve Reid [aut] (SHA-1 implementation), James Brown [aut] (SHA-1 implementation), Bob Trower [aut] (base64 implementation), Alexander Peslyak [aut] (MD5 implementation), Trantor Standard Systems [cph] (base64 implementation), Igor Sysoev [cph] (http-parser)", - "Maintainer": "Winston Chang ", - "Repository": "CRAN" - }, - "httr": { - "Package": "httr", - "Version": "1.4.7", - "Source": "Repository", - "Title": "Tools for Working with URLs and HTTP", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Useful tools for working with HTTP organised by HTTP verbs (GET(), POST(), etc). Configuration functions make it easy to control additional request components (authenticate(), add_headers() and so on).", - "License": "MIT + file LICENSE", - "URL": "https://httr.r-lib.org/, https://github.com/r-lib/httr", - "BugReports": "https://github.com/r-lib/httr/issues", - "Depends": [ - "R (>= 3.5)" - ], - "Imports": [ - "curl (>= 5.0.2)", - "jsonlite", - "mime", - "openssl (>= 0.8)", - "R6" - ], - "Suggests": [ - "covr", - "httpuv", - "jpeg", - "knitr", - "png", - "readr", - "rmarkdown", - "testthat (>= 0.8.0)", - "xml2" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.3", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre], Posit, PBC [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "RSPM" - }, - "httr2": { - "Package": "httr2", - "Version": "1.2.1", - "Source": "Repository", - "Title": "Perform HTTP Requests and Process the Responses", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(\"Maximilian\", \"Girlich\", role = \"ctb\") )", - "Description": "Tools for creating and modifying HTTP requests, then performing them and processing the results. 'httr2' is a modern re-imagining of 'httr' that uses a pipe-based interface and solves more of the problems that API wrapping packages face.", - "License": "MIT + file LICENSE", - "URL": "https://httr2.r-lib.org, https://github.com/r-lib/httr2", - "BugReports": "https://github.com/r-lib/httr2/issues", - "Depends": [ - "R (>= 4.1)" - ], - "Imports": [ - "cli (>= 3.0.0)", - "curl (>= 6.4.0)", - "glue", - "lifecycle", - "magrittr", - "openssl", - "R6", - "rappdirs", - "rlang (>= 1.1.0)", - "vctrs (>= 0.6.3)", - "withr" - ], - "Suggests": [ - "askpass", - "bench", - "clipr", - "covr", - "docopt", - "httpuv", - "jose", - "jsonlite", - "knitr", - "later (>= 1.4.0)", - "nanonext", - "paws.common", - "promises", - "rmarkdown", - "testthat (>= 3.1.8)", - "tibble", - "webfakes (>= 1.4.0)", - "xml2" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "true", - "Config/testthat/start-first": "resp-stream, req-perform", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre], Posit Software, PBC [cph, fnd], Maximilian Girlich [ctb]", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "ids": { - "Package": "ids", - "Version": "1.0.1", - "Source": "Repository", - "Title": "Generate Random Identifiers", - "Authors@R": "person(\"Rich\", \"FitzJohn\", role = c(\"aut\", \"cre\"), email = \"rich.fitzjohn@gmail.com\")", - "Description": "Generate random or human readable and pronounceable identifiers.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/richfitz/ids", - "BugReports": "https://github.com/richfitz/ids/issues", - "Imports": [ - "openssl", - "uuid" - ], - "Suggests": [ - "knitr", - "rcorpora", - "rmarkdown", - "testthat" - ], - "RoxygenNote": "6.0.1", - "VignetteBuilder": "knitr", - "NeedsCompilation": "no", - "Author": "Rich FitzJohn [aut, cre]", - "Maintainer": "Rich FitzJohn ", - "Repository": "CRAN" - }, - "ini": { - "Package": "ini", - "Version": "0.3.1", - "Source": "Repository", - "Type": "Package", - "Title": "Read and Write '.ini' Files", - "Date": "2018-05-19", - "Author": "David Valentim Dias", - "Maintainer": "David Valentim Dias ", - "Description": "Parse simple '.ini' configuration files to an structured list. Users can manipulate this resulting list with lapply() functions. This same structured list can be used to write back to file after modifications.", - "License": "GPL-3", - "URL": "https://github.com/dvdscripter/ini", - "BugReports": "https://github.com/dvdscripter/ini/issues", - "LazyData": "FALSE", - "RoxygenNote": "6.0.1", - "Suggests": [ - "testthat" - ], - "NeedsCompilation": "no", - "Repository": "CRAN" - }, - "isoband": { - "Package": "isoband", - "Version": "0.2.7", - "Source": "Repository", - "Title": "Generate Isolines and Isobands from Regularly Spaced Elevation Grids", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Claus O.\", \"Wilke\", , \"wilke@austin.utexas.edu\", role = \"aut\", comment = c(\"Original author\", ORCID = \"0000-0002-7470-9261\")), person(\"Thomas Lin\", \"Pedersen\", , \"thomasp85@gmail.com\", role = \"aut\", comment = c(ORCID = \"0000-0002-5147-4711\")) )", - "Description": "A fast C++ implementation to generate contour lines (isolines) and contour polygons (isobands) from regularly spaced grids containing elevation data.", - "License": "MIT + file LICENSE", - "URL": "https://isoband.r-lib.org", - "BugReports": "https://github.com/r-lib/isoband/issues", - "Imports": [ - "grid", - "utils" - ], - "Suggests": [ - "covr", - "ggplot2", - "knitr", - "magick", - "microbenchmark", - "rmarkdown", - "sf", - "testthat", - "xml2" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.3", - "SystemRequirements": "C++11", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut, cre] (), Claus O. Wilke [aut] (Original author, ), Thomas Lin Pedersen [aut] ()", - "Maintainer": "Hadley Wickham ", - "Repository": "RSPM" - }, - "jquerylib": { - "Package": "jquerylib", - "Version": "0.1.4", - "Source": "Repository", - "Title": "Obtain 'jQuery' as an HTML Dependency Object", - "Authors@R": "c( person(\"Carson\", \"Sievert\", role = c(\"aut\", \"cre\"), email = \"carson@rstudio.com\", comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Joe\", \"Cheng\", role = \"aut\", email = \"joe@rstudio.com\"), person(family = \"RStudio\", role = \"cph\"), person(family = \"jQuery Foundation\", role = \"cph\", comment = \"jQuery library and jQuery UI library\"), person(family = \"jQuery contributors\", role = c(\"ctb\", \"cph\"), comment = \"jQuery library; authors listed in inst/lib/jquery-AUTHORS.txt\") )", - "Description": "Obtain any major version of 'jQuery' () and use it in any webpage generated by 'htmltools' (e.g. 'shiny', 'htmlwidgets', and 'rmarkdown'). Most R users don't need to use this package directly, but other R packages (e.g. 'shiny', 'rmarkdown', etc.) depend on this package to avoid bundling redundant copies of 'jQuery'.", - "License": "MIT + file LICENSE", - "Encoding": "UTF-8", - "Config/testthat/edition": "3", - "RoxygenNote": "7.0.2", - "Imports": [ - "htmltools" - ], - "Suggests": [ - "testthat" - ], - "NeedsCompilation": "no", - "Author": "Carson Sievert [aut, cre] (), Joe Cheng [aut], RStudio [cph], jQuery Foundation [cph] (jQuery library and jQuery UI library), jQuery contributors [ctb, cph] (jQuery library; authors listed in inst/lib/jquery-AUTHORS.txt)", - "Maintainer": "Carson Sievert ", - "Repository": "RSPM" - }, - "jsonlite": { - "Package": "jsonlite", - "Version": "2.0.0", - "Source": "Repository", - "Title": "A Simple and Robust JSON Parser and Generator for R", - "License": "MIT + file LICENSE", - "Depends": [ - "methods" - ], - "Authors@R": "c( person(\"Jeroen\", \"Ooms\", role = c(\"aut\", \"cre\"), email = \"jeroenooms@gmail.com\", comment = c(ORCID = \"0000-0002-4035-0289\")), person(\"Duncan\", \"Temple Lang\", role = \"ctb\"), person(\"Lloyd\", \"Hilaiel\", role = \"cph\", comment=\"author of bundled libyajl\"))", - "URL": "https://jeroen.r-universe.dev/jsonlite https://arxiv.org/abs/1403.2805", - "BugReports": "https://github.com/jeroen/jsonlite/issues", - "Maintainer": "Jeroen Ooms ", - "VignetteBuilder": "knitr, R.rsp", - "Description": "A reasonably fast JSON parser and generator, optimized for statistical data and the web. Offers simple, flexible tools for working with JSON in R, and is particularly powerful for building pipelines and interacting with a web API. The implementation is based on the mapping described in the vignette (Ooms, 2014). In addition to converting JSON data from/to R objects, 'jsonlite' contains functions to stream, validate, and prettify JSON data. The unit tests included with the package verify that all edge cases are encoded and decoded consistently for use with dynamic data in systems and applications.", - "Suggests": [ - "httr", - "vctrs", - "testthat", - "knitr", - "rmarkdown", - "R.rsp", - "sf" - ], - "RoxygenNote": "7.3.2", - "Encoding": "UTF-8", - "NeedsCompilation": "yes", - "Author": "Jeroen Ooms [aut, cre] (), Duncan Temple Lang [ctb], Lloyd Hilaiel [cph] (author of bundled libyajl)", - "Repository": "CRAN" - }, - "knitr": { - "Package": "knitr", - "Version": "1.50", - "Source": "Repository", - "Type": "Package", - "Title": "A General-Purpose Package for Dynamic Report Generation in R", - "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\", URL = \"https://yihui.org\")), person(\"Abhraneel\", \"Sarma\", role = \"ctb\"), person(\"Adam\", \"Vogt\", role = \"ctb\"), person(\"Alastair\", \"Andrew\", role = \"ctb\"), person(\"Alex\", \"Zvoleff\", role = \"ctb\"), person(\"Amar\", \"Al-Zubaidi\", role = \"ctb\"), person(\"Andre\", \"Simon\", role = \"ctb\", comment = \"the CSS files under inst/themes/ were derived from the Highlight package http://www.andre-simon.de\"), person(\"Aron\", \"Atkins\", role = \"ctb\"), person(\"Aaron\", \"Wolen\", role = \"ctb\"), person(\"Ashley\", \"Manton\", role = \"ctb\"), person(\"Atsushi\", \"Yasumoto\", role = \"ctb\", comment = c(ORCID = \"0000-0002-8335-495X\")), person(\"Ben\", \"Baumer\", role = \"ctb\"), person(\"Brian\", \"Diggs\", role = \"ctb\"), person(\"Brian\", \"Zhang\", role = \"ctb\"), person(\"Bulat\", \"Yapparov\", role = \"ctb\"), person(\"Cassio\", \"Pereira\", role = \"ctb\"), person(\"Christophe\", \"Dervieux\", role = \"ctb\"), person(\"David\", \"Hall\", role = \"ctb\"), person(\"David\", \"Hugh-Jones\", role = \"ctb\"), person(\"David\", \"Robinson\", role = \"ctb\"), person(\"Doug\", \"Hemken\", role = \"ctb\"), person(\"Duncan\", \"Murdoch\", role = \"ctb\"), person(\"Elio\", \"Campitelli\", role = \"ctb\"), person(\"Ellis\", \"Hughes\", role = \"ctb\"), person(\"Emily\", \"Riederer\", role = \"ctb\"), person(\"Fabian\", \"Hirschmann\", role = \"ctb\"), person(\"Fitch\", \"Simeon\", role = \"ctb\"), person(\"Forest\", \"Fang\", role = \"ctb\"), person(c(\"Frank\", \"E\", \"Harrell\", \"Jr\"), role = \"ctb\", comment = \"the Sweavel package at inst/misc/Sweavel.sty\"), person(\"Garrick\", \"Aden-Buie\", role = \"ctb\"), person(\"Gregoire\", \"Detrez\", role = \"ctb\"), person(\"Hadley\", \"Wickham\", role = \"ctb\"), person(\"Hao\", \"Zhu\", role = \"ctb\"), person(\"Heewon\", \"Jeon\", role = \"ctb\"), person(\"Henrik\", \"Bengtsson\", role = \"ctb\"), person(\"Hiroaki\", \"Yutani\", role = \"ctb\"), person(\"Ian\", \"Lyttle\", role = \"ctb\"), person(\"Hodges\", \"Daniel\", role = \"ctb\"), person(\"Jacob\", \"Bien\", role = \"ctb\"), person(\"Jake\", \"Burkhead\", role = \"ctb\"), person(\"James\", \"Manton\", role = \"ctb\"), person(\"Jared\", \"Lander\", role = \"ctb\"), person(\"Jason\", \"Punyon\", role = \"ctb\"), person(\"Javier\", \"Luraschi\", role = \"ctb\"), person(\"Jeff\", \"Arnold\", role = \"ctb\"), person(\"Jenny\", \"Bryan\", role = \"ctb\"), person(\"Jeremy\", \"Ashkenas\", role = c(\"ctb\", \"cph\"), comment = \"the CSS file at inst/misc/docco-classic.css\"), person(\"Jeremy\", \"Stephens\", role = \"ctb\"), person(\"Jim\", \"Hester\", role = \"ctb\"), person(\"Joe\", \"Cheng\", role = \"ctb\"), person(\"Johannes\", \"Ranke\", role = \"ctb\"), person(\"John\", \"Honaker\", role = \"ctb\"), person(\"John\", \"Muschelli\", role = \"ctb\"), person(\"Jonathan\", \"Keane\", role = \"ctb\"), person(\"JJ\", \"Allaire\", role = \"ctb\"), person(\"Johan\", \"Toloe\", role = \"ctb\"), person(\"Jonathan\", \"Sidi\", role = \"ctb\"), person(\"Joseph\", \"Larmarange\", role = \"ctb\"), person(\"Julien\", \"Barnier\", role = \"ctb\"), person(\"Kaiyin\", \"Zhong\", role = \"ctb\"), person(\"Kamil\", \"Slowikowski\", role = \"ctb\"), person(\"Karl\", \"Forner\", role = \"ctb\"), person(c(\"Kevin\", \"K.\"), \"Smith\", role = \"ctb\"), person(\"Kirill\", \"Mueller\", role = \"ctb\"), person(\"Kohske\", \"Takahashi\", role = \"ctb\"), person(\"Lorenz\", \"Walthert\", role = \"ctb\"), person(\"Lucas\", \"Gallindo\", role = \"ctb\"), person(\"Marius\", \"Hofert\", role = \"ctb\"), person(\"Martin\", \"Modrák\", role = \"ctb\"), person(\"Michael\", \"Chirico\", role = \"ctb\"), person(\"Michael\", \"Friendly\", role = \"ctb\"), person(\"Michal\", \"Bojanowski\", role = \"ctb\"), person(\"Michel\", \"Kuhlmann\", role = \"ctb\"), person(\"Miller\", \"Patrick\", role = \"ctb\"), person(\"Nacho\", \"Caballero\", role = \"ctb\"), person(\"Nick\", \"Salkowski\", role = \"ctb\"), person(\"Niels Richard\", \"Hansen\", role = \"ctb\"), person(\"Noam\", \"Ross\", role = \"ctb\"), person(\"Obada\", \"Mahdi\", role = \"ctb\"), person(\"Pavel N.\", \"Krivitsky\", role = \"ctb\", comment=c(ORCID = \"0000-0002-9101-3362\")), person(\"Pedro\", \"Faria\", role = \"ctb\"), person(\"Qiang\", \"Li\", role = \"ctb\"), person(\"Ramnath\", \"Vaidyanathan\", role = \"ctb\"), person(\"Richard\", \"Cotton\", role = \"ctb\"), person(\"Robert\", \"Krzyzanowski\", role = \"ctb\"), person(\"Rodrigo\", \"Copetti\", role = \"ctb\"), person(\"Romain\", \"Francois\", role = \"ctb\"), person(\"Ruaridh\", \"Williamson\", role = \"ctb\"), person(\"Sagiru\", \"Mati\", role = \"ctb\", comment = c(ORCID = \"0000-0003-1413-3974\")), person(\"Scott\", \"Kostyshak\", role = \"ctb\"), person(\"Sebastian\", \"Meyer\", role = \"ctb\"), person(\"Sietse\", \"Brouwer\", role = \"ctb\"), person(c(\"Simon\", \"de\"), \"Bernard\", role = \"ctb\"), person(\"Sylvain\", \"Rousseau\", role = \"ctb\"), person(\"Taiyun\", \"Wei\", role = \"ctb\"), person(\"Thibaut\", \"Assus\", role = \"ctb\"), person(\"Thibaut\", \"Lamadon\", role = \"ctb\"), person(\"Thomas\", \"Leeper\", role = \"ctb\"), person(\"Tim\", \"Mastny\", role = \"ctb\"), person(\"Tom\", \"Torsney-Weir\", role = \"ctb\"), person(\"Trevor\", \"Davis\", role = \"ctb\"), person(\"Viktoras\", \"Veitas\", role = \"ctb\"), person(\"Weicheng\", \"Zhu\", role = \"ctb\"), person(\"Wush\", \"Wu\", role = \"ctb\"), person(\"Zachary\", \"Foster\", role = \"ctb\"), person(\"Zhian N.\", \"Kamvar\", role = \"ctb\", comment = c(ORCID = \"0000-0003-1458-7108\")), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Provides a general-purpose tool for dynamic report generation in R using Literate Programming techniques.", - "Depends": [ - "R (>= 3.6.0)" - ], - "Imports": [ - "evaluate (>= 0.15)", - "highr (>= 0.11)", - "methods", - "tools", - "xfun (>= 0.51)", - "yaml (>= 2.1.19)" - ], - "Suggests": [ - "bslib", - "codetools", - "DBI (>= 0.4-1)", - "digest", - "formatR", - "gifski", - "gridSVG", - "htmlwidgets (>= 0.7)", - "jpeg", - "JuliaCall (>= 0.11.1)", - "magick", - "litedown", - "markdown (>= 1.3)", - "png", - "ragg", - "reticulate (>= 1.4)", - "rgl (>= 0.95.1201)", - "rlang", - "rmarkdown", - "sass", - "showtext", - "styler (>= 1.2.0)", - "targets (>= 0.6.0)", - "testit", - "tibble", - "tikzDevice (>= 0.10)", - "tinytex (>= 0.56)", - "webshot", - "rstudioapi", - "svglite" - ], - "License": "GPL", - "URL": "https://yihui.org/knitr/", - "BugReports": "https://github.com/yihui/knitr/issues", - "Encoding": "UTF-8", - "VignetteBuilder": "litedown, knitr", - "SystemRequirements": "Package vignettes based on R Markdown v2 or reStructuredText require Pandoc (http://pandoc.org). The function rst2pdf() requires rst2pdf (https://github.com/rst2pdf/rst2pdf).", - "Collate": "'block.R' 'cache.R' 'citation.R' 'hooks-html.R' 'plot.R' 'utils.R' 'defaults.R' 'concordance.R' 'engine.R' 'highlight.R' 'themes.R' 'header.R' 'hooks-asciidoc.R' 'hooks-chunk.R' 'hooks-extra.R' 'hooks-latex.R' 'hooks-md.R' 'hooks-rst.R' 'hooks-textile.R' 'hooks.R' 'output.R' 'package.R' 'pandoc.R' 'params.R' 'parser.R' 'pattern.R' 'rocco.R' 'spin.R' 'table.R' 'template.R' 'utils-conversion.R' 'utils-rd2html.R' 'utils-string.R' 'utils-sweave.R' 'utils-upload.R' 'utils-vignettes.R' 'zzz.R'", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Yihui Xie [aut, cre] (, https://yihui.org), Abhraneel Sarma [ctb], Adam Vogt [ctb], Alastair Andrew [ctb], Alex Zvoleff [ctb], Amar Al-Zubaidi [ctb], Andre Simon [ctb] (the CSS files under inst/themes/ were derived from the Highlight package http://www.andre-simon.de), Aron Atkins [ctb], Aaron Wolen [ctb], Ashley Manton [ctb], Atsushi Yasumoto [ctb] (), Ben Baumer [ctb], Brian Diggs [ctb], Brian Zhang [ctb], Bulat Yapparov [ctb], Cassio Pereira [ctb], Christophe Dervieux [ctb], David Hall [ctb], David Hugh-Jones [ctb], David Robinson [ctb], Doug Hemken [ctb], Duncan Murdoch [ctb], Elio Campitelli [ctb], Ellis Hughes [ctb], Emily Riederer [ctb], Fabian Hirschmann [ctb], Fitch Simeon [ctb], Forest Fang [ctb], Frank E Harrell Jr [ctb] (the Sweavel package at inst/misc/Sweavel.sty), Garrick Aden-Buie [ctb], Gregoire Detrez [ctb], Hadley Wickham [ctb], Hao Zhu [ctb], Heewon Jeon [ctb], Henrik Bengtsson [ctb], Hiroaki Yutani [ctb], Ian Lyttle [ctb], Hodges Daniel [ctb], Jacob Bien [ctb], Jake Burkhead [ctb], James Manton [ctb], Jared Lander [ctb], Jason Punyon [ctb], Javier Luraschi [ctb], Jeff Arnold [ctb], Jenny Bryan [ctb], Jeremy Ashkenas [ctb, cph] (the CSS file at inst/misc/docco-classic.css), Jeremy Stephens [ctb], Jim Hester [ctb], Joe Cheng [ctb], Johannes Ranke [ctb], John Honaker [ctb], John Muschelli [ctb], Jonathan Keane [ctb], JJ Allaire [ctb], Johan Toloe [ctb], Jonathan Sidi [ctb], Joseph Larmarange [ctb], Julien Barnier [ctb], Kaiyin Zhong [ctb], Kamil Slowikowski [ctb], Karl Forner [ctb], Kevin K. Smith [ctb], Kirill Mueller [ctb], Kohske Takahashi [ctb], Lorenz Walthert [ctb], Lucas Gallindo [ctb], Marius Hofert [ctb], Martin Modrák [ctb], Michael Chirico [ctb], Michael Friendly [ctb], Michal Bojanowski [ctb], Michel Kuhlmann [ctb], Miller Patrick [ctb], Nacho Caballero [ctb], Nick Salkowski [ctb], Niels Richard Hansen [ctb], Noam Ross [ctb], Obada Mahdi [ctb], Pavel N. Krivitsky [ctb] (), Pedro Faria [ctb], Qiang Li [ctb], Ramnath Vaidyanathan [ctb], Richard Cotton [ctb], Robert Krzyzanowski [ctb], Rodrigo Copetti [ctb], Romain Francois [ctb], Ruaridh Williamson [ctb], Sagiru Mati [ctb] (), Scott Kostyshak [ctb], Sebastian Meyer [ctb], Sietse Brouwer [ctb], Simon de Bernard [ctb], Sylvain Rousseau [ctb], Taiyun Wei [ctb], Thibaut Assus [ctb], Thibaut Lamadon [ctb], Thomas Leeper [ctb], Tim Mastny [ctb], Tom Torsney-Weir [ctb], Trevor Davis [ctb], Viktoras Veitas [ctb], Weicheng Zhu [ctb], Wush Wu [ctb], Zachary Foster [ctb], Zhian N. Kamvar [ctb] (), Posit Software, PBC [cph, fnd]", - "Maintainer": "Yihui Xie ", - "Repository": "CRAN" - }, - "labeling": { - "Package": "labeling", - "Version": "0.4.3", - "Source": "Repository", - "Type": "Package", - "Title": "Axis Labeling", - "Date": "2023-08-29", - "Author": "Justin Talbot,", - "Maintainer": "Nuno Sempere ", - "Description": "Functions which provide a range of axis labeling algorithms.", - "License": "MIT + file LICENSE | Unlimited", - "Collate": "'labeling.R'", - "NeedsCompilation": "no", - "Imports": [ - "stats", - "graphics" - ], - "Repository": "CRAN" - }, - "later": { - "Package": "later", - "Version": "1.4.4", - "Source": "Repository", - "Type": "Package", - "Title": "Utilities for Scheduling Functions to Execute Later with Event Loops", - "Authors@R": "c( person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = \"aut\"), person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Charlie\", \"Gao\", , \"charlie.gao@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-0750-061X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")), person(\"Marcus\", \"Geelnard\", role = c(\"ctb\", \"cph\"), comment = \"TinyCThread library, https://tinycthread.github.io/\"), person(\"Evan\", \"Nemerson\", role = c(\"ctb\", \"cph\"), comment = \"TinyCThread library, https://tinycthread.github.io/\") )", - "Description": "Executes arbitrary R or C functions some time after the current time, after the R execution stack has emptied. The functions are scheduled in an event loop.", - "License": "MIT + file LICENSE", - "URL": "https://later.r-lib.org, https://github.com/r-lib/later", - "BugReports": "https://github.com/r-lib/later/issues", - "Imports": [ - "Rcpp (>= 0.12.9)", - "rlang" - ], - "Suggests": [ - "knitr", - "nanonext", - "rmarkdown", - "testthat (>= 3.0.0)" - ], - "LinkingTo": [ - "Rcpp" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/usethis/last-upkeep": "2025-07-18", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "yes", - "Author": "Winston Chang [aut], Joe Cheng [aut], Charlie Gao [aut, cre] (ORCID: ), Posit Software, PBC [cph, fnd] (ROR: ), Marcus Geelnard [ctb, cph] (TinyCThread library, https://tinycthread.github.io/), Evan Nemerson [ctb, cph] (TinyCThread library, https://tinycthread.github.io/)", - "Maintainer": "Charlie Gao ", - "Repository": "CRAN" - }, - "lattice": { - "Package": "lattice", - "Version": "0.22-7", - "Source": "Repository", - "Date": "2025-03-31", - "Priority": "recommended", - "Title": "Trellis Graphics for R", - "Authors@R": "c(person(\"Deepayan\", \"Sarkar\", role = c(\"aut\", \"cre\"), email = \"deepayan.sarkar@r-project.org\", comment = c(ORCID = \"0000-0003-4107-1553\")), person(\"Felix\", \"Andrews\", role = \"ctb\"), person(\"Kevin\", \"Wright\", role = \"ctb\", comment = \"documentation\"), person(\"Neil\", \"Klepeis\", role = \"ctb\"), person(\"Johan\", \"Larsson\", role = \"ctb\", comment = \"miscellaneous improvements\"), person(\"Zhijian (Jason)\", \"Wen\", role = \"cph\", comment = \"filled contour code\"), person(\"Paul\", \"Murrell\", role = \"ctb\", email = \"paul@stat.auckland.ac.nz\"), person(\"Stefan\", \"Eng\", role = \"ctb\", comment = \"violin plot improvements\"), person(\"Achim\", \"Zeileis\", role = \"ctb\", comment = \"modern colors\"), person(\"Alexandre\", \"Courtiol\", role = \"ctb\", comment = \"generics for larrows, lpolygon, lrect and lsegments\") )", - "Description": "A powerful and elegant high-level data visualization system inspired by Trellis graphics, with an emphasis on multivariate data. Lattice is sufficient for typical graphics needs, and is also flexible enough to handle most nonstandard requirements. See ?Lattice for an introduction.", - "Depends": [ - "R (>= 4.0.0)" - ], - "Suggests": [ - "KernSmooth", - "MASS", - "latticeExtra", - "colorspace" - ], - "Imports": [ - "grid", - "grDevices", - "graphics", - "stats", - "utils" - ], - "Enhances": [ - "chron", - "zoo" - ], - "LazyLoad": "yes", - "LazyData": "yes", - "License": "GPL (>= 2)", - "URL": "https://lattice.r-forge.r-project.org/", - "BugReports": "https://github.com/deepayan/lattice/issues", - "NeedsCompilation": "yes", - "Author": "Deepayan Sarkar [aut, cre] (), Felix Andrews [ctb], Kevin Wright [ctb] (documentation), Neil Klepeis [ctb], Johan Larsson [ctb] (miscellaneous improvements), Zhijian (Jason) Wen [cph] (filled contour code), Paul Murrell [ctb], Stefan Eng [ctb] (violin plot improvements), Achim Zeileis [ctb] (modern colors), Alexandre Courtiol [ctb] (generics for larrows, lpolygon, lrect and lsegments)", - "Maintainer": "Deepayan Sarkar ", - "Repository": "CRAN" - }, - "lazyeval": { - "Package": "lazyeval", - "Version": "0.2.2", - "Source": "Repository", - "Title": "Lazy (Non-Standard) Evaluation", - "Description": "An alternative approach to non-standard evaluation using formulas. Provides a full implementation of LISP style 'quasiquotation', making it easier to generate code with other code.", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", ,\"hadley@rstudio.com\", c(\"aut\", \"cre\")), person(\"RStudio\", role = \"cph\") )", - "License": "GPL-3", - "LazyData": "true", - "Depends": [ - "R (>= 3.1.0)" - ], - "Suggests": [ - "knitr", - "rmarkdown (>= 0.2.65)", - "testthat", - "covr" - ], - "VignetteBuilder": "knitr", - "RoxygenNote": "6.1.1", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut, cre], RStudio [cph]", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "lifecycle": { - "Package": "lifecycle", - "Version": "1.0.4", - "Source": "Repository", - "Title": "Manage the Life Cycle of your Package Functions", - "Authors@R": "c( person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Manage the life cycle of your exported functions with shared conventions, documentation badges, and user-friendly deprecation warnings.", - "License": "MIT + file LICENSE", - "URL": "https://lifecycle.r-lib.org/, https://github.com/r-lib/lifecycle", - "BugReports": "https://github.com/r-lib/lifecycle/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "cli (>= 3.4.0)", - "glue", - "rlang (>= 1.1.0)" - ], - "Suggests": [ - "covr", - "crayon", - "knitr", - "lintr", - "rmarkdown", - "testthat (>= 3.0.1)", - "tibble", - "tidyverse", - "tools", - "vctrs", - "withr" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate, usethis", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.1", - "NeedsCompilation": "no", - "Author": "Lionel Henry [aut, cre], Hadley Wickham [aut] (), Posit Software, PBC [cph, fnd]", - "Maintainer": "Lionel Henry ", - "Repository": "RSPM" - }, - "lubridate": { - "Package": "lubridate", - "Version": "1.9.4", - "Source": "Repository", - "Type": "Package", - "Title": "Make Dealing with Dates a Little Easier", - "Authors@R": "c( person(\"Vitalie\", \"Spinu\", , \"spinuvit@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Garrett\", \"Grolemund\", role = \"aut\"), person(\"Hadley\", \"Wickham\", role = \"aut\"), person(\"Davis\", \"Vaughan\", role = \"ctb\"), person(\"Ian\", \"Lyttle\", role = \"ctb\"), person(\"Imanuel\", \"Costigan\", role = \"ctb\"), person(\"Jason\", \"Law\", role = \"ctb\"), person(\"Doug\", \"Mitarotonda\", role = \"ctb\"), person(\"Joseph\", \"Larmarange\", role = \"ctb\"), person(\"Jonathan\", \"Boiser\", role = \"ctb\"), person(\"Chel Hee\", \"Lee\", role = \"ctb\") )", - "Maintainer": "Vitalie Spinu ", - "Description": "Functions to work with date-times and time-spans: fast and user friendly parsing of date-time data, extraction and updating of components of a date-time (years, months, days, hours, minutes, and seconds), algebraic manipulation on date-time and time-span objects. The 'lubridate' package has a consistent and memorable syntax that makes working with dates easy and fun.", - "License": "GPL (>= 2)", - "URL": "https://lubridate.tidyverse.org, https://github.com/tidyverse/lubridate", - "BugReports": "https://github.com/tidyverse/lubridate/issues", - "Depends": [ - "methods", - "R (>= 3.2)" - ], - "Imports": [ - "generics", - "timechange (>= 0.3.0)" - ], - "Suggests": [ - "covr", - "knitr", - "rmarkdown", - "testthat (>= 2.1.0)", - "vctrs (>= 0.6.5)" - ], - "Enhances": [ - "chron", - "data.table", - "timeDate", - "tis", - "zoo" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "LazyData": "true", - "RoxygenNote": "7.2.3", - "SystemRequirements": "C++11, A system with zoneinfo data (e.g. /usr/share/zoneinfo). On Windows the zoneinfo included with R is used.", - "Collate": "'Dates.r' 'POSIXt.r' 'util.r' 'parse.r' 'timespans.r' 'intervals.r' 'difftimes.r' 'durations.r' 'periods.r' 'accessors-date.R' 'accessors-day.r' 'accessors-dst.r' 'accessors-hour.r' 'accessors-minute.r' 'accessors-month.r' 'accessors-quarter.r' 'accessors-second.r' 'accessors-tz.r' 'accessors-week.r' 'accessors-year.r' 'am-pm.r' 'time-zones.r' 'numeric.r' 'coercion.r' 'constants.r' 'cyclic_encoding.r' 'data.r' 'decimal-dates.r' 'deprecated.r' 'format_ISO8601.r' 'guess.r' 'hidden.r' 'instants.r' 'leap-years.r' 'ops-addition.r' 'ops-compare.r' 'ops-division.r' 'ops-integer-division.r' 'ops-m+.r' 'ops-modulo.r' 'ops-multiplication.r' 'ops-subtraction.r' 'package.r' 'pretty.r' 'round.r' 'stamp.r' 'tzdir.R' 'update.r' 'vctrs.R' 'zzz.R'", - "NeedsCompilation": "yes", - "Author": "Vitalie Spinu [aut, cre], Garrett Grolemund [aut], Hadley Wickham [aut], Davis Vaughan [ctb], Ian Lyttle [ctb], Imanuel Costigan [ctb], Jason Law [ctb], Doug Mitarotonda [ctb], Joseph Larmarange [ctb], Jonathan Boiser [ctb], Chel Hee Lee [ctb]", - "Repository": "CRAN" - }, - "magrittr": { - "Package": "magrittr", - "Version": "2.0.4", - "Source": "Repository", - "Type": "Package", - "Title": "A Forward-Pipe Operator for R", - "Authors@R": "c( person(\"Stefan Milton\", \"Bache\", , \"stefan@stefanbache.dk\", role = c(\"aut\", \"cph\"), comment = \"Original author and creator of magrittr\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = \"cre\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Provides a mechanism for chaining commands with a new forward-pipe operator, %>%. This operator will forward a value, or the result of an expression, into the next function call/expression. There is flexible support for the type of right-hand side expressions. For more information, see package vignette. To quote Rene Magritte, \"Ceci n'est pas un pipe.\"", - "License": "MIT + file LICENSE", - "URL": "https://magrittr.tidyverse.org, https://github.com/tidyverse/magrittr", - "BugReports": "https://github.com/tidyverse/magrittr/issues", - "Depends": [ - "R (>= 3.4.0)" - ], - "Suggests": [ - "covr", - "knitr", - "rlang", - "rmarkdown", - "testthat" - ], - "VignetteBuilder": "knitr", - "ByteCompile": "Yes", - "Config/Needs/website": "tidyverse/tidytemplate", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.3", - "NeedsCompilation": "yes", - "Author": "Stefan Milton Bache [aut, cph] (Original author and creator of magrittr), Hadley Wickham [aut], Lionel Henry [cre], Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Lionel Henry ", - "Repository": "CRAN" - }, - "memoise": { - "Package": "memoise", - "Version": "2.0.1", - "Source": "Repository", - "Title": "'Memoisation' of Functions", - "Authors@R": "c(person(given = \"Hadley\", family = \"Wickham\", role = \"aut\", email = \"hadley@rstudio.com\"), person(given = \"Jim\", family = \"Hester\", role = \"aut\"), person(given = \"Winston\", family = \"Chang\", role = c(\"aut\", \"cre\"), email = \"winston@rstudio.com\"), person(given = \"Kirill\", family = \"Müller\", role = \"aut\", email = \"krlmlr+r@mailbox.org\"), person(given = \"Daniel\", family = \"Cook\", role = \"aut\", email = \"danielecook@gmail.com\"), person(given = \"Mark\", family = \"Edmondson\", role = \"ctb\", email = \"r@sunholo.com\"))", - "Description": "Cache the results of a function so that when you call it again with the same arguments it returns the previously computed value.", - "License": "MIT + file LICENSE", - "URL": "https://memoise.r-lib.org, https://github.com/r-lib/memoise", - "BugReports": "https://github.com/r-lib/memoise/issues", - "Imports": [ - "rlang (>= 0.4.10)", - "cachem" - ], - "Suggests": [ - "digest", - "aws.s3", - "covr", - "googleAuthR", - "googleCloudStorageR", - "httr", - "testthat" - ], - "Encoding": "UTF-8", - "RoxygenNote": "7.1.2", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut], Jim Hester [aut], Winston Chang [aut, cre], Kirill Müller [aut], Daniel Cook [aut], Mark Edmondson [ctb]", - "Maintainer": "Winston Chang ", - "Repository": "RSPM" - }, - "memuse": { - "Package": "memuse", - "Version": "4.2-3", - "Source": "Repository", - "Title": "Memory Estimation Utilities", - "Description": "How much ram do you need to store a 100,000 by 100,000 matrix? How much ram is your current R session using? How much ram do you even have? Learn the scintillating answer to these and many more such questions with the 'memuse' package.", - "License": "BSD 2-clause License + file LICENSE", - "Depends": [ - "R (>= 3.0.0)" - ], - "Imports": [ - "methods", - "utils" - ], - "NeedsCompilation": "yes", - "ByteCompile": "yes", - "Authors@R": "c( person(\"Drew\", \"Schmidt\", email=\"wrathematics@gmail.com\", role=c(\"aut\", \"cre\")), person(\"Christian\", \"Heckendorf\", role=\"ctb\", comment=\"FreeBSD improvements to meminfo\"), person(\"Wei-Chen\", \"Chen\", role=\"ctb\", comment=\"Windows build fixes\"), person(\"Dan\", \"Burgess\", role=\"ctb\", comment=\"donation of a Mac for development and testing\"))", - "Maintainer": "Drew Schmidt ", - "URL": "https://github.com/shinra-dev/memuse", - "BugReports": "https://github.com/shinra-dev/memuse/issues", - "RoxygenNote": "7.1.2", - "Author": "Drew Schmidt [aut, cre], Christian Heckendorf [ctb] (FreeBSD improvements to meminfo), Wei-Chen Chen [ctb] (Windows build fixes), Dan Burgess [ctb] (donation of a Mac for development and testing)", - "Repository": "CRAN" - }, - "mime": { - "Package": "mime", - "Version": "0.13", - "Source": "Repository", - "Type": "Package", - "Title": "Map Filenames to MIME Types", - "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\", URL = \"https://yihui.org\")), person(\"Jeffrey\", \"Horner\", role = \"ctb\"), person(\"Beilei\", \"Bian\", role = \"ctb\") )", - "Description": "Guesses the MIME type from a filename extension using the data derived from /etc/mime.types in UNIX-type systems.", - "Imports": [ - "tools" - ], - "License": "GPL", - "URL": "https://github.com/yihui/mime", - "BugReports": "https://github.com/yihui/mime/issues", - "RoxygenNote": "7.3.2", - "Encoding": "UTF-8", - "NeedsCompilation": "yes", - "Author": "Yihui Xie [aut, cre] (, https://yihui.org), Jeffrey Horner [ctb], Beilei Bian [ctb]", - "Maintainer": "Yihui Xie ", - "Repository": "CRAN" - }, - "miniUI": { - "Package": "miniUI", - "Version": "0.1.2", - "Source": "Repository", - "Type": "Package", - "Title": "Shiny UI Widgets for Small Screens", - "Authors@R": "c( person(\"Joe\", \"Cheng\", role = c(\"cre\", \"aut\"), email = \"joe@posit.co\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Provides UI widget and layout functions for writing Shiny apps that work well on small screens.", - "License": "GPL-3", - "URL": "https://github.com/rstudio/miniUI", - "BugReports": "https://github.com/rstudio/miniUI/issues", - "Imports": [ - "shiny (>= 0.13)", - "htmltools (>= 0.3)", - "utils" - ], - "RoxygenNote": "7.3.2", - "Encoding": "UTF-8", - "NeedsCompilation": "no", - "Author": "Joe Cheng [cre, aut], Posit Software, PBC [cph, fnd] (03wc8by49)", - "Maintainer": "Joe Cheng ", - "Repository": "CRAN" - }, - "modelr": { - "Package": "modelr", - "Version": "0.1.11", - "Source": "Repository", - "Title": "Modelling Functions that Work with the Pipe", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Functions for modelling that help you seamlessly integrate modelling into a pipeline of data manipulation and visualisation.", - "License": "GPL-3", - "URL": "https://modelr.tidyverse.org, https://github.com/tidyverse/modelr", - "BugReports": "https://github.com/tidyverse/modelr/issues", - "Depends": [ - "R (>= 3.2)" - ], - "Imports": [ - "broom", - "magrittr", - "purrr (>= 0.2.2)", - "rlang (>= 1.0.6)", - "tibble", - "tidyr (>= 0.8.0)", - "tidyselect", - "vctrs" - ], - "Suggests": [ - "compiler", - "covr", - "ggplot2", - "testthat (>= 3.0.0)" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Encoding": "UTF-8", - "LazyData": "true", - "RoxygenNote": "7.2.3", - "Config/testthat/edition": "3", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre], Posit Software, PBC [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "RSPM" - }, - "openssl": { - "Package": "openssl", - "Version": "2.3.4", - "Source": "Repository", - "Type": "Package", - "Title": "Toolkit for Encryption, Signatures and Certificates Based on OpenSSL", - "Authors@R": "c(person(\"Jeroen\", \"Ooms\", role = c(\"aut\", \"cre\"), email = \"jeroenooms@gmail.com\", comment = c(ORCID = \"0000-0002-4035-0289\")), person(\"Oliver\", \"Keyes\", role = \"ctb\"))", - "Description": "Bindings to OpenSSL libssl and libcrypto, plus custom SSH key parsers. Supports RSA, DSA and EC curves P-256, P-384, P-521, and curve25519. Cryptographic signatures can either be created and verified manually or via x509 certificates. AES can be used in cbc, ctr or gcm mode for symmetric encryption; RSA for asymmetric (public key) encryption or EC for Diffie Hellman. High-level envelope functions combine RSA and AES for encrypting arbitrary sized data. Other utilities include key generators, hash functions (md5, sha1, sha256, etc), base64 encoder, a secure random number generator, and 'bignum' math methods for manually performing crypto calculations on large multibyte integers.", - "License": "MIT + file LICENSE", - "URL": "https://jeroen.r-universe.dev/openssl", - "BugReports": "https://github.com/jeroen/openssl/issues", - "SystemRequirements": "OpenSSL >= 1.0.2", - "VignetteBuilder": "knitr", - "Imports": [ - "askpass" - ], - "Suggests": [ - "curl", - "testthat (>= 2.1.0)", - "digest", - "knitr", - "rmarkdown", - "jsonlite", - "jose", - "sodium" - ], - "RoxygenNote": "7.3.2", - "Encoding": "UTF-8", - "NeedsCompilation": "yes", - "Author": "Jeroen Ooms [aut, cre] (ORCID: ), Oliver Keyes [ctb]", - "Maintainer": "Jeroen Ooms ", - "Repository": "CRAN" - }, - "openxlsx": { - "Package": "openxlsx", - "Version": "4.2.8.1", - "Source": "Repository", - "Type": "Package", - "Title": "Read, Write and Edit xlsx Files", - "Date": "2025-10-30", - "Authors@R": "c(person(given = \"Philipp\", family = \"Schauberger\", role = \"aut\", email = \"philipp@schauberger.co.at\"), person(given = \"Alexander\", family = \"Walker\", role = \"aut\", email = \"Alexander.Walker1989@gmail.com\"), person(given = \"Luca\", family = \"Braglia\", role = \"ctb\"), person(given = \"Joshua\", family = \"Sturm\", role = \"ctb\"), person(given = \"Jan Marvin\", family = \"Garbuszus\", role = c(\"ctb\", \"cre\"), email = \"jan.garbuszus@ruhr-uni-bochum.de\"), person(given = \"Jordan Mark\", family = \"Barbone\", role = \"ctb\", email = \"jmbarbone@gmail.com\", comment = c(ORCID = \"0000-0001-9788-3628\")), person(given = \"David\", family = \"Zimmermann\", role = \"ctb\", email = \"david_j_zimmermann@hotmail.com\"), person(given = \"Reinhold\", family = \"Kainhofer\", role = \"ctb\", email = \"reinhold@kainhofer.com\"))", - "Description": "Simplifies the creation of Excel .xlsx files by providing a high level interface to writing, styling and editing worksheets. Through the use of 'Rcpp', read/write times are comparable to the 'xlsx' and 'XLConnect' packages with the added benefit of removing the dependency on Java.", - "License": "MIT + file LICENSE", - "URL": "https://ycphs.github.io/openxlsx/index.html, https://github.com/ycphs/openxlsx", - "BugReports": "https://github.com/ycphs/openxlsx/issues", - "Depends": [ - "R (>= 3.3.0)" - ], - "Imports": [ - "grDevices", - "methods", - "Rcpp", - "stats", - "stringi", - "utils", - "zip" - ], - "Suggests": [ - "curl", - "formula.tools", - "knitr", - "rmarkdown", - "testthat" - ], - "LinkingTo": [ - "Rcpp" - ], - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.3", - "Collate": "'CommentClass.R' 'HyperlinkClass.R' 'RcppExports.R' 'class_definitions.R' 'StyleClass.R' 'WorkbookClass.R' 'asserts.R' 'baseXML.R' 'borderFunctions.R' 'build_workbook.R' 'chartsheet_class.R' 'conditional_formatting.R' 'data-fontSizeLookupTables.R' 'helperFunctions.R' 'loadWorkbook.R' 'onUnload.R' 'openXL.R' 'openxlsx-package.R' 'openxlsx.R' 'openxlsxCoerce.R' 'readWorkbook.R' 'setWindowSize.R' 'sheet_data_class.R' 'utils.R' 'workbook_column_widths.R' 'workbook_read_workbook.R' 'workbook_write_data.R' 'worksheet_class.R' 'wrappers.R' 'writeData.R' 'writeDataTable.R' 'writexlsx.R' 'zzz.R'", - "LazyData": "true", - "NeedsCompilation": "yes", - "Author": "Philipp Schauberger [aut], Alexander Walker [aut], Luca Braglia [ctb], Joshua Sturm [ctb], Jan Marvin Garbuszus [ctb, cre], Jordan Mark Barbone [ctb] (ORCID: ), David Zimmermann [ctb], Reinhold Kainhofer [ctb]", - "Maintainer": "Jan Marvin Garbuszus ", - "Repository": "CRAN" - }, - "otel": { - "Package": "otel", - "Version": "0.2.0", - "Source": "Repository", - "Title": "OpenTelemetry R API", - "Authors@R": "person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\"))", - "Description": "High-quality, ubiquitous, and portable telemetry to enable effective observability. OpenTelemetry is a collection of tools, APIs, and SDKs used to instrument, generate, collect, and export telemetry data (metrics, logs, and traces) for analysis in order to understand your software's performance and behavior. This package implements the OpenTelemetry API: . Use this package as a dependency if you want to instrument your R package for OpenTelemetry.", - "License": "MIT + file LICENSE", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2.9000", - "Depends": [ - "R (>= 3.6.0)" - ], - "Suggests": [ - "callr", - "cli", - "glue", - "jsonlite", - "otelsdk", - "processx", - "shiny", - "spelling", - "testthat (>= 3.0.0)", - "utils", - "withr" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "URL": "https://otel.r-lib.org, https://github.com/r-lib/otel", - "Additional_repositories": "https://github.com/r-lib/otelsdk/releases/download/devel", - "BugReports": "https://github.com/r-lib/otel/issues", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [aut, cre]", - "Maintainer": "Gábor Csárdi ", - "Repository": "CRAN" - }, - "pillar": { - "Package": "pillar", - "Version": "1.11.1", - "Source": "Repository", - "Title": "Coloured Formatting for Columns", - "Authors@R": "c(person(given = \"Kirill\", family = \"M\\u00fcller\", role = c(\"aut\", \"cre\"), email = \"kirill@cynkra.com\", comment = c(ORCID = \"0000-0002-1416-3412\")), person(given = \"Hadley\", family = \"Wickham\", role = \"aut\"), person(given = \"RStudio\", role = \"cph\"))", - "Description": "Provides 'pillar' and 'colonnade' generics designed for formatting columns of data using the full range of colours provided by modern terminals.", - "License": "MIT + file LICENSE", - "URL": "https://pillar.r-lib.org/, https://github.com/r-lib/pillar", - "BugReports": "https://github.com/r-lib/pillar/issues", - "Imports": [ - "cli (>= 2.3.0)", - "glue", - "lifecycle", - "rlang (>= 1.0.2)", - "utf8 (>= 1.1.0)", - "utils", - "vctrs (>= 0.5.0)" - ], - "Suggests": [ - "bit64", - "DBI", - "debugme", - "DiagrammeR", - "dplyr", - "formattable", - "ggplot2", - "knitr", - "lubridate", - "nanotime", - "nycflights13", - "palmerpenguins", - "rmarkdown", - "scales", - "stringi", - "survival", - "testthat (>= 3.1.1)", - "tibble", - "units (>= 0.7.2)", - "vdiffr", - "withr" - ], - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.3.9000", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "true", - "Config/testthat/start-first": "format_multi_fuzz, format_multi_fuzz_2, format_multi, ctl_colonnade, ctl_colonnade_1, ctl_colonnade_2", - "Config/autostyle/scope": "line_breaks", - "Config/autostyle/strict": "true", - "Config/gha/extra-packages": "units=?ignore-before-r=4.3.0", - "Config/Needs/website": "tidyverse/tidytemplate", - "NeedsCompilation": "no", - "Author": "Kirill Müller [aut, cre] (ORCID: ), Hadley Wickham [aut], RStudio [cph]", - "Maintainer": "Kirill Müller ", - "Repository": "CRAN" - }, - "pkgbuild": { - "Package": "pkgbuild", - "Version": "1.4.8", - "Source": "Repository", - "Title": "Find Tools Needed to Build R Packages", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", role = \"aut\"), person(\"Jim\", \"Hester\", role = \"aut\"), person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Provides functions used to build R packages. Locates compilers needed to build R packages on various platforms and ensures the PATH is configured appropriately so R can use them.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/r-lib/pkgbuild, https://pkgbuild.r-lib.org", - "BugReports": "https://github.com/r-lib/pkgbuild/issues", - "Depends": [ - "R (>= 3.5)" - ], - "Imports": [ - "callr (>= 3.2.0)", - "cli (>= 3.4.0)", - "desc", - "processx", - "R6" - ], - "Suggests": [ - "covr", - "cpp11", - "knitr", - "Rcpp", - "rmarkdown", - "testthat (>= 3.2.0)", - "withr (>= 2.3.0)" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/usethis/last-upkeep": "2025-04-30", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut], Jim Hester [aut], Gábor Csárdi [aut, cre], Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Gábor Csárdi ", - "Repository": "CRAN" - }, - "pkgconfig": { - "Package": "pkgconfig", - "Version": "2.0.3", - "Source": "Repository", - "Title": "Private Configuration for 'R' Packages", - "Author": "Gábor Csárdi", - "Maintainer": "Gábor Csárdi ", - "Description": "Set configuration options on a per-package basis. Options set by a given package only apply to that package, other packages are unaffected.", - "License": "MIT + file LICENSE", - "LazyData": "true", - "Imports": [ - "utils" - ], - "Suggests": [ - "covr", - "testthat", - "disposables (>= 1.0.3)" - ], - "URL": "https://github.com/r-lib/pkgconfig#readme", - "BugReports": "https://github.com/r-lib/pkgconfig/issues", - "Encoding": "UTF-8", - "NeedsCompilation": "no", - "Repository": "RSPM" - }, - "pkgdown": { - "Package": "pkgdown", - "Version": "2.2.0", - "Source": "Repository", - "Title": "Make Static HTML Documentation for a Package", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Jay\", \"Hesselberth\", role = \"aut\", comment = c(ORCID = \"0000-0002-6299-179X\")), person(\"Maëlle\", \"Salmon\", role = \"aut\", comment = c(ORCID = \"0000-0002-2815-0399\")), person(\"Olivier\", \"Roy\", role = \"aut\"), person(\"Salim\", \"Brüggemann\", role = \"aut\", comment = c(ORCID = \"0000-0002-5329-5987\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Generate an attractive and useful website from a source package. 'pkgdown' converts your documentation, vignettes, 'README', and more to 'HTML' making it easy to share information about your package online.", - "License": "MIT + file LICENSE", - "URL": "https://pkgdown.r-lib.org/, https://github.com/r-lib/pkgdown", - "BugReports": "https://github.com/r-lib/pkgdown/issues", - "Depends": [ - "R (>= 4.1)" - ], - "Imports": [ - "bslib (>= 0.5.1)", - "callr (>= 3.7.3)", - "cli (>= 3.6.1)", - "desc (>= 1.4.0)", - "downlit (>= 0.4.4)", - "fontawesome", - "fs (>= 1.4.0)", - "httr2 (>= 1.0.2)", - "jsonlite", - "lifecycle", - "openssl", - "purrr (>= 1.0.0)", - "ragg (>= 1.4.0)", - "rlang (>= 1.1.4)", - "rmarkdown (>= 2.27)", - "tibble", - "whisker", - "withr (>= 2.4.3)", - "xml2 (>= 1.3.1)", - "yaml (>= 2.3.9)" - ], - "Suggests": [ - "covr", - "diffviewer", - "evaluate (>= 0.24.0)", - "gert", - "gt", - "htmltools", - "htmlwidgets", - "knitr (>= 1.50)", - "magick", - "methods", - "pkgload (>= 1.0.2)", - "quarto", - "rsconnect", - "rstudioapi", - "rticles", - "sass", - "testthat (>= 3.1.3)", - "tools" - ], - "VignetteBuilder": "knitr, quarto", - "Config/Needs/website": "usethis, servr", - "Config/potools/style": "explicit", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "true", - "Config/testthat/start-first": "build-article, build-quarto-article, build-reference, build", - "Config/usethis/last-upkeep": "2025-09-07", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.3", - "SystemRequirements": "pandoc", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre] (ORCID: ), Jay Hesselberth [aut] (ORCID: ), Maëlle Salmon [aut] (ORCID: ), Olivier Roy [aut], Salim Brüggemann [aut] (ORCID: ), Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "pkgload": { - "Package": "pkgload", - "Version": "1.4.1", - "Source": "Repository", - "Title": "Simulate Package Installation and Attach", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", role = \"aut\"), person(\"Winston\", \"Chang\", role = \"aut\"), person(\"Jim\", \"Hester\", role = \"aut\"), person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(\"R Core team\", role = \"ctb\", comment = \"Some namespace and vignette code extracted from base R\") )", - "Description": "Simulates the process of installing a package and then attaching it. This is a key part of the 'devtools' package as it allows you to rapidly iterate while developing a package.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/r-lib/pkgload, https://pkgload.r-lib.org", - "BugReports": "https://github.com/r-lib/pkgload/issues", - "Depends": [ - "R (>= 3.4.0)" - ], - "Imports": [ - "cli (>= 3.3.0)", - "desc", - "fs", - "glue", - "lifecycle", - "methods", - "pkgbuild", - "processx", - "rlang (>= 1.1.1)", - "rprojroot", - "utils" - ], - "Suggests": [ - "bitops", - "jsonlite", - "mathjaxr", - "pak", - "Rcpp", - "remotes", - "rstudioapi", - "testthat (>= 3.2.1.1)", - "usethis", - "withr" - ], - "Config/Needs/website": "tidyverse/tidytemplate, ggplot2", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "TRUE", - "Config/testthat/start-first": "dll", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut], Winston Chang [aut], Jim Hester [aut], Lionel Henry [aut, cre], Posit Software, PBC [cph, fnd], R Core team [ctb] (Some namespace and vignette code extracted from base R)", - "Maintainer": "Lionel Henry ", - "Repository": "CRAN" - }, - "plotly": { - "Package": "plotly", - "Version": "4.11.0", - "Source": "Repository", - "Title": "Create Interactive Web Graphics via 'plotly.js'", - "Authors@R": "c(person(\"Carson\", \"Sievert\", role = c(\"aut\", \"cre\"), email = \"cpsievert1@gmail.com\", comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Chris\", \"Parmer\", role = \"aut\", email = \"chris@plot.ly\"), person(\"Toby\", \"Hocking\", role = \"aut\", email = \"tdhock5@gmail.com\"), person(\"Scott\", \"Chamberlain\", role = \"aut\", email = \"myrmecocystus@gmail.com\"), person(\"Karthik\", \"Ram\", role = \"aut\", email = \"karthik.ram@gmail.com\"), person(\"Marianne\", \"Corvellec\", role = \"aut\", email = \"marianne.corvellec@igdore.org\", comment = c(ORCID = \"0000-0002-1994-3581\")), person(\"Pedro\", \"Despouy\", role = \"aut\", email = \"pedro@plot.ly\"), person(\"Salim\", \"Brüggemann\", role = \"ctb\", email = \"salim-b@pm.me\", comment = c(ORCID = \"0000-0002-5329-5987\")), person(\"Plotly Technologies Inc.\", role = \"cph\"))", - "License": "MIT + file LICENSE", - "Description": "Create interactive web graphics from 'ggplot2' graphs and/or a custom interface to the (MIT-licensed) JavaScript library 'plotly.js' inspired by the grammar of graphics.", - "URL": "https://plotly-r.com, https://github.com/plotly/plotly.R, https://plotly.com/r/", - "BugReports": "https://github.com/plotly/plotly.R/issues", - "Depends": [ - "R (>= 3.2.0)", - "ggplot2 (>= 3.0.0)" - ], - "Imports": [ - "tools", - "scales", - "httr (>= 1.3.0)", - "jsonlite (>= 1.6)", - "magrittr", - "digest", - "viridisLite", - "base64enc", - "htmltools (>= 0.3.6)", - "htmlwidgets (>= 1.5.2.9001)", - "tidyr (>= 1.0.0)", - "RColorBrewer", - "dplyr", - "vctrs", - "tibble", - "lazyeval (>= 0.2.0)", - "rlang (>= 1.0.0)", - "crosstalk", - "purrr", - "data.table", - "promises" - ], - "Suggests": [ - "MASS", - "maps", - "hexbin", - "ggthemes", - "GGally", - "ggalluvial", - "testthat", - "knitr", - "shiny (>= 1.1.0)", - "shinytest2", - "curl", - "rmarkdown", - "Cairo", - "broom", - "webshot", - "listviewer", - "dendextend", - "sf", - "png", - "IRdisplay", - "processx", - "plotlyGeoAssets", - "forcats", - "withr", - "palmerpenguins", - "rversions", - "reticulate", - "rsvg", - "ggridges" - ], - "LazyData": "true", - "RoxygenNote": "7.3.2", - "Encoding": "UTF-8", - "Config/Needs/check": "tidyverse/ggplot2, ggobi/GGally, rcmdcheck, devtools, reshape2, s2", - "NeedsCompilation": "no", - "Author": "Carson Sievert [aut, cre] (ORCID: ), Chris Parmer [aut], Toby Hocking [aut], Scott Chamberlain [aut], Karthik Ram [aut], Marianne Corvellec [aut] (ORCID: ), Pedro Despouy [aut], Salim Brüggemann [ctb] (ORCID: ), Plotly Technologies Inc. [cph]", - "Maintainer": "Carson Sievert ", - "Repository": "CRAN" - }, - "praise": { - "Package": "praise", - "Version": "1.0.0", - "Source": "Repository", - "Title": "Praise Users", - "Author": "Gabor Csardi, Sindre Sorhus", - "Maintainer": "Gabor Csardi ", - "Description": "Build friendly R packages that praise their users if they have done something good, or they just need it to feel better.", - "License": "MIT + file LICENSE", - "LazyData": "true", - "URL": "https://github.com/gaborcsardi/praise", - "BugReports": "https://github.com/gaborcsardi/praise/issues", - "Suggests": [ - "testthat" - ], - "Collate": "'adjective.R' 'adverb.R' 'exclamation.R' 'verb.R' 'rpackage.R' 'package.R'", - "NeedsCompilation": "no", - "Repository": "CRAN" - }, - "prettyunits": { - "Package": "prettyunits", - "Version": "1.2.0", - "Source": "Repository", - "Title": "Pretty, Human Readable Formatting of Quantities", - "Authors@R": "c( person(\"Gabor\", \"Csardi\", email=\"csardi.gabor@gmail.com\", role=c(\"aut\", \"cre\")), person(\"Bill\", \"Denney\", email=\"wdenney@humanpredictions.com\", role=c(\"ctb\"), comment=c(ORCID=\"0000-0002-5759-428X\")), person(\"Christophe\", \"Regouby\", email=\"christophe.regouby@free.fr\", role=c(\"ctb\")) )", - "Description": "Pretty, human readable formatting of quantities. Time intervals: '1337000' -> '15d 11h 23m 20s'. Vague time intervals: '2674000' -> 'about a month ago'. Bytes: '1337' -> '1.34 kB'. Rounding: '99' with 3 significant digits -> '99.0' p-values: '0.00001' -> '<0.0001'. Colors: '#FF0000' -> 'red'. Quantities: '1239437' -> '1.24 M'.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/r-lib/prettyunits", - "BugReports": "https://github.com/r-lib/prettyunits/issues", - "Depends": [ - "R(>= 2.10)" - ], - "Suggests": [ - "codetools", - "covr", - "testthat" - ], - "RoxygenNote": "7.2.3", - "Encoding": "UTF-8", - "NeedsCompilation": "no", - "Author": "Gabor Csardi [aut, cre], Bill Denney [ctb] (), Christophe Regouby [ctb]", - "Maintainer": "Gabor Csardi ", - "Repository": "RSPM" - }, - "processx": { - "Package": "processx", - "Version": "3.8.6", - "Source": "Repository", - "Title": "Execute and Control System Processes", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\", \"cph\"), comment = c(ORCID = \"0000-0001-7098-9676\")), person(\"Winston\", \"Chang\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(\"Ascent Digital Services\", role = c(\"cph\", \"fnd\")) )", - "Description": "Tools to run system processes in the background. It can check if a background process is running; wait on a background process to finish; get the exit status of finished processes; kill background processes. It can read the standard output and error of the processes, using non-blocking connections. 'processx' can poll a process for standard output or error, with a timeout. It can also poll several processes at once.", - "License": "MIT + file LICENSE", - "URL": "https://processx.r-lib.org, https://github.com/r-lib/processx", - "BugReports": "https://github.com/r-lib/processx/issues", - "Depends": [ - "R (>= 3.4.0)" - ], - "Imports": [ - "ps (>= 1.2.0)", - "R6", - "utils" - ], - "Suggests": [ - "callr (>= 3.7.3)", - "cli (>= 3.3.0)", - "codetools", - "covr", - "curl", - "debugme", - "parallel", - "rlang (>= 1.0.2)", - "testthat (>= 3.0.0)", - "webfakes", - "withr" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.1.9000", - "NeedsCompilation": "yes", - "Author": "Gábor Csárdi [aut, cre, cph] (), Winston Chang [aut], Posit Software, PBC [cph, fnd], Ascent Digital Services [cph, fnd]", - "Maintainer": "Gábor Csárdi ", - "Repository": "CRAN" - }, - "profvis": { - "Package": "profvis", - "Version": "0.4.0", - "Source": "Repository", - "Title": "Interactive Visualizations for Profiling R Code", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Winston\", \"Chang\", role = \"aut\"), person(\"Javier\", \"Luraschi\", role = \"aut\"), person(\"Timothy\", \"Mastny\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(, \"jQuery Foundation\", role = \"cph\", comment = \"jQuery library\"), person(, \"jQuery contributors\", role = c(\"ctb\", \"cph\"), comment = \"jQuery library; authors listed in inst/htmlwidgets/lib/jquery/AUTHORS.txt\"), person(\"Mike\", \"Bostock\", role = c(\"ctb\", \"cph\"), comment = \"D3 library\"), person(, \"D3 contributors\", role = \"ctb\", comment = \"D3 library\"), person(\"Ivan\", \"Sagalaev\", role = c(\"ctb\", \"cph\"), comment = \"highlight.js library\") )", - "Description": "Interactive visualizations for profiling R code.", - "License": "MIT + file LICENSE", - "URL": "https://profvis.r-lib.org, https://github.com/r-lib/profvis", - "BugReports": "https://github.com/r-lib/profvis/issues", - "Depends": [ - "R (>= 4.0)" - ], - "Imports": [ - "htmlwidgets (>= 0.3.2)", - "rlang (>= 1.1.0)", - "vctrs" - ], - "Suggests": [ - "htmltools", - "knitr", - "rmarkdown", - "shiny", - "testthat (>= 3.0.0)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate, rmarkdown", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut, cre], Winston Chang [aut], Javier Luraschi [aut], Timothy Mastny [aut], Posit Software, PBC [cph, fnd], jQuery Foundation [cph] (jQuery library), jQuery contributors [ctb, cph] (jQuery library; authors listed in inst/htmlwidgets/lib/jquery/AUTHORS.txt), Mike Bostock [ctb, cph] (D3 library), D3 contributors [ctb] (D3 library), Ivan Sagalaev [ctb, cph] (highlight.js library)", - "Maintainer": "Hadley Wickham ", - "Repository": "RSPM" - }, - "progress": { - "Package": "progress", - "Version": "1.2.3", - "Source": "Repository", - "Title": "Terminal Progress Bars", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Rich\", \"FitzJohn\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Configurable Progress bars, they may include percentage, elapsed time, and/or the estimated completion time. They work in terminals, in 'Emacs' 'ESS', 'RStudio', 'Windows' 'Rgui' and the 'macOS' 'R.app'. The package also provides a 'C++' 'API', that works with or without 'Rcpp'.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/r-lib/progress#readme, http://r-lib.github.io/progress/", - "BugReports": "https://github.com/r-lib/progress/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "crayon", - "hms", - "prettyunits", - "R6" - ], - "Suggests": [ - "Rcpp", - "testthat (>= 3.0.0)", - "withr" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.3", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [aut, cre], Rich FitzJohn [aut], Posit Software, PBC [cph, fnd]", - "Maintainer": "Gábor Csárdi ", - "Repository": "RSPM" - }, - "promises": { - "Package": "promises", - "Version": "1.5.0", - "Source": "Repository", - "Type": "Package", - "Title": "Abstractions for Promise-Based Asynchronous Programming", - "Authors@R": "c( person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Barret\", \"Schloerke\", , \"barret@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0001-9986-114X\")), person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0002-1576-2126\")), person(\"Charlie\", \"Gao\", , \"charlie.gao@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0002-0750-061X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Provides fundamental abstractions for doing asynchronous programming in R using promises. Asynchronous programming is useful for allowing a single R process to orchestrate multiple tasks in the background while also attending to something else. Semantics are similar to 'JavaScript' promises, but with a syntax that is idiomatic R.", - "License": "MIT + file LICENSE", - "URL": "https://rstudio.github.io/promises/, https://github.com/rstudio/promises", - "BugReports": "https://github.com/rstudio/promises/issues", - "Depends": [ - "R (>= 4.1.0)" - ], - "Imports": [ - "fastmap (>= 1.1.0)", - "later", - "lifecycle", - "magrittr (>= 1.5)", - "otel (>= 0.2.0)", - "R6", - "rlang" - ], - "Suggests": [ - "future (>= 1.21.0)", - "knitr", - "mirai", - "otelsdk (>= 0.2.0)", - "purrr", - "Rcpp", - "rmarkdown", - "spelling", - "testthat (>= 3.0.0)", - "vembedr" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "rsconnect, tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/usethis/last-upkeep": "2025-05-27", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.3", - "NeedsCompilation": "no", - "Author": "Joe Cheng [aut], Barret Schloerke [aut, cre] (ORCID: ), Winston Chang [aut] (ORCID: ), Charlie Gao [aut] (ORCID: ), Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Barret Schloerke ", - "Repository": "CRAN" - }, - "ps": { - "Package": "ps", - "Version": "1.9.1", - "Source": "Repository", - "Title": "List, Query, Manipulate System Processes", - "Authors@R": "c( person(\"Jay\", \"Loden\", role = \"aut\"), person(\"Dave\", \"Daeschler\", role = \"aut\"), person(\"Giampaolo\", \"Rodola'\", role = \"aut\"), person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "List, query and manipulate all system processes, on 'Windows', 'Linux' and 'macOS'.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/r-lib/ps, https://ps.r-lib.org/", - "BugReports": "https://github.com/r-lib/ps/issues", - "Depends": [ - "R (>= 3.4)" - ], - "Imports": [ - "utils" - ], - "Suggests": [ - "callr", - "covr", - "curl", - "pillar", - "pingr", - "processx (>= 3.1.0)", - "R6", - "rlang", - "testthat (>= 3.0.0)", - "webfakes", - "withr" - ], - "Biarch": "true", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "yes", - "Author": "Jay Loden [aut], Dave Daeschler [aut], Giampaolo Rodola' [aut], Gábor Csárdi [aut, cre], Posit Software, PBC [cph, fnd]", - "Maintainer": "Gábor Csárdi ", - "Repository": "CRAN" - }, - "purrr": { - "Package": "purrr", - "Version": "1.2.0", - "Source": "Repository", - "Title": "Functional Programming Tools", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"https://ror.org/03wc8by49\")) )", - "Description": "A complete and consistent functional programming toolkit for R.", - "License": "MIT + file LICENSE", - "URL": "https://purrr.tidyverse.org/, https://github.com/tidyverse/purrr", - "BugReports": "https://github.com/tidyverse/purrr/issues", - "Depends": [ - "R (>= 4.1)" - ], - "Imports": [ - "cli (>= 3.6.1)", - "lifecycle (>= 1.0.3)", - "magrittr (>= 1.5.0)", - "rlang (>= 1.1.1)", - "vctrs (>= 0.6.3)" - ], - "Suggests": [ - "carrier (>= 0.3.0)", - "covr", - "dplyr (>= 0.7.8)", - "httr", - "knitr", - "lubridate", - "mirai (>= 2.5.1)", - "rmarkdown", - "testthat (>= 3.0.0)", - "tibble", - "tidyselect" - ], - "LinkingTo": [ - "cli" - ], - "VignetteBuilder": "knitr", - "Biarch": "true", - "Config/build/compilation-database": "true", - "Config/Needs/website": "tidyverse/tidytemplate, tidyr", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "TRUE", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.3", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut, cre] (ORCID: ), Lionel Henry [aut], Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "ragg": { - "Package": "ragg", - "Version": "1.5.0", - "Source": "Repository", - "Type": "Package", - "Title": "Graphic Devices Based on AGG", - "Authors@R": "c( person(\"Thomas Lin\", \"Pedersen\", , \"thomas.pedersen@posit.co\", role = c(\"cre\", \"aut\"), comment = c(ORCID = \"0000-0002-5147-4711\")), person(\"Maxim\", \"Shemanarev\", role = c(\"aut\", \"cph\"), comment = \"Author of AGG\"), person(\"Tony\", \"Juricic\", , \"tonygeek@yahoo.com\", role = c(\"ctb\", \"cph\"), comment = \"Contributor to AGG\"), person(\"Milan\", \"Marusinec\", , \"milan@marusinec.sk\", role = c(\"ctb\", \"cph\"), comment = \"Contributor to AGG\"), person(\"Spencer\", \"Garrett\", role = \"ctb\", comment = \"Contributor to AGG\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Maintainer": "Thomas Lin Pedersen ", - "Description": "Anti-Grain Geometry (AGG) is a high-quality and high-performance 2D drawing library. The 'ragg' package provides a set of graphic devices based on AGG to use as alternative to the raster devices provided through the 'grDevices' package.", - "License": "MIT + file LICENSE", - "URL": "https://ragg.r-lib.org, https://github.com/r-lib/ragg", - "BugReports": "https://github.com/r-lib/ragg/issues", - "Imports": [ - "systemfonts (>= 1.0.3)", - "textshaping (>= 0.3.0)" - ], - "Suggests": [ - "covr", - "graphics", - "grid", - "testthat (>= 3.0.0)" - ], - "LinkingTo": [ - "systemfonts", - "textshaping" - ], - "Config/build/compilation-database": "true", - "Config/Needs/website": "ggplot2, devoid, magick, bench, tidyr, ggridges, hexbin, sessioninfo, pkgdown, tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/usethis/last-upkeep": "2025-04-25", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "SystemRequirements": "freetype2, libpng, libtiff, libjpeg, libwebp, libwebpmux", - "NeedsCompilation": "yes", - "Author": "Thomas Lin Pedersen [cre, aut] (ORCID: ), Maxim Shemanarev [aut, cph] (Author of AGG), Tony Juricic [ctb, cph] (Contributor to AGG), Milan Marusinec [ctb, cph] (Contributor to AGG), Spencer Garrett [ctb] (Contributor to AGG), Posit Software, PBC [cph, fnd] (ROR: )", - "Repository": "CRAN" - }, - "rappdirs": { - "Package": "rappdirs", - "Version": "0.3.3", - "Source": "Repository", - "Type": "Package", - "Title": "Application Directories: Determine Where to Save Data, Caches, and Logs", - "Authors@R": "c(person(given = \"Hadley\", family = \"Wickham\", role = c(\"trl\", \"cre\", \"cph\"), email = \"hadley@rstudio.com\"), person(given = \"RStudio\", role = \"cph\"), person(given = \"Sridhar\", family = \"Ratnakumar\", role = \"aut\"), person(given = \"Trent\", family = \"Mick\", role = \"aut\"), person(given = \"ActiveState\", role = \"cph\", comment = \"R/appdir.r, R/cache.r, R/data.r, R/log.r translated from appdirs\"), person(given = \"Eddy\", family = \"Petrisor\", role = \"ctb\"), person(given = \"Trevor\", family = \"Davis\", role = c(\"trl\", \"aut\")), person(given = \"Gabor\", family = \"Csardi\", role = \"ctb\"), person(given = \"Gregory\", family = \"Jefferis\", role = \"ctb\"))", - "Description": "An easy way to determine which directories on the users computer you should use to save data, caches and logs. A port of Python's 'Appdirs' () to R.", - "License": "MIT + file LICENSE", - "URL": "https://rappdirs.r-lib.org, https://github.com/r-lib/rappdirs", - "BugReports": "https://github.com/r-lib/rappdirs/issues", - "Depends": [ - "R (>= 3.2)" - ], - "Suggests": [ - "roxygen2", - "testthat (>= 3.0.0)", - "covr", - "withr" - ], - "Copyright": "Original python appdirs module copyright (c) 2010 ActiveState Software Inc. R port copyright Hadley Wickham, RStudio. See file LICENSE for details.", - "Encoding": "UTF-8", - "RoxygenNote": "7.1.1", - "Config/testthat/edition": "3", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [trl, cre, cph], RStudio [cph], Sridhar Ratnakumar [aut], Trent Mick [aut], ActiveState [cph] (R/appdir.r, R/cache.r, R/data.r, R/log.r translated from appdirs), Eddy Petrisor [ctb], Trevor Davis [trl, aut], Gabor Csardi [ctb], Gregory Jefferis [ctb]", - "Maintainer": "Hadley Wickham ", - "Repository": "RSPM" - }, - "rcmdcheck": { - "Package": "rcmdcheck", - "Version": "1.4.0", - "Source": "Repository", - "Title": "Run 'R CMD check' from 'R' and Capture Results", - "Authors@R": "person(given = \"Gábor\", family = \"Csárdi\", role = c(\"cre\", \"aut\"), email = \"csardi.gabor@gmail.com\")", - "Description": "Run 'R CMD check' from 'R' and capture the results of the individual checks. Supports running checks in the background, timeouts, pretty printing and comparing check results.", - "License": "MIT + file LICENSE", - "URL": "https://r-lib.github.io/rcmdcheck/, https://github.com/r-Lib/rcmdcheck#readme", - "BugReports": "https://github.com/r-Lib/rcmdcheck/issues", - "Imports": [ - "callr (>= 3.1.1.9000)", - "cli (>= 3.0.0)", - "curl", - "desc (>= 1.2.0)", - "digest", - "pkgbuild", - "prettyunits", - "R6", - "rprojroot", - "sessioninfo (>= 1.1.1)", - "utils", - "withr", - "xopen" - ], - "Suggests": [ - "covr", - "knitr", - "mockery", - "processx", - "ps", - "rmarkdown", - "svglite", - "testthat", - "webfakes" - ], - "Encoding": "UTF-8", - "RoxygenNote": "7.1.2", - "Config/testthat/edition": "3", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [cre, aut]", - "Maintainer": "Gábor Csárdi ", - "Repository": "RSPM" - }, - "readr": { - "Package": "readr", - "Version": "2.1.5", - "Source": "Repository", - "Title": "Read Rectangular Text Data", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Jim\", \"Hester\", role = \"aut\"), person(\"Romain\", \"Francois\", role = \"ctb\"), person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Shelby\", \"Bearrows\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(\"https://github.com/mandreyel/\", role = \"cph\", comment = \"mio library\"), person(\"Jukka\", \"Jylänki\", role = c(\"ctb\", \"cph\"), comment = \"grisu3 implementation\"), person(\"Mikkel\", \"Jørgensen\", role = c(\"ctb\", \"cph\"), comment = \"grisu3 implementation\") )", - "Description": "The goal of 'readr' is to provide a fast and friendly way to read rectangular data (like 'csv', 'tsv', and 'fwf'). It is designed to flexibly parse many types of data found in the wild, while still cleanly failing when data unexpectedly changes.", - "License": "MIT + file LICENSE", - "URL": "https://readr.tidyverse.org, https://github.com/tidyverse/readr", - "BugReports": "https://github.com/tidyverse/readr/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "cli (>= 3.2.0)", - "clipr", - "crayon", - "hms (>= 0.4.1)", - "lifecycle (>= 0.2.0)", - "methods", - "R6", - "rlang", - "tibble", - "utils", - "vroom (>= 1.6.0)" - ], - "Suggests": [ - "covr", - "curl", - "datasets", - "knitr", - "rmarkdown", - "spelling", - "stringi", - "testthat (>= 3.2.0)", - "tzdb (>= 0.1.1)", - "waldo", - "withr", - "xml2" - ], - "LinkingTo": [ - "cpp11", - "tzdb (>= 0.1.1)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse, tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "false", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.2.3", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut], Jim Hester [aut], Romain Francois [ctb], Jennifer Bryan [aut, cre] (), Shelby Bearrows [ctb], Posit Software, PBC [cph, fnd], https://github.com/mandreyel/ [cph] (mio library), Jukka Jylänki [ctb, cph] (grisu3 implementation), Mikkel Jørgensen [ctb, cph] (grisu3 implementation)", - "Maintainer": "Jennifer Bryan ", - "Repository": "RSPM" - }, - "readxl": { - "Package": "readxl", - "Version": "1.4.5", - "Source": "Repository", - "Title": "Read Excel Files", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Posit, PBC\", role = c(\"cph\", \"fnd\"), comment = \"Copyright holder of all R code and all C/C++ code without explicit copyright attribution\"), person(\"Marcin\", \"Kalicinski\", role = c(\"ctb\", \"cph\"), comment = \"Author of included RapidXML code\"), person(\"Komarov Valery\", role = c(\"ctb\", \"cph\"), comment = \"Author of included libxls code\"), person(\"Christophe Leitienne\", role = c(\"ctb\", \"cph\"), comment = \"Author of included libxls code\"), person(\"Bob Colbert\", role = c(\"ctb\", \"cph\"), comment = \"Author of included libxls code\"), person(\"David Hoerl\", role = c(\"ctb\", \"cph\"), comment = \"Author of included libxls code\"), person(\"Evan Miller\", role = c(\"ctb\", \"cph\"), comment = \"Author of included libxls code\") )", - "Description": "Import excel files into R. Supports '.xls' via the embedded 'libxls' C library and '.xlsx' via the embedded 'RapidXML' C++ library . Works on Windows, Mac and Linux without external dependencies.", - "License": "MIT + file LICENSE", - "URL": "https://readxl.tidyverse.org, https://github.com/tidyverse/readxl", - "BugReports": "https://github.com/tidyverse/readxl/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "cellranger", - "tibble (>= 2.0.1)", - "utils" - ], - "Suggests": [ - "covr", - "knitr", - "rmarkdown", - "testthat (>= 3.1.6)", - "withr" - ], - "LinkingTo": [ - "cpp11 (>= 0.4.0)", - "progress" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate, tidyverse", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "Note": "libxls v1.6.3 c199d13", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut] (), Jennifer Bryan [aut, cre] (), Posit, PBC [cph, fnd] (Copyright holder of all R code and all C/C++ code without explicit copyright attribution), Marcin Kalicinski [ctb, cph] (Author of included RapidXML code), Komarov Valery [ctb, cph] (Author of included libxls code), Christophe Leitienne [ctb, cph] (Author of included libxls code), Bob Colbert [ctb, cph] (Author of included libxls code), David Hoerl [ctb, cph] (Author of included libxls code), Evan Miller [ctb, cph] (Author of included libxls code)", - "Maintainer": "Jennifer Bryan ", - "Repository": "CRAN" - }, - "rematch": { - "Package": "rematch", - "Version": "2.0.0", - "Source": "Repository", - "Title": "Match Regular Expressions with a Nicer 'API'", - "Author": "Gabor Csardi", - "Maintainer": "Gabor Csardi ", - "Description": "A small wrapper on 'regexpr' to extract the matches and captured groups from the match of a regular expression to a character vector.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/gaborcsardi/rematch", - "BugReports": "https://github.com/gaborcsardi/rematch/issues", - "RoxygenNote": "5.0.1.9000", - "Suggests": [ - "covr", - "testthat" - ], - "Encoding": "UTF-8", - "NeedsCompilation": "no", - "Repository": "RSPM" - }, - "rematch2": { - "Package": "rematch2", - "Version": "2.1.2", - "Source": "Repository", - "Title": "Tidy Output from Regular Expression Matching", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", email = \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Matthew\", \"Lincoln\", email = \"matthew.d.lincoln@gmail.com\", role = c(\"ctb\")))", - "Description": "Wrappers on 'regexpr' and 'gregexpr' to return the match results in tidy data frames.", - "License": "MIT + file LICENSE", - "LazyData": "true", - "URL": "https://github.com/r-lib/rematch2#readme", - "BugReports": "https://github.com/r-lib/rematch2/issues", - "RoxygenNote": "7.1.0", - "Imports": [ - "tibble" - ], - "Suggests": [ - "covr", - "testthat" - ], - "Encoding": "UTF-8", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [aut, cre], Matthew Lincoln [ctb]", - "Maintainer": "Gábor Csárdi ", - "Repository": "RSPM" - }, - "remotes": { - "Package": "remotes", - "Version": "2.5.0", - "Source": "Repository", - "Title": "R Package Installation from Remote Repositories, Including 'GitHub'", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Jim\", \"Hester\", role = \"aut\"), person(\"Hadley\", \"Wickham\", role = \"aut\"), person(\"Winston\", \"Chang\", role = \"aut\"), person(\"Martin\", \"Morgan\", role = \"aut\"), person(\"Dan\", \"Tenenbaum\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(\"Ascent Digital Services\", role = \"cph\") )", - "Description": "Download and install R packages stored in 'GitHub', 'GitLab', 'Bitbucket', 'Bioconductor', or plain 'subversion' or 'git' repositories. This package provides the 'install_*' functions in 'devtools'. Indeed most of the code was copied over from 'devtools'.", - "License": "MIT + file LICENSE", - "URL": "https://remotes.r-lib.org, https://github.com/r-lib/remotes#readme", - "BugReports": "https://github.com/r-lib/remotes/issues", - "Depends": [ - "R (>= 3.0.0)" - ], - "Imports": [ - "methods", - "stats", - "tools", - "utils" - ], - "Suggests": [ - "brew", - "callr", - "codetools", - "covr", - "curl", - "git2r (>= 0.23.0)", - "knitr", - "mockery", - "pingr", - "pkgbuild (>= 1.0.1)", - "rmarkdown", - "rprojroot", - "testthat (>= 3.0.0)", - "webfakes", - "withr" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.3", - "SystemRequirements": "Subversion for install_svn, git for install_git", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [aut, cre], Jim Hester [aut], Hadley Wickham [aut], Winston Chang [aut], Martin Morgan [aut], Dan Tenenbaum [aut], Posit Software, PBC [cph, fnd], Ascent Digital Services [cph]", - "Maintainer": "Gábor Csárdi ", - "Repository": "RSPM" - }, - "renv": { - "Package": "renv", - "Version": "1.1.5", - "Source": "Repository", - "Type": "Package", - "Title": "Project Environments", - "Authors@R": "c( person(\"Kevin\", \"Ushey\", role = c(\"aut\", \"cre\"), email = \"kevin@rstudio.com\", comment = c(ORCID = \"0000-0003-2880-7407\")), person(\"Hadley\", \"Wickham\", role = c(\"aut\"), email = \"hadley@rstudio.com\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "A dependency management toolkit for R. Using 'renv', you can create and manage project-local R libraries, save the state of these libraries to a 'lockfile', and later restore your library as required. Together, these tools can help make your projects more isolated, portable, and reproducible.", - "License": "MIT + file LICENSE", - "URL": "https://rstudio.github.io/renv/, https://github.com/rstudio/renv", - "BugReports": "https://github.com/rstudio/renv/issues", - "Imports": [ - "utils" - ], - "Suggests": [ - "BiocManager", - "cli", - "compiler", - "covr", - "cpp11", - "devtools", - "generics", - "gitcreds", - "jsonlite", - "jsonvalidate", - "knitr", - "miniUI", - "modules", - "packrat", - "pak", - "R6", - "remotes", - "reticulate", - "rmarkdown", - "rstudioapi", - "shiny", - "testthat", - "uuid", - "waldo", - "yaml", - "webfakes" - ], - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "true", - "Config/testthat/start-first": "bioconductor,python,install,restore,snapshot,retrieve,remotes", - "NeedsCompilation": "no", - "Author": "Kevin Ushey [aut, cre] (ORCID: ), Hadley Wickham [aut] (ORCID: ), Posit Software, PBC [cph, fnd]", - "Maintainer": "Kevin Ushey ", - "Repository": "CRAN" - }, - "reprex": { - "Package": "reprex", - "Version": "2.1.1", - "Source": "Repository", - "Title": "Prepare Reproducible Example Code via the Clipboard", - "Authors@R": "c( person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Jim\", \"Hester\", role = \"aut\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"David\", \"Robinson\", , \"admiral.david@gmail.com\", role = \"aut\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Christophe\", \"Dervieux\", , \"cderv@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4474-2498\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Convenience wrapper that uses the 'rmarkdown' package to render small snippets of code to target formats that include both code and output. The goal is to encourage the sharing of small, reproducible, and runnable examples on code-oriented websites, such as and , or in email. The user's clipboard is the default source of input code and the default target for rendered output. 'reprex' also extracts clean, runnable R code from various common formats, such as copy/paste from an R session.", - "License": "MIT + file LICENSE", - "URL": "https://reprex.tidyverse.org, https://github.com/tidyverse/reprex", - "BugReports": "https://github.com/tidyverse/reprex/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "callr (>= 3.6.0)", - "cli (>= 3.2.0)", - "clipr (>= 0.4.0)", - "fs", - "glue", - "knitr (>= 1.23)", - "lifecycle", - "rlang (>= 1.0.0)", - "rmarkdown", - "rstudioapi", - "utils", - "withr (>= 2.3.0)" - ], - "Suggests": [ - "covr", - "fortunes", - "miniUI", - "rprojroot", - "sessioninfo", - "shiny", - "spelling", - "styler (>= 1.2.0)", - "testthat (>= 3.2.1)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "dplyr, tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "TRUE", - "Config/testthat/start-first": "knitr-options, venues, reprex", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.2", - "SystemRequirements": "pandoc (>= 2.0) - https://pandoc.org/", - "NeedsCompilation": "no", - "Author": "Jennifer Bryan [aut, cre] (), Jim Hester [aut] (), David Robinson [aut], Hadley Wickham [aut] (), Christophe Dervieux [aut] (), Posit Software, PBC [cph, fnd]", - "Maintainer": "Jennifer Bryan ", - "Repository": "RSPM" - }, - "rlang": { - "Package": "rlang", - "Version": "1.1.6", - "Source": "Repository", - "Title": "Functions for Base Types and Core R and 'Tidyverse' Features", - "Description": "A toolbox for working with base types, core R features like the condition system, and core 'Tidyverse' features like tidy evaluation.", - "Authors@R": "c( person(\"Lionel\", \"Henry\", ,\"lionel@posit.co\", c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", ,\"hadley@posit.co\", \"aut\"), person(given = \"mikefc\", email = \"mikefc@coolbutuseless.com\", role = \"cph\", comment = \"Hash implementation based on Mike's xxhashlite\"), person(given = \"Yann\", family = \"Collet\", role = \"cph\", comment = \"Author of the embedded xxHash library\"), person(given = \"Posit, PBC\", role = c(\"cph\", \"fnd\")) )", - "License": "MIT + file LICENSE", - "ByteCompile": "true", - "Biarch": "true", - "Depends": [ - "R (>= 3.5.0)" - ], - "Imports": [ - "utils" - ], - "Suggests": [ - "cli (>= 3.1.0)", - "covr", - "crayon", - "desc", - "fs", - "glue", - "knitr", - "magrittr", - "methods", - "pillar", - "pkgload", - "rmarkdown", - "stats", - "testthat (>= 3.2.0)", - "tibble", - "usethis", - "vctrs (>= 0.2.3)", - "withr" - ], - "Enhances": [ - "winch" - ], - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "URL": "https://rlang.r-lib.org, https://github.com/r-lib/rlang", - "BugReports": "https://github.com/r-lib/rlang/issues", - "Config/build/compilation-database": "true", - "Config/testthat/edition": "3", - "Config/Needs/website": "dplyr, tidyverse/tidytemplate", - "NeedsCompilation": "yes", - "Author": "Lionel Henry [aut, cre], Hadley Wickham [aut], mikefc [cph] (Hash implementation based on Mike's xxhashlite), Yann Collet [cph] (Author of the embedded xxHash library), Posit, PBC [cph, fnd]", - "Maintainer": "Lionel Henry ", - "Repository": "CRAN" - }, - "rmarkdown": { - "Package": "rmarkdown", - "Version": "2.30", - "Source": "Repository", - "Type": "Package", - "Title": "Dynamic Documents for R", - "Authors@R": "c( person(\"JJ\", \"Allaire\", , \"jj@posit.co\", role = \"aut\"), person(\"Yihui\", \"Xie\", , \"xie@yihui.name\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Christophe\", \"Dervieux\", , \"cderv@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4474-2498\")), person(\"Jonathan\", \"McPherson\", , \"jonathan@posit.co\", role = \"aut\"), person(\"Javier\", \"Luraschi\", role = \"aut\"), person(\"Kevin\", \"Ushey\", , \"kevin@posit.co\", role = \"aut\"), person(\"Aron\", \"Atkins\", , \"aron@posit.co\", role = \"aut\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Joe\", \"Cheng\", , \"joe@posit.co\", role = \"aut\"), person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = \"aut\"), person(\"Richard\", \"Iannone\", , \"rich@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-3925-190X\")), person(\"Andrew\", \"Dunning\", role = \"ctb\", comment = c(ORCID = \"0000-0003-0464-5036\")), person(\"Atsushi\", \"Yasumoto\", role = c(\"ctb\", \"cph\"), comment = c(ORCID = \"0000-0002-8335-495X\", cph = \"Number sections Lua filter\")), person(\"Barret\", \"Schloerke\", role = \"ctb\"), person(\"Carson\", \"Sievert\", role = \"ctb\", comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Devon\", \"Ryan\", , \"dpryan79@gmail.com\", role = \"ctb\", comment = c(ORCID = \"0000-0002-8549-0971\")), person(\"Frederik\", \"Aust\", , \"frederik.aust@uni-koeln.de\", role = \"ctb\", comment = c(ORCID = \"0000-0003-4900-788X\")), person(\"Jeff\", \"Allen\", , \"jeff@posit.co\", role = \"ctb\"), person(\"JooYoung\", \"Seo\", role = \"ctb\", comment = c(ORCID = \"0000-0002-4064-6012\")), person(\"Malcolm\", \"Barrett\", role = \"ctb\"), person(\"Rob\", \"Hyndman\", , \"Rob.Hyndman@monash.edu\", role = \"ctb\"), person(\"Romain\", \"Lesur\", role = \"ctb\"), person(\"Roy\", \"Storey\", role = \"ctb\"), person(\"Ruben\", \"Arslan\", , \"ruben.arslan@uni-goettingen.de\", role = \"ctb\"), person(\"Sergio\", \"Oller\", role = \"ctb\"), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(, \"jQuery UI contributors\", role = c(\"ctb\", \"cph\"), comment = \"jQuery UI library; authors listed in inst/rmd/h/jqueryui/AUTHORS.txt\"), person(\"Mark\", \"Otto\", role = \"ctb\", comment = \"Bootstrap library\"), person(\"Jacob\", \"Thornton\", role = \"ctb\", comment = \"Bootstrap library\"), person(, \"Bootstrap contributors\", role = \"ctb\", comment = \"Bootstrap library\"), person(, \"Twitter, Inc\", role = \"cph\", comment = \"Bootstrap library\"), person(\"Alexander\", \"Farkas\", role = c(\"ctb\", \"cph\"), comment = \"html5shiv library\"), person(\"Scott\", \"Jehl\", role = c(\"ctb\", \"cph\"), comment = \"Respond.js library\"), person(\"Ivan\", \"Sagalaev\", role = c(\"ctb\", \"cph\"), comment = \"highlight.js library\"), person(\"Greg\", \"Franko\", role = c(\"ctb\", \"cph\"), comment = \"tocify library\"), person(\"John\", \"MacFarlane\", role = c(\"ctb\", \"cph\"), comment = \"Pandoc templates\"), person(, \"Google, Inc.\", role = c(\"ctb\", \"cph\"), comment = \"ioslides library\"), person(\"Dave\", \"Raggett\", role = \"ctb\", comment = \"slidy library\"), person(, \"W3C\", role = \"cph\", comment = \"slidy library\"), person(\"Dave\", \"Gandy\", role = c(\"ctb\", \"cph\"), comment = \"Font-Awesome\"), person(\"Ben\", \"Sperry\", role = \"ctb\", comment = \"Ionicons\"), person(, \"Drifty\", role = \"cph\", comment = \"Ionicons\"), person(\"Aidan\", \"Lister\", role = c(\"ctb\", \"cph\"), comment = \"jQuery StickyTabs\"), person(\"Benct Philip\", \"Jonsson\", role = c(\"ctb\", \"cph\"), comment = \"pagebreak Lua filter\"), person(\"Albert\", \"Krewinkel\", role = c(\"ctb\", \"cph\"), comment = \"pagebreak Lua filter\") )", - "Description": "Convert R Markdown documents into a variety of formats.", - "License": "GPL-3", - "URL": "https://github.com/rstudio/rmarkdown, https://pkgs.rstudio.com/rmarkdown/", - "BugReports": "https://github.com/rstudio/rmarkdown/issues", - "Depends": [ - "R (>= 3.0)" - ], - "Imports": [ - "bslib (>= 0.2.5.1)", - "evaluate (>= 0.13)", - "fontawesome (>= 0.5.0)", - "htmltools (>= 0.5.1)", - "jquerylib", - "jsonlite", - "knitr (>= 1.43)", - "methods", - "tinytex (>= 0.31)", - "tools", - "utils", - "xfun (>= 0.36)", - "yaml (>= 2.1.19)" - ], - "Suggests": [ - "digest", - "dygraphs", - "fs", - "rsconnect", - "downlit (>= 0.4.0)", - "katex (>= 1.4.0)", - "sass (>= 0.4.0)", - "shiny (>= 1.6.0)", - "testthat (>= 3.0.3)", - "tibble", - "vctrs", - "cleanrmd", - "withr (>= 2.4.2)", - "xml2" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "rstudio/quillt, pkgdown", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "SystemRequirements": "pandoc (>= 1.14) - http://pandoc.org", - "NeedsCompilation": "no", - "Author": "JJ Allaire [aut], Yihui Xie [aut, cre] (ORCID: ), Christophe Dervieux [aut] (ORCID: ), Jonathan McPherson [aut], Javier Luraschi [aut], Kevin Ushey [aut], Aron Atkins [aut], Hadley Wickham [aut], Joe Cheng [aut], Winston Chang [aut], Richard Iannone [aut] (ORCID: ), Andrew Dunning [ctb] (ORCID: ), Atsushi Yasumoto [ctb, cph] (ORCID: , cph: Number sections Lua filter), Barret Schloerke [ctb], Carson Sievert [ctb] (ORCID: ), Devon Ryan [ctb] (ORCID: ), Frederik Aust [ctb] (ORCID: ), Jeff Allen [ctb], JooYoung Seo [ctb] (ORCID: ), Malcolm Barrett [ctb], Rob Hyndman [ctb], Romain Lesur [ctb], Roy Storey [ctb], Ruben Arslan [ctb], Sergio Oller [ctb], Posit Software, PBC [cph, fnd], jQuery UI contributors [ctb, cph] (jQuery UI library; authors listed in inst/rmd/h/jqueryui/AUTHORS.txt), Mark Otto [ctb] (Bootstrap library), Jacob Thornton [ctb] (Bootstrap library), Bootstrap contributors [ctb] (Bootstrap library), Twitter, Inc [cph] (Bootstrap library), Alexander Farkas [ctb, cph] (html5shiv library), Scott Jehl [ctb, cph] (Respond.js library), Ivan Sagalaev [ctb, cph] (highlight.js library), Greg Franko [ctb, cph] (tocify library), John MacFarlane [ctb, cph] (Pandoc templates), Google, Inc. [ctb, cph] (ioslides library), Dave Raggett [ctb] (slidy library), W3C [cph] (slidy library), Dave Gandy [ctb, cph] (Font-Awesome), Ben Sperry [ctb] (Ionicons), Drifty [cph] (Ionicons), Aidan Lister [ctb, cph] (jQuery StickyTabs), Benct Philip Jonsson [ctb, cph] (pagebreak Lua filter), Albert Krewinkel [ctb, cph] (pagebreak Lua filter)", - "Maintainer": "Yihui Xie ", - "Repository": "CRAN" - }, - "roxygen2": { - "Package": "roxygen2", - "Version": "7.3.3", - "Source": "Repository", - "Title": "In-Line Documentation for R", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\", \"cph\"), comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Peter\", \"Danenberg\", , \"pcd@roxygen.org\", role = c(\"aut\", \"cph\")), person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = \"aut\"), person(\"Manuel\", \"Eugster\", role = c(\"aut\", \"cph\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Generate your Rd documentation, 'NAMESPACE' file, and collation field using specially formatted comments. Writing documentation in-line with code makes it easier to keep your documentation up-to-date as your requirements change. 'roxygen2' is inspired by the 'Doxygen' system for C++.", - "License": "MIT + file LICENSE", - "URL": "https://roxygen2.r-lib.org/, https://github.com/r-lib/roxygen2", - "BugReports": "https://github.com/r-lib/roxygen2/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "brew", - "cli (>= 3.3.0)", - "commonmark", - "desc (>= 1.2.0)", - "knitr", - "methods", - "pkgload (>= 1.0.2)", - "purrr (>= 1.0.0)", - "R6 (>= 2.1.2)", - "rlang (>= 1.0.6)", - "stringi", - "stringr (>= 1.0.0)", - "utils", - "withr", - "xml2" - ], - "Suggests": [ - "covr", - "R.methodsS3", - "R.oo", - "rmarkdown (>= 2.16)", - "testthat (>= 3.1.2)", - "yaml" - ], - "LinkingTo": [ - "cpp11" - ], - "VignetteBuilder": "knitr", - "Config/Needs/development": "testthat", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "TRUE", - "Encoding": "UTF-8", - "Language": "en-GB", - "RoxygenNote": "7.3.2.9000", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut, cre, cph] (ORCID: ), Peter Danenberg [aut, cph], Gábor Csárdi [aut], Manuel Eugster [aut, cph], Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "rprojroot": { - "Package": "rprojroot", - "Version": "2.1.1", - "Source": "Repository", - "Title": "Finding Files in Project Subdirectories", - "Authors@R": "person(given = \"Kirill\", family = \"M\\u00fcller\", role = c(\"aut\", \"cre\"), email = \"kirill@cynkra.com\", comment = c(ORCID = \"0000-0002-1416-3412\"))", - "Description": "Robust, reliable and flexible paths to files below a project root. The 'root' of a project is defined as a directory that matches a certain criterion, e.g., it contains a certain regular file.", - "License": "MIT + file LICENSE", - "URL": "https://rprojroot.r-lib.org/, https://github.com/r-lib/rprojroot", - "BugReports": "https://github.com/r-lib/rprojroot/issues", - "Depends": [ - "R (>= 3.0.0)" - ], - "Suggests": [ - "covr", - "knitr", - "lifecycle", - "rlang", - "rmarkdown", - "testthat (>= 3.2.0)", - "withr" - ], - "VignetteBuilder": "knitr", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2.9000", - "Config/autostyle/scope": "line_breaks", - "Config/autostyle/strict": "true", - "Config/Needs/website": "tidyverse/tidytemplate", - "NeedsCompilation": "no", - "Author": "Kirill Müller [aut, cre] (ORCID: )", - "Maintainer": "Kirill Müller ", - "Repository": "CRAN" - }, - "rstudioapi": { - "Package": "rstudioapi", - "Version": "0.17.1", - "Source": "Repository", - "Title": "Safely Access the RStudio API", - "Description": "Access the RStudio API (if available) and provide informative error messages when it's not.", - "Authors@R": "c( person(\"Kevin\", \"Ushey\", role = c(\"aut\", \"cre\"), email = \"kevin@rstudio.com\"), person(\"JJ\", \"Allaire\", role = c(\"aut\"), email = \"jj@posit.co\"), person(\"Hadley\", \"Wickham\", role = c(\"aut\"), email = \"hadley@posit.co\"), person(\"Gary\", \"Ritchie\", role = c(\"aut\"), email = \"gary@posit.co\"), person(family = \"RStudio\", role = \"cph\") )", - "Maintainer": "Kevin Ushey ", - "License": "MIT + file LICENSE", - "URL": "https://rstudio.github.io/rstudioapi/, https://github.com/rstudio/rstudioapi", - "BugReports": "https://github.com/rstudio/rstudioapi/issues", - "RoxygenNote": "7.3.2", - "Suggests": [ - "testthat", - "knitr", - "rmarkdown", - "clipr", - "covr" - ], - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "NeedsCompilation": "no", - "Author": "Kevin Ushey [aut, cre], JJ Allaire [aut], Hadley Wickham [aut], Gary Ritchie [aut], RStudio [cph]", - "Repository": "CRAN" - }, - "rversions": { - "Package": "rversions", - "Version": "3.0.0", - "Source": "Repository", - "Title": "Query 'R' Versions, Including 'r-release' and 'r-oldrel'", - "Authors@R": "c(person(given = \"Gábor\", family = \"Csárdi\", role = c(\"aut\", \"cre\"), email = \"csardi.gabor@gmail.com\"), person(given = \"Jeroen\", family = \"Ooms\", role = \"ctb\", email = \"jeroen.ooms@stat.ucla.edu\"), person(given = \"R Consortium\", role = \"fnd\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")))", - "Description": "Query the main 'R' 'SVN' repository to find the versions 'r-release' and 'r-oldrel' refer to, and also all previous 'R' versions and their release dates.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/r-hub/rversions, https://r-hub.github.io/rversions/", - "BugReports": "https://github.com/r-hub/rversions/issues", - "Imports": [ - "curl" - ], - "Suggests": [ - "pillar", - "testthat (>= 3.0.0)", - "webfakes", - "withr" - ], - "Encoding": "UTF-8", - "RoxygenNote": "7.3.3", - "Config/testthat/edition": "3", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [aut, cre], Jeroen Ooms [ctb], R Consortium [fnd], Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Gábor Csárdi ", - "Repository": "CRAN" - }, - "rvest": { - "Package": "rvest", - "Version": "1.0.5", - "Source": "Repository", - "Title": "Easily Harvest (Scrape) Web Pages", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Wrappers around the 'xml2' and 'httr' packages to make it easy to download, then manipulate, HTML and XML.", - "License": "MIT + file LICENSE", - "URL": "https://rvest.tidyverse.org/, https://github.com/tidyverse/rvest", - "BugReports": "https://github.com/tidyverse/rvest/issues", - "Depends": [ - "R (>= 4.1)" - ], - "Imports": [ - "cli", - "glue", - "httr (>= 0.5)", - "lifecycle (>= 1.0.3)", - "magrittr", - "rlang (>= 1.1.0)", - "selectr", - "tibble", - "xml2 (>= 1.4.0)" - ], - "Suggests": [ - "chromote", - "covr", - "knitr", - "purrr", - "R6", - "readr", - "repurrrsive", - "rmarkdown", - "spelling", - "stringi (>= 0.3.1)", - "testthat (>= 3.0.2)", - "tidyr", - "webfakes" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "true", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre], Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "sass": { - "Package": "sass", - "Version": "0.4.10", - "Source": "Repository", - "Type": "Package", - "Title": "Syntactically Awesome Style Sheets ('Sass')", - "Description": "An 'SCSS' compiler, powered by the 'LibSass' library. With this, R developers can use variables, inheritance, and functions to generate dynamic style sheets. The package uses the 'Sass CSS' extension language, which is stable, powerful, and CSS compatible.", - "Authors@R": "c( person(\"Joe\", \"Cheng\", , \"joe@rstudio.com\", \"aut\"), person(\"Timothy\", \"Mastny\", , \"tim.mastny@gmail.com\", \"aut\"), person(\"Richard\", \"Iannone\", , \"rich@rstudio.com\", \"aut\", comment = c(ORCID = \"0000-0003-3925-190X\")), person(\"Barret\", \"Schloerke\", , \"barret@rstudio.com\", \"aut\", comment = c(ORCID = \"0000-0001-9986-114X\")), person(\"Carson\", \"Sievert\", , \"carson@rstudio.com\", c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Christophe\", \"Dervieux\", , \"cderv@rstudio.com\", c(\"ctb\"), comment = c(ORCID = \"0000-0003-4474-2498\")), person(family = \"RStudio\", role = c(\"cph\", \"fnd\")), person(family = \"Sass Open Source Foundation\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Greter\", \"Marcel\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Mifsud\", \"Michael\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Hampton\", \"Catlin\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Natalie\", \"Weizenbaum\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Chris\", \"Eppstein\", role = c(\"ctb\", \"cph\"), comment = \"LibSass library\"), person(\"Adams\", \"Joseph\", role = c(\"ctb\", \"cph\"), comment = \"json.cpp\"), person(\"Trifunovic\", \"Nemanja\", role = c(\"ctb\", \"cph\"), comment = \"utf8.h\") )", - "License": "MIT + file LICENSE", - "URL": "https://rstudio.github.io/sass/, https://github.com/rstudio/sass", - "BugReports": "https://github.com/rstudio/sass/issues", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "SystemRequirements": "GNU make", - "Imports": [ - "fs (>= 1.2.4)", - "rlang (>= 0.4.10)", - "htmltools (>= 0.5.1)", - "R6", - "rappdirs" - ], - "Suggests": [ - "testthat", - "knitr", - "rmarkdown", - "withr", - "shiny", - "curl" - ], - "VignetteBuilder": "knitr", - "Config/testthat/edition": "3", - "NeedsCompilation": "yes", - "Author": "Joe Cheng [aut], Timothy Mastny [aut], Richard Iannone [aut] (), Barret Schloerke [aut] (), Carson Sievert [aut, cre] (), Christophe Dervieux [ctb] (), RStudio [cph, fnd], Sass Open Source Foundation [ctb, cph] (LibSass library), Greter Marcel [ctb, cph] (LibSass library), Mifsud Michael [ctb, cph] (LibSass library), Hampton Catlin [ctb, cph] (LibSass library), Natalie Weizenbaum [ctb, cph] (LibSass library), Chris Eppstein [ctb, cph] (LibSass library), Adams Joseph [ctb, cph] (json.cpp), Trifunovic Nemanja [ctb, cph] (utf8.h)", - "Maintainer": "Carson Sievert ", - "Repository": "CRAN" - }, - "scales": { - "Package": "scales", - "Version": "1.4.0", - "Source": "Repository", - "Title": "Scale Functions for Visualization", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Thomas Lin\", \"Pedersen\", , \"thomas.pedersen@posit.co\", role = c(\"cre\", \"aut\"), comment = c(ORCID = \"0000-0002-5147-4711\")), person(\"Dana\", \"Seidel\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Graphical scales map data to aesthetics, and provide methods for automatically determining breaks and labels for axes and legends.", - "License": "MIT + file LICENSE", - "URL": "https://scales.r-lib.org, https://github.com/r-lib/scales", - "BugReports": "https://github.com/r-lib/scales/issues", - "Depends": [ - "R (>= 4.1)" - ], - "Imports": [ - "cli", - "farver (>= 2.0.3)", - "glue", - "labeling", - "lifecycle", - "R6", - "RColorBrewer", - "rlang (>= 1.1.0)", - "viridisLite" - ], - "Suggests": [ - "bit64", - "covr", - "dichromat", - "ggplot2", - "hms (>= 0.5.0)", - "stringi", - "testthat (>= 3.0.0)" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/usethis/last-upkeep": "2025-04-23", - "Encoding": "UTF-8", - "LazyLoad": "yes", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut], Thomas Lin Pedersen [cre, aut] (), Dana Seidel [aut], Posit Software, PBC [cph, fnd] (03wc8by49)", - "Maintainer": "Thomas Lin Pedersen ", - "Repository": "CRAN" - }, - "selectr": { - "Package": "selectr", - "Version": "0.4-2", - "Source": "Repository", - "Type": "Package", - "Title": "Translate CSS Selectors to XPath Expressions", - "Date": "2019-11-20", - "Authors@R": "c(person(\"Simon\", \"Potter\", role = c(\"aut\", \"trl\", \"cre\"), email = \"simon@sjp.co.nz\"), person(\"Simon\", \"Sapin\", role = \"aut\"), person(\"Ian\", \"Bicking\", role = \"aut\"))", - "License": "BSD_3_clause + file LICENCE", - "Depends": [ - "R (>= 3.0)" - ], - "Imports": [ - "methods", - "stringr", - "R6" - ], - "Suggests": [ - "testthat", - "XML", - "xml2" - ], - "URL": "https://sjp.co.nz/projects/selectr", - "BugReports": "https://github.com/sjp/selectr/issues", - "Description": "Translates a CSS3 selector into an equivalent XPath expression. This allows us to use CSS selectors when working with the XML package as it can only evaluate XPath expressions. Also provided are convenience functions useful for using CSS selectors on XML nodes. This package is a port of the Python package 'cssselect' ().", - "NeedsCompilation": "no", - "Author": "Simon Potter [aut, trl, cre], Simon Sapin [aut], Ian Bicking [aut]", - "Maintainer": "Simon Potter ", - "Repository": "CRAN" - }, - "sessioninfo": { - "Package": "sessioninfo", - "Version": "1.2.3", - "Source": "Repository", - "Title": "R Session Information", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = \"cre\"), person(\"Hadley\", \"Wickham\", role = \"aut\"), person(\"Winston\", \"Chang\", role = \"aut\"), person(\"Robert\", \"Flight\", role = \"aut\"), person(\"Kirill\", \"Müller\", role = \"aut\"), person(\"Jim\", \"Hester\", role = \"aut\"), person(\"R Core team\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Maintainer": "Gábor Csárdi ", - "Description": "Query and print information about the current R session. It is similar to 'utils::sessionInfo()', but includes more information about packages, and where they were installed from.", - "License": "GPL-2", - "URL": "https://github.com/r-lib/sessioninfo#readme, https://sessioninfo.r-lib.org", - "BugReports": "https://github.com/r-lib/sessioninfo/issues", - "Depends": [ - "R (>= 3.4)" - ], - "Imports": [ - "cli (>= 3.1.0)", - "tools", - "utils" - ], - "Suggests": [ - "callr", - "covr", - "gh", - "reticulate", - "rmarkdown", - "testthat (>= 3.2.0)", - "withr" - ], - "Config/Needs/website": "pkgdown, tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "true", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [cre], Hadley Wickham [aut], Winston Chang [aut], Robert Flight [aut], Kirill Müller [aut], Jim Hester [aut], R Core team [ctb], Posit Software, PBC [cph, fnd]", - "Repository": "CRAN" - }, - "shiny": { - "Package": "shiny", - "Version": "1.11.1", - "Source": "Repository", - "Type": "Package", - "Title": "Web Application Framework for R", - "Authors@R": "c( person(\"Winston\", \"Chang\", role = c(\"aut\", \"cre\"), email = \"winston@posit.co\", comment = c(ORCID = \"0000-0002-1576-2126\")), person(\"Joe\", \"Cheng\", role = \"aut\", email = \"joe@posit.co\"), person(\"JJ\", \"Allaire\", role = \"aut\", email = \"jj@posit.co\"), person(\"Carson\", \"Sievert\", role = \"aut\", email = \"carson@posit.co\", comment = c(ORCID = \"0000-0002-4958-2844\")), person(\"Barret\", \"Schloerke\", role = \"aut\", email = \"barret@posit.co\", comment = c(ORCID = \"0000-0001-9986-114X\")), person(\"Yihui\", \"Xie\", role = \"aut\", email = \"yihui@posit.co\"), person(\"Jeff\", \"Allen\", role = \"aut\"), person(\"Jonathan\", \"McPherson\", role = \"aut\", email = \"jonathan@posit.co\"), person(\"Alan\", \"Dipert\", role = \"aut\"), person(\"Barbara\", \"Borges\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(family = \"jQuery Foundation\", role = \"cph\", comment = \"jQuery library and jQuery UI library\"), person(family = \"jQuery contributors\", role = c(\"ctb\", \"cph\"), comment = \"jQuery library; authors listed in inst/www/shared/jquery-AUTHORS.txt\"), person(family = \"jQuery UI contributors\", role = c(\"ctb\", \"cph\"), comment = \"jQuery UI library; authors listed in inst/www/shared/jqueryui/AUTHORS.txt\"), person(\"Mark\", \"Otto\", role = \"ctb\", comment = \"Bootstrap library\"), person(\"Jacob\", \"Thornton\", role = \"ctb\", comment = \"Bootstrap library\"), person(family = \"Bootstrap contributors\", role = \"ctb\", comment = \"Bootstrap library\"), person(family = \"Twitter, Inc\", role = \"cph\", comment = \"Bootstrap library\"), person(\"Prem Nawaz\", \"Khan\", role = \"ctb\", comment = \"Bootstrap accessibility plugin\"), person(\"Victor\", \"Tsaran\", role = \"ctb\", comment = \"Bootstrap accessibility plugin\"), person(\"Dennis\", \"Lembree\", role = \"ctb\", comment = \"Bootstrap accessibility plugin\"), person(\"Srinivasu\", \"Chakravarthula\", role = \"ctb\", comment = \"Bootstrap accessibility plugin\"), person(\"Cathy\", \"O'Connor\", role = \"ctb\", comment = \"Bootstrap accessibility plugin\"), person(family = \"PayPal, Inc\", role = \"cph\", comment = \"Bootstrap accessibility plugin\"), person(\"Stefan\", \"Petre\", role = c(\"ctb\", \"cph\"), comment = \"Bootstrap-datepicker library\"), person(\"Andrew\", \"Rowls\", role = c(\"ctb\", \"cph\"), comment = \"Bootstrap-datepicker library\"), person(\"Brian\", \"Reavis\", role = c(\"ctb\", \"cph\"), comment = \"selectize.js library\"), person(\"Salmen\", \"Bejaoui\", role = c(\"ctb\", \"cph\"), comment = \"selectize-plugin-a11y library\"), person(\"Denis\", \"Ineshin\", role = c(\"ctb\", \"cph\"), comment = \"ion.rangeSlider library\"), person(\"Sami\", \"Samhuri\", role = c(\"ctb\", \"cph\"), comment = \"Javascript strftime library\"), person(family = \"SpryMedia Limited\", role = c(\"ctb\", \"cph\"), comment = \"DataTables library\"), person(\"John\", \"Fraser\", role = c(\"ctb\", \"cph\"), comment = \"showdown.js library\"), person(\"John\", \"Gruber\", role = c(\"ctb\", \"cph\"), comment = \"showdown.js library\"), person(\"Ivan\", \"Sagalaev\", role = c(\"ctb\", \"cph\"), comment = \"highlight.js library\"), person(given = \"R Core Team\", role = c(\"ctb\", \"cph\"), comment = \"tar implementation from R\") )", - "Description": "Makes it incredibly easy to build interactive web applications with R. Automatic \"reactive\" binding between inputs and outputs and extensive prebuilt widgets make it possible to build beautiful, responsive, and powerful applications with minimal effort.", - "License": "GPL-3 | file LICENSE", - "Depends": [ - "R (>= 3.0.2)", - "methods" - ], - "Imports": [ - "utils", - "grDevices", - "httpuv (>= 1.5.2)", - "mime (>= 0.3)", - "jsonlite (>= 0.9.16)", - "xtable", - "fontawesome (>= 0.4.0)", - "htmltools (>= 0.5.4)", - "R6 (>= 2.0)", - "sourcetools", - "later (>= 1.0.0)", - "promises (>= 1.3.2)", - "tools", - "cli", - "rlang (>= 0.4.10)", - "fastmap (>= 1.1.1)", - "withr", - "commonmark (>= 1.7)", - "glue (>= 1.3.2)", - "bslib (>= 0.6.0)", - "cachem (>= 1.1.0)", - "lifecycle (>= 0.2.0)" - ], - "Suggests": [ - "coro (>= 1.1.0)", - "datasets", - "DT", - "Cairo (>= 1.5-5)", - "testthat (>= 3.2.1)", - "knitr (>= 1.6)", - "markdown", - "rmarkdown", - "ggplot2", - "reactlog (>= 1.0.0)", - "magrittr", - "yaml", - "mirai", - "future", - "dygraphs", - "ragg", - "showtext", - "sass", - "watcher" - ], - "URL": "https://shiny.posit.co/, https://github.com/rstudio/shiny", - "BugReports": "https://github.com/rstudio/shiny/issues", - "Collate": "'globals.R' 'app-state.R' 'app_template.R' 'bind-cache.R' 'bind-event.R' 'bookmark-state-local.R' 'bookmark-state.R' 'bootstrap-deprecated.R' 'bootstrap-layout.R' 'conditions.R' 'map.R' 'utils.R' 'bootstrap.R' 'busy-indicators-spinners.R' 'busy-indicators.R' 'cache-utils.R' 'deprecated.R' 'devmode.R' 'diagnose.R' 'extended-task.R' 'fileupload.R' 'graph.R' 'reactives.R' 'reactive-domains.R' 'history.R' 'hooks.R' 'html-deps.R' 'image-interact-opts.R' 'image-interact.R' 'imageutils.R' 'input-action.R' 'input-checkbox.R' 'input-checkboxgroup.R' 'input-date.R' 'input-daterange.R' 'input-file.R' 'input-numeric.R' 'input-password.R' 'input-radiobuttons.R' 'input-select.R' 'input-slider.R' 'input-submit.R' 'input-text.R' 'input-textarea.R' 'input-utils.R' 'insert-tab.R' 'insert-ui.R' 'jqueryui.R' 'knitr.R' 'middleware-shiny.R' 'middleware.R' 'timer.R' 'shiny.R' 'mock-session.R' 'modal.R' 'modules.R' 'notifications.R' 'priorityqueue.R' 'progress.R' 'react.R' 'reexports.R' 'render-cached-plot.R' 'render-plot.R' 'render-table.R' 'run-url.R' 'runapp.R' 'serializers.R' 'server-input-handlers.R' 'server-resource-paths.R' 'server.R' 'shiny-options.R' 'shiny-package.R' 'shinyapp.R' 'shinyui.R' 'shinywrappers.R' 'showcase.R' 'snapshot.R' 'staticimports.R' 'tar.R' 'test-export.R' 'test-server.R' 'test.R' 'update-input.R' 'utils-lang.R' 'version_bs_date_picker.R' 'version_ion_range_slider.R' 'version_jquery.R' 'version_jqueryui.R' 'version_selectize.R' 'version_strftime.R' 'viewer.R'", - "RoxygenNote": "7.3.2", - "Encoding": "UTF-8", - "Config/testthat/edition": "3", - "Config/Needs/check": "shinytest2", - "NeedsCompilation": "no", - "Author": "Winston Chang [aut, cre] (ORCID: ), Joe Cheng [aut], JJ Allaire [aut], Carson Sievert [aut] (ORCID: ), Barret Schloerke [aut] (ORCID: ), Yihui Xie [aut], Jeff Allen [aut], Jonathan McPherson [aut], Alan Dipert [aut], Barbara Borges [aut], Posit Software, PBC [cph, fnd], jQuery Foundation [cph] (jQuery library and jQuery UI library), jQuery contributors [ctb, cph] (jQuery library; authors listed in inst/www/shared/jquery-AUTHORS.txt), jQuery UI contributors [ctb, cph] (jQuery UI library; authors listed in inst/www/shared/jqueryui/AUTHORS.txt), Mark Otto [ctb] (Bootstrap library), Jacob Thornton [ctb] (Bootstrap library), Bootstrap contributors [ctb] (Bootstrap library), Twitter, Inc [cph] (Bootstrap library), Prem Nawaz Khan [ctb] (Bootstrap accessibility plugin), Victor Tsaran [ctb] (Bootstrap accessibility plugin), Dennis Lembree [ctb] (Bootstrap accessibility plugin), Srinivasu Chakravarthula [ctb] (Bootstrap accessibility plugin), Cathy O'Connor [ctb] (Bootstrap accessibility plugin), PayPal, Inc [cph] (Bootstrap accessibility plugin), Stefan Petre [ctb, cph] (Bootstrap-datepicker library), Andrew Rowls [ctb, cph] (Bootstrap-datepicker library), Brian Reavis [ctb, cph] (selectize.js library), Salmen Bejaoui [ctb, cph] (selectize-plugin-a11y library), Denis Ineshin [ctb, cph] (ion.rangeSlider library), Sami Samhuri [ctb, cph] (Javascript strftime library), SpryMedia Limited [ctb, cph] (DataTables library), John Fraser [ctb, cph] (showdown.js library), John Gruber [ctb, cph] (showdown.js library), Ivan Sagalaev [ctb, cph] (highlight.js library), R Core Team [ctb, cph] (tar implementation from R)", - "Maintainer": "Winston Chang ", - "Repository": "CRAN" - }, - "shinydashboard": { - "Package": "shinydashboard", - "Version": "0.7.3", - "Source": "Repository", - "Title": "Create Dashboards with 'Shiny'", - "Authors@R": "c( person(\"Winston\", \"Chang\", , \"winston@posit.co\", role = c(\"aut\", \"cre\")), person(\"Barbara\", \"Borges Ribeiro\", role = \"aut\"), person(, \"Posit Software, PBC\", role = \"cph\"), person(, \"Almasaeed Studio\", role = c(\"ctb\", \"cph\"), comment = \"AdminLTE theme for Bootstrap\"), person(, \"Adobe Systems Incorporated\", role = c(\"ctb\", \"cph\"), comment = \"Source Sans Pro font\") )", - "Description": "Create dashboards with 'Shiny'. This package provides a theme on top of 'Shiny', making it easy to create attractive dashboards.", - "License": "MIT + file LICENSE", - "URL": "https://rstudio.github.io/shinydashboard/, https://github.com/rstudio/shinydashboard", - "BugReports": "https://github.com/rstudio/shinydashboard/issues", - "Depends": [ - "R (>= 3.0)" - ], - "Imports": [ - "htmltools (>= 0.2.6)", - "promises", - "shiny (>= 1.0.0)", - "utils" - ], - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Winston Chang [aut, cre], Barbara Borges Ribeiro [aut], Posit Software, PBC [cph], Almasaeed Studio [ctb, cph] (AdminLTE theme for Bootstrap), Adobe Systems Incorporated [ctb, cph] (Source Sans Pro font)", - "Maintainer": "Winston Chang ", - "Repository": "CRAN" - }, - "snow": { - "Package": "snow", - "Version": "0.4-4", - "Source": "Repository", - "Title": "Simple Network of Workstations", - "Author": "Luke Tierney, A. J. Rossini, Na Li, H. Sevcikova", - "Description": "Support for simple parallel computing in R.", - "Maintainer": "Luke Tierney ", - "Suggests": [ - "rlecuyer" - ], - "Enhances": [ - "Rmpi" - ], - "License": "GPL", - "Depends": [ - "R (>= 2.13.1)", - "utils" - ], - "NeedsCompilation": "no", - "Repository": "CRAN" - }, - "sourcetools": { - "Package": "sourcetools", - "Version": "0.1.7-1", - "Source": "Repository", - "Type": "Package", - "Title": "Tools for Reading, Tokenizing and Parsing R Code", - "Author": "Kevin Ushey", - "Maintainer": "Kevin Ushey ", - "Description": "Tools for the reading and tokenization of R code. The 'sourcetools' package provides both an R and C++ interface for the tokenization of R code, and helpers for interacting with the tokenized representation of R code.", - "License": "MIT + file LICENSE", - "Depends": [ - "R (>= 3.0.2)" - ], - "Suggests": [ - "testthat" - ], - "RoxygenNote": "5.0.1", - "BugReports": "https://github.com/kevinushey/sourcetools/issues", - "Encoding": "UTF-8", - "NeedsCompilation": "yes", - "Repository": "RSPM" - }, - "stringi": { - "Package": "stringi", - "Version": "1.8.7", - "Source": "Repository", - "Date": "2025-03-27", - "Title": "Fast and Portable Character String Processing Facilities", - "Description": "A collection of character string/text/natural language processing tools for pattern searching (e.g., with 'Java'-like regular expressions or the 'Unicode' collation algorithm), random string generation, case mapping, string transliteration, concatenation, sorting, padding, wrapping, Unicode normalisation, date-time formatting and parsing, and many more. They are fast, consistent, convenient, and - thanks to 'ICU' (International Components for Unicode) - portable across all locales and platforms. Documentation about 'stringi' is provided via its website at and the paper by Gagolewski (2022, ).", - "URL": "https://stringi.gagolewski.com/, https://github.com/gagolews/stringi, https://icu.unicode.org/", - "BugReports": "https://github.com/gagolews/stringi/issues", - "SystemRequirements": "ICU4C (>= 61, optional)", - "Type": "Package", - "Depends": [ - "R (>= 3.4)" - ], - "Imports": [ - "tools", - "utils", - "stats" - ], - "Biarch": "TRUE", - "License": "file LICENSE", - "Authors@R": "c(person(given = \"Marek\", family = \"Gagolewski\", role = c(\"aut\", \"cre\", \"cph\"), email = \"marek@gagolewski.com\", comment = c(ORCID = \"0000-0003-0637-6028\")), person(given = \"Bartek\", family = \"Tartanus\", role = \"ctb\"), person(\"Unicode, Inc. and others\", role=\"ctb\", comment = \"ICU4C source code, Unicode Character Database\") )", - "RoxygenNote": "7.3.2", - "Encoding": "UTF-8", - "NeedsCompilation": "yes", - "Author": "Marek Gagolewski [aut, cre, cph] (), Bartek Tartanus [ctb], Unicode, Inc. and others [ctb] (ICU4C source code, Unicode Character Database)", - "Maintainer": "Marek Gagolewski ", - "License_is_FOSS": "yes", - "Repository": "CRAN" - }, - "stringr": { - "Package": "stringr", - "Version": "1.6.0", - "Source": "Repository", - "Title": "Simple, Consistent Wrappers for Common String Operations", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\", \"cph\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "A consistent, simple and easy to use set of wrappers around the fantastic 'stringi' package. All function and argument names (and positions) are consistent, all functions deal with \"NA\"'s and zero length vectors in the same way, and the output from one function is easy to feed into the input of another.", - "License": "MIT + file LICENSE", - "URL": "https://stringr.tidyverse.org, https://github.com/tidyverse/stringr", - "BugReports": "https://github.com/tidyverse/stringr/issues", - "Depends": [ - "R (>= 4.1.0)" - ], - "Imports": [ - "cli", - "glue (>= 1.6.1)", - "lifecycle (>= 1.0.3)", - "magrittr", - "rlang (>= 1.0.0)", - "stringi (>= 1.5.3)", - "vctrs (>= 0.4.0)" - ], - "Suggests": [ - "covr", - "dplyr", - "gt", - "htmltools", - "htmlwidgets", - "knitr", - "rmarkdown", - "testthat (>= 3.0.0)", - "tibble" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/potools/style": "explicit", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "LazyData": "true", - "RoxygenNote": "7.3.3", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre, cph], Posit Software, PBC [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "styler": { - "Package": "styler", - "Version": "1.11.0", - "Source": "Repository", - "Type": "Package", - "Title": "Non-Invasive Pretty Printing of R Code", - "Authors@R": "c(person(given = \"Kirill\", family = \"Müller\", role = \"aut\", email = \"kirill@cynkra.com\", comment = c(ORCID = \"0000-0002-1416-3412\")), person(given = \"Lorenz\", family = \"Walthert\", role = c(\"cre\", \"aut\"), email = \"lorenz.walthert@icloud.com\"), person(given = \"Indrajeet\", family = \"Patil\", role = \"aut\", email = \"patilindrajeet.science@gmail.com\", comment = c(ORCID = \"0000-0003-1995-6531\")))", - "Description": "Pretty-prints R code without changing the user's formatting intent.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/r-lib/styler, https://styler.r-lib.org", - "BugReports": "https://github.com/r-lib/styler/issues", - "Depends": [ - "R (>= 4.0.0)" - ], - "Imports": [ - "cli (>= 3.1.1)", - "magrittr (>= 2.0.0)", - "purrr (>= 1.0.2)", - "R.cache (>= 0.15.0)", - "rlang (>= 1.0.0)", - "rprojroot (>= 1.1)", - "tools", - "vctrs (>= 0.4.1)", - "withr (>= 2.3.0)" - ], - "Suggests": [ - "data.tree (>= 0.1.6)", - "digest", - "here", - "knitr", - "prettycode", - "rmarkdown", - "roxygen2", - "rstudioapi (>= 0.7)", - "tibble (>= 1.4.2)", - "testthat (>= 3.2.1)" - ], - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.3", - "Language": "en-US", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "true", - "Collate": "'addins.R' 'communicate.R' 'compat-dplyr.R' 'compat-tidyr.R' 'detect-alignment-utils.R' 'detect-alignment.R' 'environments.R' 'expr-is.R' 'indent.R' 'initialize.R' 'io.R' 'nest.R' 'nested-to-tree.R' 'options.R' 'parse.R' 'reindent.R' 'token-define.R' 'relevel.R' 'roxygen-examples-add-remove.R' 'roxygen-examples-find.R' 'roxygen-examples-parse.R' 'roxygen-examples.R' 'rules-indention.R' 'rules-line-breaks.R' 'rules-spaces.R' 'rules-tokens.R' 'serialize.R' 'set-assert-args.R' 'style-guides.R' 'styler-package.R' 'stylerignore.R' 'testing-mocks.R' 'testing-public-api.R' 'ui-caching.R' 'testing.R' 'token-create.R' 'transform-block.R' 'transform-code.R' 'transform-files.R' 'ui-styling.R' 'unindent.R' 'utils-cache.R' 'utils-files.R' 'utils-navigate-nest.R' 'utils-strings.R' 'utils.R' 'vertical.R' 'visit.R' 'zzz.R'", - "NeedsCompilation": "no", - "Author": "Kirill Müller [aut] (ORCID: ), Lorenz Walthert [cre, aut], Indrajeet Patil [aut] (ORCID: )", - "Maintainer": "Lorenz Walthert ", - "Repository": "CRAN" - }, - "sys": { - "Package": "sys", - "Version": "3.4.3", - "Source": "Repository", - "Type": "Package", - "Title": "Powerful and Reliable Tools for Running System Commands in R", - "Authors@R": "c(person(\"Jeroen\", \"Ooms\", role = c(\"aut\", \"cre\"), email = \"jeroenooms@gmail.com\", comment = c(ORCID = \"0000-0002-4035-0289\")), person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = \"ctb\"))", - "Description": "Drop-in replacements for the base system2() function with fine control and consistent behavior across platforms. Supports clean interruption, timeout, background tasks, and streaming STDIN / STDOUT / STDERR over binary or text connections. Arguments on Windows automatically get encoded and quoted to work on different locales.", - "License": "MIT + file LICENSE", - "URL": "https://jeroen.r-universe.dev/sys", - "BugReports": "https://github.com/jeroen/sys/issues", - "Encoding": "UTF-8", - "RoxygenNote": "7.1.1", - "Suggests": [ - "unix (>= 1.4)", - "spelling", - "testthat" - ], - "Language": "en-US", - "NeedsCompilation": "yes", - "Author": "Jeroen Ooms [aut, cre] (), Gábor Csárdi [ctb]", - "Maintainer": "Jeroen Ooms ", - "Repository": "CRAN" - }, - "systemfonts": { - "Package": "systemfonts", - "Version": "1.3.1", - "Source": "Repository", - "Type": "Package", - "Title": "System Native Font Finding", - "Authors@R": "c( person(\"Thomas Lin\", \"Pedersen\", , \"thomas.pedersen@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-5147-4711\")), person(\"Jeroen\", \"Ooms\", , \"jeroen@berkeley.edu\", role = \"aut\", comment = c(ORCID = \"0000-0002-4035-0289\")), person(\"Devon\", \"Govett\", role = \"aut\", comment = \"Author of font-manager\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Provides system native access to the font catalogue. As font handling varies between systems it is difficult to correctly locate installed fonts across different operating systems. The 'systemfonts' package provides bindings to the native libraries on Windows, macOS and Linux for finding font files that can then be used further by e.g. graphic devices. The main use is intended to be from compiled code but 'systemfonts' also provides access from R.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/r-lib/systemfonts, https://systemfonts.r-lib.org", - "BugReports": "https://github.com/r-lib/systemfonts/issues", - "Depends": [ - "R (>= 3.2.0)" - ], - "Imports": [ - "base64enc", - "grid", - "jsonlite", - "lifecycle", - "tools", - "utils" - ], - "Suggests": [ - "covr", - "farver", - "ggplot2", - "graphics", - "knitr", - "ragg", - "rmarkdown", - "svglite", - "testthat (>= 2.1.0)" - ], - "LinkingTo": [ - "cpp11 (>= 0.2.1)" - ], - "VignetteBuilder": "knitr", - "Config/build/compilation-database": "true", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/usethis/last-upkeep": "2025-04-23", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "SystemRequirements": "fontconfig, freetype2", - "NeedsCompilation": "yes", - "Author": "Thomas Lin Pedersen [aut, cre] (ORCID: ), Jeroen Ooms [aut] (ORCID: ), Devon Govett [aut] (Author of font-manager), Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Thomas Lin Pedersen ", - "Repository": "CRAN" - }, - "testit": { - "Package": "testit", - "Version": "0.13", - "Source": "Repository", - "Type": "Package", - "Title": "A Simple Package for Testing R Packages", - "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(\"Steven\", \"Mortimer\", role = \"ctb\", email=\"reportmort@gmail.com\") )", - "Description": "Provides two convenience functions assert() and test_pkg() to facilitate testing R packages.", - "License": "GPL-3", - "URL": "https://github.com/yihui/testit", - "BugReports": "https://github.com/yihui/testit/issues", - "Suggests": [ - "rstudioapi", - "covr" - ], - "RoxygenNote": "7.1.1", - "Encoding": "UTF-8", - "NeedsCompilation": "no", - "Author": "Yihui Xie [aut, cre] (), Steven Mortimer [ctb]", - "Maintainer": "Yihui Xie ", - "Repository": "RSPM" - }, - "testthat": { - "Package": "testthat", - "Version": "3.3.0", - "Source": "Repository", - "Title": "Unit Testing for R", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(\"R Core team\", role = \"ctb\", comment = \"Implementation of utils::recover()\") )", - "Description": "Software testing is important, but, in part because it is frustrating and boring, many of us avoid it. 'testthat' is a testing framework for R that is easy to learn and use, and integrates with your existing 'workflow'.", - "License": "MIT + file LICENSE", - "URL": "https://testthat.r-lib.org, https://github.com/r-lib/testthat", - "BugReports": "https://github.com/r-lib/testthat/issues", - "Depends": [ - "R (>= 4.1.0)" - ], - "Imports": [ - "brio (>= 1.1.5)", - "callr (>= 3.7.6)", - "cli (>= 3.6.5)", - "desc (>= 1.4.3)", - "evaluate (>= 1.0.4)", - "jsonlite (>= 2.0.0)", - "lifecycle (>= 1.0.4)", - "magrittr (>= 2.0.3)", - "methods", - "pkgload (>= 1.4.0)", - "praise (>= 1.0.0)", - "processx (>= 3.8.6)", - "ps (>= 1.9.1)", - "R6 (>= 2.6.1)", - "rlang (>= 1.1.6)", - "utils", - "waldo (>= 0.6.2)", - "withr (>= 3.0.2)" - ], - "Suggests": [ - "covr", - "curl (>= 0.9.5)", - "diffviewer (>= 0.1.0)", - "digest (>= 0.6.33)", - "gh", - "knitr", - "rmarkdown", - "rstudioapi", - "S7", - "shiny", - "usethis", - "vctrs (>= 0.1.0)", - "xml2" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "true", - "Config/testthat/start-first": "watcher, parallel*", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.3", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut, cre], Posit Software, PBC [cph, fnd], R Core team [ctb] (Implementation of utils::recover())", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "textshaping": { - "Package": "textshaping", - "Version": "1.0.4", - "Source": "Repository", - "Title": "Bindings to the 'HarfBuzz' and 'Fribidi' Libraries for Text Shaping", - "Authors@R": "c( person(\"Thomas Lin\", \"Pedersen\", , \"thomas.pedersen@posit.co\", role = c(\"cre\", \"aut\"), comment = c(ORCID = \"0000-0002-5147-4711\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Provides access to the text shaping functionality in the 'HarfBuzz' library and the bidirectional algorithm in the 'Fribidi' library. 'textshaping' is a low-level utility package mainly for graphic devices that expands upon the font tool-set provided by the 'systemfonts' package.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/r-lib/textshaping", - "BugReports": "https://github.com/r-lib/textshaping/issues", - "Depends": [ - "R (>= 3.2.0)" - ], - "Imports": [ - "lifecycle", - "stats", - "stringi", - "systemfonts (>= 1.3.0)", - "utils" - ], - "Suggests": [ - "covr", - "grDevices", - "grid", - "knitr", - "rmarkdown", - "testthat (>= 3.0.0)" - ], - "LinkingTo": [ - "cpp11 (>= 0.2.1)", - "systemfonts (>= 1.0.0)" - ], - "VignetteBuilder": "knitr", - "Config/build/compilation-database": "true", - "Config/testthat/edition": "3", - "Config/usethis/last-upkeep": "2025-04-23", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "SystemRequirements": "freetype2, harfbuzz, fribidi", - "NeedsCompilation": "yes", - "Author": "Thomas Lin Pedersen [cre, aut] (ORCID: ), Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Thomas Lin Pedersen ", - "Repository": "CRAN" - }, - "tibble": { - "Package": "tibble", - "Version": "3.3.0", - "Source": "Repository", - "Title": "Simple Data Frames", - "Authors@R": "c(person(given = \"Kirill\", family = \"M\\u00fcller\", role = c(\"aut\", \"cre\"), email = \"kirill@cynkra.com\", comment = c(ORCID = \"0000-0002-1416-3412\")), person(given = \"Hadley\", family = \"Wickham\", role = \"aut\", email = \"hadley@rstudio.com\"), person(given = \"Romain\", family = \"Francois\", role = \"ctb\", email = \"romain@r-enthusiasts.com\"), person(given = \"Jennifer\", family = \"Bryan\", role = \"ctb\", email = \"jenny@rstudio.com\"), person(given = \"RStudio\", role = c(\"cph\", \"fnd\")))", - "Description": "Provides a 'tbl_df' class (the 'tibble') with stricter checking and better formatting than the traditional data frame.", - "License": "MIT + file LICENSE", - "URL": "https://tibble.tidyverse.org/, https://github.com/tidyverse/tibble", - "BugReports": "https://github.com/tidyverse/tibble/issues", - "Depends": [ - "R (>= 3.4.0)" - ], - "Imports": [ - "cli", - "lifecycle (>= 1.0.0)", - "magrittr", - "methods", - "pillar (>= 1.8.1)", - "pkgconfig", - "rlang (>= 1.0.2)", - "utils", - "vctrs (>= 0.5.0)" - ], - "Suggests": [ - "bench", - "bit64", - "blob", - "brio", - "callr", - "DiagrammeR", - "dplyr", - "evaluate", - "formattable", - "ggplot2", - "here", - "hms", - "htmltools", - "knitr", - "lubridate", - "nycflights13", - "pkgload", - "purrr", - "rmarkdown", - "stringi", - "testthat (>= 3.0.2)", - "tidyr", - "withr" - ], - "VignetteBuilder": "knitr", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2.9000", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "true", - "Config/testthat/start-first": "vignette-formats, as_tibble, add, invariants", - "Config/autostyle/scope": "line_breaks", - "Config/autostyle/strict": "true", - "Config/autostyle/rmd": "false", - "Config/Needs/website": "tidyverse/tidytemplate", - "NeedsCompilation": "yes", - "Author": "Kirill Müller [aut, cre] (ORCID: ), Hadley Wickham [aut], Romain Francois [ctb], Jennifer Bryan [ctb], RStudio [cph, fnd]", - "Maintainer": "Kirill Müller ", - "Repository": "CRAN" - }, - "tictoc": { - "Package": "tictoc", - "Version": "1.2.1", - "Source": "Repository", - "Title": "Functions for Timing R Scripts, as Well as Implementations of \"Stack\" and \"StackList\" Structures", - "Author": "Sergei Izrailev", - "Maintainer": "Sergei Izrailev ", - "Description": "Code execution timing functions 'tic' and 'toc' that can be nested. One can record all timings while a complex script is running, and examine the values later. It is also possible to instrument the timing calls with custom callbacks. In addition, this package provides class 'Stack', implemented as a vector, and class 'StackList', which is a stack implemented as a list, both of which support operations 'push', 'pop', 'first_element', 'last_element' and 'clear'.", - "URL": "https://github.com/jabiru/tictoc", - "Depends": [ - "R (>= 2.15)", - "methods" - ], - "License": "Apache License (== 2.0) | file LICENSE", - "Copyright": "Copyright (C) Collective, Inc. | file inst/COPYRIGHTS", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.3", - "Suggests": [ - "testthat (>= 3.0.0)" - ], - "Config/testthat/edition": "3", - "NeedsCompilation": "no", - "Repository": "RSPM" - }, - "tidyr": { - "Package": "tidyr", - "Version": "1.3.1", - "Source": "Repository", - "Title": "Tidy Messy Data", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Davis\", \"Vaughan\", , \"davis@posit.co\", role = \"aut\"), person(\"Maximilian\", \"Girlich\", role = \"aut\"), person(\"Kevin\", \"Ushey\", , \"kevin@posit.co\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Tools to help to create tidy data, where each column is a variable, each row is an observation, and each cell contains a single value. 'tidyr' contains tools for changing the shape (pivoting) and hierarchy (nesting and 'unnesting') of a dataset, turning deeply nested lists into rectangular data frames ('rectangling'), and extracting values out of string columns. It also includes tools for working with missing values (both implicit and explicit).", - "License": "MIT + file LICENSE", - "URL": "https://tidyr.tidyverse.org, https://github.com/tidyverse/tidyr", - "BugReports": "https://github.com/tidyverse/tidyr/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "cli (>= 3.4.1)", - "dplyr (>= 1.0.10)", - "glue", - "lifecycle (>= 1.0.3)", - "magrittr", - "purrr (>= 1.0.1)", - "rlang (>= 1.1.1)", - "stringr (>= 1.5.0)", - "tibble (>= 2.1.1)", - "tidyselect (>= 1.2.0)", - "utils", - "vctrs (>= 0.5.2)" - ], - "Suggests": [ - "covr", - "data.table", - "knitr", - "readr", - "repurrrsive (>= 1.1.0)", - "rmarkdown", - "testthat (>= 3.0.0)" - ], - "LinkingTo": [ - "cpp11 (>= 0.4.0)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "LazyData": "true", - "RoxygenNote": "7.3.0", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut, cre], Davis Vaughan [aut], Maximilian Girlich [aut], Kevin Ushey [ctb], Posit Software, PBC [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "RSPM" - }, - "tidyselect": { - "Package": "tidyselect", - "Version": "1.2.1", - "Source": "Repository", - "Title": "Select from a Set of Strings", - "Authors@R": "c( person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = c(\"aut\", \"cre\")), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "A backend for the selecting functions of the 'tidyverse'. It makes it easy to implement select-like functions in your own packages in a way that is consistent with other 'tidyverse' interfaces for selection.", - "License": "MIT + file LICENSE", - "URL": "https://tidyselect.r-lib.org, https://github.com/r-lib/tidyselect", - "BugReports": "https://github.com/r-lib/tidyselect/issues", - "Depends": [ - "R (>= 3.4)" - ], - "Imports": [ - "cli (>= 3.3.0)", - "glue (>= 1.3.0)", - "lifecycle (>= 1.0.3)", - "rlang (>= 1.0.4)", - "vctrs (>= 0.5.2)", - "withr" - ], - "Suggests": [ - "covr", - "crayon", - "dplyr", - "knitr", - "magrittr", - "rmarkdown", - "stringr", - "testthat (>= 3.1.1)", - "tibble (>= 2.1.3)" - ], - "VignetteBuilder": "knitr", - "ByteCompile": "true", - "Config/testthat/edition": "3", - "Config/Needs/website": "tidyverse/tidytemplate", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.0.9000", - "NeedsCompilation": "yes", - "Author": "Lionel Henry [aut, cre], Hadley Wickham [aut], Posit Software, PBC [cph, fnd]", - "Maintainer": "Lionel Henry ", - "Repository": "RSPM" - }, - "tidyverse": { - "Package": "tidyverse", - "Version": "2.0.0", - "Source": "Repository", - "Title": "Easily Install and Load the 'Tidyverse'", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@rstudio.com\", role = c(\"aut\", \"cre\")), person(\"RStudio\", role = c(\"cph\", \"fnd\")) )", - "Description": "The 'tidyverse' is a set of packages that work in harmony because they share common data representations and 'API' design. This package is designed to make it easy to install and load multiple 'tidyverse' packages in a single step. Learn more about the 'tidyverse' at .", - "License": "MIT + file LICENSE", - "URL": "https://tidyverse.tidyverse.org, https://github.com/tidyverse/tidyverse", - "BugReports": "https://github.com/tidyverse/tidyverse/issues", - "Depends": [ - "R (>= 3.3)" - ], - "Imports": [ - "broom (>= 1.0.3)", - "conflicted (>= 1.2.0)", - "cli (>= 3.6.0)", - "dbplyr (>= 2.3.0)", - "dplyr (>= 1.1.0)", - "dtplyr (>= 1.2.2)", - "forcats (>= 1.0.0)", - "ggplot2 (>= 3.4.1)", - "googledrive (>= 2.0.0)", - "googlesheets4 (>= 1.0.1)", - "haven (>= 2.5.1)", - "hms (>= 1.1.2)", - "httr (>= 1.4.4)", - "jsonlite (>= 1.8.4)", - "lubridate (>= 1.9.2)", - "magrittr (>= 2.0.3)", - "modelr (>= 0.1.10)", - "pillar (>= 1.8.1)", - "purrr (>= 1.0.1)", - "ragg (>= 1.2.5)", - "readr (>= 2.1.4)", - "readxl (>= 1.4.2)", - "reprex (>= 2.0.2)", - "rlang (>= 1.0.6)", - "rstudioapi (>= 0.14)", - "rvest (>= 1.0.3)", - "stringr (>= 1.5.0)", - "tibble (>= 3.1.8)", - "tidyr (>= 1.3.0)", - "xml2 (>= 1.3.3)" - ], - "Suggests": [ - "covr (>= 3.6.1)", - "feather (>= 0.3.5)", - "glue (>= 1.6.2)", - "mockr (>= 0.2.0)", - "knitr (>= 1.41)", - "rmarkdown (>= 2.20)", - "testthat (>= 3.1.6)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.3", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre], RStudio [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "RSPM" - }, - "timechange": { - "Package": "timechange", - "Version": "0.3.0", - "Source": "Repository", - "Title": "Efficient Manipulation of Date-Times", - "Authors@R": "c(person(\"Vitalie\", \"Spinu\", email = \"spinuvit@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Google Inc.\", role = c(\"ctb\", \"cph\")))", - "Description": "Efficient routines for manipulation of date-time objects while accounting for time-zones and daylight saving times. The package includes utilities for updating of date-time components (year, month, day etc.), modification of time-zones, rounding of date-times, period addition and subtraction etc. Parts of the 'CCTZ' source code, released under the Apache 2.0 License, are included in this package. See for more details.", - "Depends": [ - "R (>= 3.3)" - ], - "License": "GPL (>= 3)", - "Encoding": "UTF-8", - "LinkingTo": [ - "cpp11 (>= 0.2.7)" - ], - "Suggests": [ - "testthat (>= 0.7.1.99)", - "knitr" - ], - "SystemRequirements": "A system with zoneinfo data (e.g. /usr/share/zoneinfo) as well as a recent-enough C++11 compiler (such as g++-4.8 or later). On Windows the zoneinfo included with R is used.", - "BugReports": "https://github.com/vspinu/timechange/issues", - "URL": "https://github.com/vspinu/timechange/", - "RoxygenNote": "7.2.1", - "NeedsCompilation": "yes", - "Author": "Vitalie Spinu [aut, cre], Google Inc. [ctb, cph]", - "Maintainer": "Vitalie Spinu ", - "Repository": "RSPM" - }, - "tinytex": { - "Package": "tinytex", - "Version": "0.57", - "Source": "Repository", - "Type": "Package", - "Title": "Helper Functions to Install and Maintain TeX Live, and Compile LaTeX Documents", - "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\", \"cph\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\")), person(given = \"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(\"Christophe\", \"Dervieux\", role = \"ctb\", comment = c(ORCID = \"0000-0003-4474-2498\")), person(\"Devon\", \"Ryan\", role = \"ctb\", email = \"dpryan79@gmail.com\", comment = c(ORCID = \"0000-0002-8549-0971\")), person(\"Ethan\", \"Heinzen\", role = \"ctb\"), person(\"Fernando\", \"Cagua\", role = \"ctb\"), person() )", - "Description": "Helper functions to install and maintain the 'LaTeX' distribution named 'TinyTeX' (), a lightweight, cross-platform, portable, and easy-to-maintain version of 'TeX Live'. This package also contains helper functions to compile 'LaTeX' documents, and install missing 'LaTeX' packages automatically.", - "Imports": [ - "xfun (>= 0.48)" - ], - "Suggests": [ - "testit", - "rstudioapi" - ], - "License": "MIT + file LICENSE", - "URL": "https://github.com/rstudio/tinytex", - "BugReports": "https://github.com/rstudio/tinytex/issues", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Yihui Xie [aut, cre, cph] (), Posit Software, PBC [cph, fnd], Christophe Dervieux [ctb] (), Devon Ryan [ctb] (), Ethan Heinzen [ctb], Fernando Cagua [ctb]", - "Maintainer": "Yihui Xie ", - "Repository": "CRAN" - }, - "tzdb": { - "Package": "tzdb", - "Version": "0.5.0", - "Source": "Repository", - "Title": "Time Zone Database Information", - "Authors@R": "c( person(\"Davis\", \"Vaughan\", , \"davis@posit.co\", role = c(\"aut\", \"cre\")), person(\"Howard\", \"Hinnant\", role = \"cph\", comment = \"Author of the included date library\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Provides an up-to-date copy of the Internet Assigned Numbers Authority (IANA) Time Zone Database. It is updated periodically to reflect changes made by political bodies to time zone boundaries, UTC offsets, and daylight saving time rules. Additionally, this package provides a C++ interface for working with the 'date' library. 'date' provides comprehensive support for working with dates and date-times, which this package exposes to make it easier for other R packages to utilize. Headers are provided for calendar specific calculations, along with a limited interface for time zone manipulations.", - "License": "MIT + file LICENSE", - "URL": "https://tzdb.r-lib.org, https://github.com/r-lib/tzdb", - "BugReports": "https://github.com/r-lib/tzdb/issues", - "Depends": [ - "R (>= 4.0.0)" - ], - "Suggests": [ - "covr", - "testthat (>= 3.0.0)" - ], - "LinkingTo": [ - "cpp11 (>= 0.5.2)" - ], - "Biarch": "yes", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "yes", - "Author": "Davis Vaughan [aut, cre], Howard Hinnant [cph] (Author of the included date library), Posit Software, PBC [cph, fnd]", - "Maintainer": "Davis Vaughan ", - "Repository": "CRAN" - }, - "urlchecker": { - "Package": "urlchecker", - "Version": "1.0.1", - "Source": "Repository", - "Title": "Run CRAN URL Checks from Older R Versions", - "Authors@R": "c( person(\"R Core team\", role = \"aut\", comment = \"The code in urltools.R adapted from the tools package\"), person(\"Jim\", \"Hester\", role = \"aut\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"RStudio\", role = c(\"cph\", \"fnd\")) )", - "Description": "Provide the URL checking tools available in R 4.1+ as a package for earlier versions of R. Also uses concurrent requests so can be much faster than the serial versions.", - "License": "GPL-3", - "URL": "https://github.com/r-lib/urlchecker", - "BugReports": "https://github.com/r-lib/urlchecker/issues", - "Depends": [ - "R (>= 3.3)" - ], - "Imports": [ - "cli", - "curl", - "tools", - "xml2" - ], - "Suggests": [ - "covr" - ], - "Encoding": "UTF-8", - "RoxygenNote": "7.1.2", - "NeedsCompilation": "no", - "Author": "R Core team [aut] (The code in urltools.R adapted from the tools package), Jim Hester [aut] (), Gábor Csárdi [aut, cre], RStudio [cph, fnd]", - "Maintainer": "Gábor Csárdi ", - "Repository": "RSPM" - }, - "usethis": { - "Package": "usethis", - "Version": "3.2.1", - "Source": "Repository", - "Title": "Automate Package and Project Setup", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Malcolm\", \"Barrett\", , \"malcolmbarrett@gmail.com\", role = \"aut\", comment = c(ORCID = \"0000-0003-0299-5825\")), person(\"Andy\", \"Teucher\", , \"andy.teucher@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0002-7840-692X\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Automate package and project setup tasks that are otherwise performed manually. This includes setting up unit testing, test coverage, continuous integration, Git, 'GitHub', licenses, 'Rcpp', 'RStudio' projects, and more.", - "License": "MIT + file LICENSE", - "URL": "https://usethis.r-lib.org, https://github.com/r-lib/usethis", - "BugReports": "https://github.com/r-lib/usethis/issues", - "Depends": [ - "R (>= 4.1)" - ], - "Imports": [ - "cli (>= 3.0.1)", - "clipr (>= 0.3.0)", - "crayon", - "curl (>= 2.7)", - "desc (>= 1.4.2)", - "fs (>= 1.3.0)", - "gert (>= 1.4.1)", - "gh (>= 1.2.1)", - "glue (>= 1.3.0)", - "jsonlite", - "lifecycle (>= 1.0.0)", - "purrr", - "rappdirs", - "rlang (>= 1.1.0)", - "rprojroot (>= 2.1.1)", - "rstudioapi", - "stats", - "tools", - "utils", - "whisker", - "withr (>= 2.3.0)", - "yaml" - ], - "Suggests": [ - "covr", - "knitr", - "magick", - "pkgload (>= 1.3.2.1)", - "quarto (>= 1.5.1)", - "rmarkdown", - "roxygen2 (>= 7.1.2)", - "spelling (>= 1.2)", - "testthat (>= 3.1.8)" - ], - "Config/Needs/website": "r-lib/asciicast, tidyverse/tidytemplate, xml2", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "TRUE", - "Config/testthat/start-first": "github-actions, release", - "Config/usethis/last-upkeep": "2025-04-22", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.3", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut] (ORCID: ), Jennifer Bryan [aut, cre] (ORCID: ), Malcolm Barrett [aut] (ORCID: ), Andy Teucher [aut] (ORCID: ), Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Jennifer Bryan ", - "Repository": "CRAN" - }, - "utf8": { - "Package": "utf8", - "Version": "1.2.6", - "Source": "Repository", - "Title": "Unicode Text Processing", - "Authors@R": "c(person(given = c(\"Patrick\", \"O.\"), family = \"Perry\", role = c(\"aut\", \"cph\")), person(given = \"Kirill\", family = \"M\\u00fcller\", role = \"cre\", email = \"kirill@cynkra.com\", comment = c(ORCID = \"0000-0002-1416-3412\")), person(given = \"Unicode, Inc.\", role = c(\"cph\", \"dtc\"), comment = \"Unicode Character Database\"))", - "Description": "Process and print 'UTF-8' encoded international text (Unicode). Input, validate, normalize, encode, format, and display.", - "License": "Apache License (== 2.0) | file LICENSE", - "URL": "https://krlmlr.github.io/utf8/, https://github.com/krlmlr/utf8", - "BugReports": "https://github.com/krlmlr/utf8/issues", - "Depends": [ - "R (>= 2.10)" - ], - "Suggests": [ - "cli", - "covr", - "knitr", - "rlang", - "rmarkdown", - "testthat (>= 3.0.0)", - "withr" - ], - "VignetteBuilder": "knitr, rmarkdown", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2.9000", - "NeedsCompilation": "yes", - "Author": "Patrick O. Perry [aut, cph], Kirill Müller [cre] (ORCID: ), Unicode, Inc. [cph, dtc] (Unicode Character Database)", - "Maintainer": "Kirill Müller ", - "Repository": "CRAN" - }, - "uuid": { - "Package": "uuid", - "Version": "1.2-1", - "Source": "Repository", - "Title": "Tools for Generating and Handling of UUIDs", - "Author": "Simon Urbanek [aut, cre, cph] (https://urbanek.org, ), Theodore Ts'o [aut, cph] (libuuid)", - "Maintainer": "Simon Urbanek ", - "Authors@R": "c(person(\"Simon\", \"Urbanek\", role=c(\"aut\",\"cre\",\"cph\"), email=\"Simon.Urbanek@r-project.org\", comment=c(\"https://urbanek.org\", ORCID=\"0000-0003-2297-1732\")), person(\"Theodore\",\"Ts'o\", email=\"tytso@thunk.org\", role=c(\"aut\",\"cph\"), comment=\"libuuid\"))", - "Depends": [ - "R (>= 2.9.0)" - ], - "Description": "Tools for generating and handling of UUIDs (Universally Unique Identifiers).", - "License": "MIT + file LICENSE", - "URL": "https://www.rforge.net/uuid", - "BugReports": "https://github.com/s-u/uuid/issues", - "NeedsCompilation": "yes", - "Repository": "CRAN" - }, - "vctrs": { - "Package": "vctrs", - "Version": "0.6.5", - "Source": "Repository", - "Title": "Vector Helpers", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = \"aut\"), person(\"Davis\", \"Vaughan\", , \"davis@posit.co\", role = c(\"aut\", \"cre\")), person(\"data.table team\", role = \"cph\", comment = \"Radix sort based on data.table's forder() and their contribution to R's order()\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Defines new notions of prototype and size that are used to provide tools for consistent and well-founded type-coercion and size-recycling, and are in turn connected to ideas of type- and size-stability useful for analysing function interfaces.", - "License": "MIT + file LICENSE", - "URL": "https://vctrs.r-lib.org/, https://github.com/r-lib/vctrs", - "BugReports": "https://github.com/r-lib/vctrs/issues", - "Depends": [ - "R (>= 3.5.0)" - ], - "Imports": [ - "cli (>= 3.4.0)", - "glue", - "lifecycle (>= 1.0.3)", - "rlang (>= 1.1.0)" - ], - "Suggests": [ - "bit64", - "covr", - "crayon", - "dplyr (>= 0.8.5)", - "generics", - "knitr", - "pillar (>= 1.4.4)", - "pkgdown (>= 2.0.1)", - "rmarkdown", - "testthat (>= 3.0.0)", - "tibble (>= 3.1.3)", - "waldo (>= 0.2.0)", - "withr", - "xml2", - "zeallot" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "Language": "en-GB", - "RoxygenNote": "7.2.3", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut], Lionel Henry [aut], Davis Vaughan [aut, cre], data.table team [cph] (Radix sort based on data.table's forder() and their contribution to R's order()), Posit Software, PBC [cph, fnd]", - "Maintainer": "Davis Vaughan ", - "Repository": "RSPM" - }, - "viridisLite": { - "Package": "viridisLite", - "Version": "0.4.2", - "Source": "Repository", - "Type": "Package", - "Title": "Colorblind-Friendly Color Maps (Lite Version)", - "Date": "2023-05-02", - "Authors@R": "c( person(\"Simon\", \"Garnier\", email = \"garnier@njit.edu\", role = c(\"aut\", \"cre\")), person(\"Noam\", \"Ross\", email = \"noam.ross@gmail.com\", role = c(\"ctb\", \"cph\")), person(\"Bob\", \"Rudis\", email = \"bob@rud.is\", role = c(\"ctb\", \"cph\")), person(\"Marco\", \"Sciaini\", email = \"sciaini.marco@gmail.com\", role = c(\"ctb\", \"cph\")), person(\"Antônio Pedro\", \"Camargo\", role = c(\"ctb\", \"cph\")), person(\"Cédric\", \"Scherer\", email = \"scherer@izw-berlin.de\", role = c(\"ctb\", \"cph\")) )", - "Maintainer": "Simon Garnier ", - "Description": "Color maps designed to improve graph readability for readers with common forms of color blindness and/or color vision deficiency. The color maps are also perceptually-uniform, both in regular form and also when converted to black-and-white for printing. This is the 'lite' version of the 'viridis' package that also contains 'ggplot2' bindings for discrete and continuous color and fill scales and can be found at .", - "License": "MIT + file LICENSE", - "Encoding": "UTF-8", - "Depends": [ - "R (>= 2.10)" - ], - "Suggests": [ - "hexbin (>= 1.27.0)", - "ggplot2 (>= 1.0.1)", - "testthat", - "covr" - ], - "URL": "https://sjmgarnier.github.io/viridisLite/, https://github.com/sjmgarnier/viridisLite/", - "BugReports": "https://github.com/sjmgarnier/viridisLite/issues/", - "RoxygenNote": "7.2.3", - "NeedsCompilation": "no", - "Author": "Simon Garnier [aut, cre], Noam Ross [ctb, cph], Bob Rudis [ctb, cph], Marco Sciaini [ctb, cph], Antônio Pedro Camargo [ctb, cph], Cédric Scherer [ctb, cph]", - "Repository": "RSPM" - }, - "vroom": { - "Package": "vroom", - "Version": "1.6.6", - "Source": "Repository", - "Title": "Read and Write Rectangular Text Data Quickly", - "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\", comment = c(ORCID = \"0000-0002-2739-7082\")), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\", comment = c(ORCID = \"0000-0003-4757-117X\")), person(\"Jennifer\", \"Bryan\", , \"jenny@posit.co\", role = c(\"aut\", \"cre\"), comment = c(ORCID = \"0000-0002-6983-2759\")), person(\"Shelby\", \"Bearrows\", role = \"ctb\"), person(\"https://github.com/mandreyel/\", role = \"cph\", comment = \"mio library\"), person(\"Jukka\", \"Jylänki\", role = \"cph\", comment = \"grisu3 implementation\"), person(\"Mikkel\", \"Jørgensen\", role = \"cph\", comment = \"grisu3 implementation\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "The goal of 'vroom' is to read and write data (like 'csv', 'tsv' and 'fwf') quickly. When reading it uses a quick initial indexing step, then reads the values lazily , so only the data you actually use needs to be read. The writer formats the data in parallel and writes to disk asynchronously from formatting.", - "License": "MIT + file LICENSE", - "URL": "https://vroom.r-lib.org, https://github.com/tidyverse/vroom", - "BugReports": "https://github.com/tidyverse/vroom/issues", - "Depends": [ - "R (>= 3.6)" - ], - "Imports": [ - "bit64", - "cli (>= 3.2.0)", - "crayon", - "glue", - "hms", - "lifecycle (>= 1.0.3)", - "methods", - "rlang (>= 0.4.2)", - "stats", - "tibble (>= 2.0.0)", - "tidyselect", - "tzdb (>= 0.1.1)", - "vctrs (>= 0.2.0)", - "withr" - ], - "Suggests": [ - "archive", - "bench (>= 1.1.0)", - "covr", - "curl", - "dplyr", - "forcats", - "fs", - "ggplot2", - "knitr", - "patchwork", - "prettyunits", - "purrr", - "rmarkdown", - "rstudioapi", - "scales", - "spelling", - "testthat (>= 2.1.0)", - "tidyr", - "utils", - "waldo", - "xml2" - ], - "LinkingTo": [ - "cpp11 (>= 0.2.0)", - "progress (>= 1.2.3)", - "tzdb (>= 0.1.1)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "nycflights13, tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/testthat/parallel": "false", - "Copyright": "file COPYRIGHTS", - "Encoding": "UTF-8", - "Language": "en-US", - "RoxygenNote": "7.3.3", - "NeedsCompilation": "yes", - "Author": "Jim Hester [aut] (ORCID: ), Hadley Wickham [aut] (ORCID: ), Jennifer Bryan [aut, cre] (ORCID: ), Shelby Bearrows [ctb], https://github.com/mandreyel/ [cph] (mio library), Jukka Jylänki [cph] (grisu3 implementation), Mikkel Jørgensen [cph] (grisu3 implementation), Posit Software, PBC [cph, fnd]", - "Maintainer": "Jennifer Bryan ", - "Repository": "CRAN" - }, - "waldo": { - "Package": "waldo", - "Version": "0.6.2", - "Source": "Repository", - "Title": "Find Differences Between R Objects", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Compare complex R objects and reveal the key differences. Designed particularly for use in testing packages where being able to quickly isolate key differences makes understanding test failures much easier.", - "License": "MIT + file LICENSE", - "URL": "https://waldo.r-lib.org, https://github.com/r-lib/waldo", - "BugReports": "https://github.com/r-lib/waldo/issues", - "Depends": [ - "R (>= 4.0)" - ], - "Imports": [ - "cli", - "diffobj (>= 0.3.4)", - "glue", - "methods", - "rlang (>= 1.1.0)" - ], - "Suggests": [ - "bit64", - "R6", - "S7", - "testthat (>= 3.0.0)", - "withr", - "xml2" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "NeedsCompilation": "no", - "Author": "Hadley Wickham [aut, cre], Posit Software, PBC [cph, fnd]", - "Maintainer": "Hadley Wickham ", - "Repository": "CRAN" - }, - "whisker": { - "Package": "whisker", - "Version": "0.4.1", - "Source": "Repository", - "Maintainer": "Edwin de Jonge ", - "License": "GPL-3", - "Title": "{{mustache}} for R, Logicless Templating", - "Type": "Package", - "LazyLoad": "yes", - "Author": "Edwin de Jonge", - "Description": "Implements 'Mustache' logicless templating.", - "URL": "https://github.com/edwindj/whisker", - "Suggests": [ - "markdown" - ], - "RoxygenNote": "6.1.1", - "NeedsCompilation": "no", - "Repository": "CRAN" - }, - "withr": { - "Package": "withr", - "Version": "3.0.2", - "Source": "Repository", - "Title": "Run Code 'With' Temporarily Modified Global State", - "Authors@R": "c( person(\"Jim\", \"Hester\", role = \"aut\"), person(\"Lionel\", \"Henry\", , \"lionel@posit.co\", role = c(\"aut\", \"cre\")), person(\"Kirill\", \"Müller\", , \"krlmlr+r@mailbox.org\", role = \"aut\"), person(\"Kevin\", \"Ushey\", , \"kevinushey@gmail.com\", role = \"aut\"), person(\"Hadley\", \"Wickham\", , \"hadley@posit.co\", role = \"aut\"), person(\"Winston\", \"Chang\", role = \"aut\"), person(\"Jennifer\", \"Bryan\", role = \"ctb\"), person(\"Richard\", \"Cotton\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "A set of functions to run code 'with' safely and temporarily modified global state. Many of these functions were originally a part of the 'devtools' package, this provides a simple package with limited dependencies to provide access to these functions.", - "License": "MIT + file LICENSE", - "URL": "https://withr.r-lib.org, https://github.com/r-lib/withr#readme", - "BugReports": "https://github.com/r-lib/withr/issues", - "Depends": [ - "R (>= 3.6.0)" - ], - "Imports": [ - "graphics", - "grDevices" - ], - "Suggests": [ - "callr", - "DBI", - "knitr", - "methods", - "rlang", - "rmarkdown (>= 2.12)", - "RSQLite", - "testthat (>= 3.0.0)" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2", - "Collate": "'aaa.R' 'collate.R' 'connection.R' 'db.R' 'defer-exit.R' 'standalone-defer.R' 'defer.R' 'devices.R' 'local_.R' 'with_.R' 'dir.R' 'env.R' 'file.R' 'language.R' 'libpaths.R' 'locale.R' 'makevars.R' 'namespace.R' 'options.R' 'par.R' 'path.R' 'rng.R' 'seed.R' 'wrap.R' 'sink.R' 'tempfile.R' 'timezone.R' 'torture.R' 'utils.R' 'with.R'", - "NeedsCompilation": "no", - "Author": "Jim Hester [aut], Lionel Henry [aut, cre], Kirill Müller [aut], Kevin Ushey [aut], Hadley Wickham [aut], Winston Chang [aut], Jennifer Bryan [ctb], Richard Cotton [ctb], Posit Software, PBC [cph, fnd]", - "Maintainer": "Lionel Henry ", - "Repository": "CRAN" - }, - "xfun": { - "Package": "xfun", - "Version": "0.54", - "Source": "Repository", - "Type": "Package", - "Title": "Supporting Functions for Packages Maintained by 'Yihui Xie'", - "Authors@R": "c( person(\"Yihui\", \"Xie\", role = c(\"aut\", \"cre\", \"cph\"), email = \"xie@yihui.name\", comment = c(ORCID = \"0000-0003-0645-5666\", URL = \"https://yihui.org\")), person(\"Wush\", \"Wu\", role = \"ctb\"), person(\"Daijiang\", \"Li\", role = \"ctb\"), person(\"Xianying\", \"Tan\", role = \"ctb\"), person(\"Salim\", \"Brüggemann\", role = \"ctb\", email = \"salim-b@pm.me\", comment = c(ORCID = \"0000-0002-5329-5987\")), person(\"Christophe\", \"Dervieux\", role = \"ctb\"), person() )", - "Description": "Miscellaneous functions commonly used in other packages maintained by 'Yihui Xie'.", - "Depends": [ - "R (>= 3.2.0)" - ], - "Imports": [ - "grDevices", - "stats", - "tools" - ], - "Suggests": [ - "testit", - "parallel", - "codetools", - "methods", - "rstudioapi", - "tinytex (>= 0.30)", - "mime", - "litedown (>= 0.6)", - "commonmark", - "knitr (>= 1.50)", - "remotes", - "pak", - "curl", - "xml2", - "jsonlite", - "magick", - "yaml", - "data.table", - "qs" - ], - "License": "MIT + file LICENSE", - "URL": "https://github.com/yihui/xfun", - "BugReports": "https://github.com/yihui/xfun/issues", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.3", - "VignetteBuilder": "litedown", - "NeedsCompilation": "yes", - "Author": "Yihui Xie [aut, cre, cph] (ORCID: , URL: https://yihui.org), Wush Wu [ctb], Daijiang Li [ctb], Xianying Tan [ctb], Salim Brüggemann [ctb] (ORCID: ), Christophe Dervieux [ctb]", - "Maintainer": "Yihui Xie ", - "Repository": "CRAN" - }, - "xml2": { - "Package": "xml2", - "Version": "1.4.1", - "Source": "Repository", - "Title": "Parse XML", - "Authors@R": "c( person(\"Hadley\", \"Wickham\", role = \"aut\"), person(\"Jim\", \"Hester\", role = \"aut\"), person(\"Jeroen\", \"Ooms\", email = \"jeroenooms@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")), person(\"R Foundation\", role = \"ctb\", comment = \"Copy of R-project homepage cached as example\") )", - "Description": "Bindings to 'libxml2' for working with XML data using a simple, consistent interface based on 'XPath' expressions. Also supports XML schema validation; for 'XSLT' transformations see the 'xslt' package.", - "License": "MIT + file LICENSE", - "URL": "https://xml2.r-lib.org, https://r-lib.r-universe.dev/xml2", - "BugReports": "https://github.com/r-lib/xml2/issues", - "Depends": [ - "R (>= 3.6.0)" - ], - "Imports": [ - "cli", - "methods", - "rlang (>= 1.1.0)" - ], - "Suggests": [ - "covr", - "curl", - "httr", - "knitr", - "mockery", - "rmarkdown", - "testthat (>= 3.2.0)", - "xslt" - ], - "VignetteBuilder": "knitr", - "Config/Needs/website": "tidyverse/tidytemplate", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.3", - "SystemRequirements": "libxml2: libxml2-dev (deb), libxml2-devel (rpm)", - "Collate": "'S4.R' 'as_list.R' 'xml_parse.R' 'as_xml_document.R' 'classes.R' 'format.R' 'import-standalone-obj-type.R' 'import-standalone-purrr.R' 'import-standalone-types-check.R' 'init.R' 'nodeset_apply.R' 'paths.R' 'utils.R' 'xml2-package.R' 'xml_attr.R' 'xml_children.R' 'xml_document.R' 'xml_find.R' 'xml_missing.R' 'xml_modify.R' 'xml_name.R' 'xml_namespaces.R' 'xml_node.R' 'xml_nodeset.R' 'xml_path.R' 'xml_schema.R' 'xml_serialize.R' 'xml_structure.R' 'xml_text.R' 'xml_type.R' 'xml_url.R' 'xml_write.R' 'zzz.R'", - "Config/testthat/edition": "3", - "NeedsCompilation": "yes", - "Author": "Hadley Wickham [aut], Jim Hester [aut], Jeroen Ooms [aut, cre], Posit Software, PBC [cph, fnd], R Foundation [ctb] (Copy of R-project homepage cached as example)", - "Maintainer": "Jeroen Ooms ", - "Repository": "CRAN" - }, - "xopen": { - "Package": "xopen", - "Version": "1.0.1", - "Source": "Repository", - "Title": "Open System Files, 'URLs', Anything", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Fathi\", \"Boudra\", role = \"aut\"), person(\"Rex\", \"Dieter\", role = \"aut\"), person(\"Kevin\", \"Krammer\", role = \"aut\"), person(\"Jeremy\", \"White\", role = \"aut\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\")) )", - "Description": "Cross platform solution to open files, directories or 'URLs' with their associated programs.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/r-lib/xopen#readme, https://r-lib.github.io/xopen/", - "BugReports": "https://github.com/r-lib/xopen/issues", - "Depends": [ - "R (>= 3.1)" - ], - "Imports": [ - "processx" - ], - "Suggests": [ - "ps", - "testthat (>= 3.0.0)" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Encoding": "UTF-8", - "RoxygenNote": "7.2.3", - "NeedsCompilation": "no", - "Author": "Gábor Csárdi [aut, cre], Fathi Boudra [aut], Rex Dieter [aut], Kevin Krammer [aut], Jeremy White [aut], Posit Software, PBC [cph, fnd]", - "Maintainer": "Gábor Csárdi ", - "Repository": "RSPM" - }, - "xtable": { - "Package": "xtable", - "Version": "1.8-4", - "Source": "Repository", - "Date": "2019-04-08", - "Title": "Export Tables to LaTeX or HTML", - "Authors@R": "c(person(\"David B.\", \"Dahl\", role=\"aut\"), person(\"David\", \"Scott\", role=c(\"aut\",\"cre\"), email=\"d.scott@auckland.ac.nz\"), person(\"Charles\", \"Roosen\", role=\"aut\"), person(\"Arni\", \"Magnusson\", role=\"aut\"), person(\"Jonathan\", \"Swinton\", role=\"aut\"), person(\"Ajay\", \"Shah\", role=\"ctb\"), person(\"Arne\", \"Henningsen\", role=\"ctb\"), person(\"Benno\", \"Puetz\", role=\"ctb\"), person(\"Bernhard\", \"Pfaff\", role=\"ctb\"), person(\"Claudio\", \"Agostinelli\", role=\"ctb\"), person(\"Claudius\", \"Loehnert\", role=\"ctb\"), person(\"David\", \"Mitchell\", role=\"ctb\"), person(\"David\", \"Whiting\", role=\"ctb\"), person(\"Fernando da\", \"Rosa\", role=\"ctb\"), person(\"Guido\", \"Gay\", role=\"ctb\"), person(\"Guido\", \"Schulz\", role=\"ctb\"), person(\"Ian\", \"Fellows\", role=\"ctb\"), person(\"Jeff\", \"Laake\", role=\"ctb\"), person(\"John\", \"Walker\", role=\"ctb\"), person(\"Jun\", \"Yan\", role=\"ctb\"), person(\"Liviu\", \"Andronic\", role=\"ctb\"), person(\"Markus\", \"Loecher\", role=\"ctb\"), person(\"Martin\", \"Gubri\", role=\"ctb\"), person(\"Matthieu\", \"Stigler\", role=\"ctb\"), person(\"Robert\", \"Castelo\", role=\"ctb\"), person(\"Seth\", \"Falcon\", role=\"ctb\"), person(\"Stefan\", \"Edwards\", role=\"ctb\"), person(\"Sven\", \"Garbade\", role=\"ctb\"), person(\"Uwe\", \"Ligges\", role=\"ctb\"))", - "Maintainer": "David Scott ", - "Imports": [ - "stats", - "utils" - ], - "Suggests": [ - "knitr", - "plm", - "zoo", - "survival" - ], - "VignetteBuilder": "knitr", - "Description": "Coerce data to LaTeX and HTML tables.", - "URL": "http://xtable.r-forge.r-project.org/", - "Depends": [ - "R (>= 2.10.0)" - ], - "License": "GPL (>= 2)", - "Repository": "CRAN", - "NeedsCompilation": "no", - "Author": "David B. Dahl [aut], David Scott [aut, cre], Charles Roosen [aut], Arni Magnusson [aut], Jonathan Swinton [aut], Ajay Shah [ctb], Arne Henningsen [ctb], Benno Puetz [ctb], Bernhard Pfaff [ctb], Claudio Agostinelli [ctb], Claudius Loehnert [ctb], David Mitchell [ctb], David Whiting [ctb], Fernando da Rosa [ctb], Guido Gay [ctb], Guido Schulz [ctb], Ian Fellows [ctb], Jeff Laake [ctb], John Walker [ctb], Jun Yan [ctb], Liviu Andronic [ctb], Markus Loecher [ctb], Martin Gubri [ctb], Matthieu Stigler [ctb], Robert Castelo [ctb], Seth Falcon [ctb], Stefan Edwards [ctb], Sven Garbade [ctb], Uwe Ligges [ctb]" - }, - "yaml": { - "Package": "yaml", - "Version": "2.3.10", - "Source": "Repository", - "Type": "Package", - "Title": "Methods to Convert R Data to YAML and Back", - "Date": "2024-07-22", - "Suggests": [ - "RUnit" - ], - "Author": "Shawn P Garbett [aut], Jeremy Stephens [aut, cre], Kirill Simonov [aut], Yihui Xie [ctb], Zhuoer Dong [ctb], Hadley Wickham [ctb], Jeffrey Horner [ctb], reikoch [ctb], Will Beasley [ctb], Brendan O'Connor [ctb], Gregory R. Warnes [ctb], Michael Quinn [ctb], Zhian N. Kamvar [ctb], Charlie Gao [ctb]", - "Maintainer": "Shawn Garbett ", - "License": "BSD_3_clause + file LICENSE", - "Description": "Implements the 'libyaml' 'YAML' 1.1 parser and emitter () for R.", - "URL": "https://github.com/vubiostat/r-yaml/", - "BugReports": "https://github.com/vubiostat/r-yaml/issues", - "NeedsCompilation": "yes", - "Repository": "CRAN" - }, - "zip": { - "Package": "zip", - "Version": "2.3.3", - "Source": "Repository", - "Title": "Cross-Platform 'zip' Compression", - "Authors@R": "c( person(\"Gábor\", \"Csárdi\", , \"csardi.gabor@gmail.com\", role = c(\"aut\", \"cre\")), person(\"Kuba\", \"Podgórski\", role = \"ctb\"), person(\"Rich\", \"Geldreich\", role = \"ctb\"), person(\"Posit Software, PBC\", role = c(\"cph\", \"fnd\"), comment = c(ROR = \"03wc8by49\")) )", - "Description": "Cross-Platform 'zip' Compression Library. A replacement for the 'zip' function, that does not require any additional external tools on any platform.", - "License": "MIT + file LICENSE", - "URL": "https://github.com/r-lib/zip, https://r-lib.github.io/zip/", - "BugReports": "https://github.com/r-lib/zip/issues", - "Suggests": [ - "covr", - "pillar", - "processx", - "R6", - "testthat", - "withr" - ], - "Config/Needs/website": "tidyverse/tidytemplate", - "Config/testthat/edition": "3", - "Config/usethis/last-upkeep": "2025-05-07", - "Encoding": "UTF-8", - "RoxygenNote": "7.3.2.9000", - "NeedsCompilation": "yes", - "Author": "Gábor Csárdi [aut, cre], Kuba Podgórski [ctb], Rich Geldreich [ctb], Posit Software, PBC [cph, fnd] (ROR: )", - "Maintainer": "Gábor Csárdi ", - "Repository": "CRAN" - }, - "zoo": { - "Package": "zoo", - "Version": "1.8-14", - "Source": "Repository", - "Date": "2025-04-09", - "Title": "S3 Infrastructure for Regular and Irregular Time Series (Z's Ordered Observations)", - "Authors@R": "c(person(given = \"Achim\", family = \"Zeileis\", role = c(\"aut\", \"cre\"), email = \"Achim.Zeileis@R-project.org\", comment = c(ORCID = \"0000-0003-0918-3766\")), person(given = \"Gabor\", family = \"Grothendieck\", role = \"aut\", email = \"ggrothendieck@gmail.com\"), person(given = c(\"Jeffrey\", \"A.\"), family = \"Ryan\", role = \"aut\", email = \"jeff.a.ryan@gmail.com\"), person(given = c(\"Joshua\", \"M.\"), family = \"Ulrich\", role = \"ctb\", email = \"josh.m.ulrich@gmail.com\"), person(given = \"Felix\", family = \"Andrews\", role = \"ctb\", email = \"felix@nfrac.org\"))", - "Description": "An S3 class with methods for totally ordered indexed observations. It is particularly aimed at irregular time series of numeric vectors/matrices and factors. zoo's key design goals are independence of a particular index/date/time class and consistency with ts and base R by providing methods to extend standard generics.", - "Depends": [ - "R (>= 3.1.0)", - "stats" - ], - "Suggests": [ - "AER", - "coda", - "chron", - "ggplot2 (>= 3.5.0)", - "mondate", - "scales", - "stinepack", - "strucchange", - "timeDate", - "timeSeries", - "tinyplot", - "tis", - "tseries", - "xts" - ], - "Imports": [ - "utils", - "graphics", - "grDevices", - "lattice (>= 0.20-27)" - ], - "License": "GPL-2 | GPL-3", - "URL": "https://zoo.R-Forge.R-project.org/", - "NeedsCompilation": "yes", - "Author": "Achim Zeileis [aut, cre] (), Gabor Grothendieck [aut], Jeffrey A. Ryan [aut], Joshua M. Ulrich [ctb], Felix Andrews [ctb]", - "Maintainer": "Achim Zeileis ", - "Repository": "CRAN" - } - } -} diff --git a/renv/.gitignore b/renv/.gitignore deleted file mode 100644 index 0ec0cbb..0000000 --- a/renv/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -library/ -local/ -cellar/ -lock/ -python/ -sandbox/ -staging/ diff --git a/renv/activate.R b/renv/activate.R deleted file mode 100644 index 2753ae5..0000000 --- a/renv/activate.R +++ /dev/null @@ -1,1334 +0,0 @@ - -local({ - - # the requested version of renv - version <- "1.1.5" - attr(version, "sha") <- NULL - - # the project directory - project <- Sys.getenv("RENV_PROJECT") - if (!nzchar(project)) - project <- getwd() - - # use start-up diagnostics if enabled - diagnostics <- Sys.getenv("RENV_STARTUP_DIAGNOSTICS", unset = "FALSE") - if (diagnostics) { - start <- Sys.time() - profile <- tempfile("renv-startup-", fileext = ".Rprof") - utils::Rprof(profile) - on.exit({ - utils::Rprof(NULL) - elapsed <- signif(difftime(Sys.time(), start, units = "auto"), digits = 2L) - writeLines(sprintf("- renv took %s to run the autoloader.", format(elapsed))) - writeLines(sprintf("- Profile: %s", profile)) - print(utils::summaryRprof(profile)) - }, add = TRUE) - } - - # figure out whether the autoloader is enabled - enabled <- local({ - - # first, check config option - override <- getOption("renv.config.autoloader.enabled") - if (!is.null(override)) - return(override) - - # if we're being run in a context where R_LIBS is already set, - # don't load -- presumably we're being run as a sub-process and - # the parent process has already set up library paths for us - rcmd <- Sys.getenv("R_CMD", unset = NA) - rlibs <- Sys.getenv("R_LIBS", unset = NA) - if (!is.na(rlibs) && !is.na(rcmd)) - return(FALSE) - - # next, check environment variables - # prefer using the configuration one in the future - envvars <- c( - "RENV_CONFIG_AUTOLOADER_ENABLED", - "RENV_AUTOLOADER_ENABLED", - "RENV_ACTIVATE_PROJECT" - ) - - for (envvar in envvars) { - envval <- Sys.getenv(envvar, unset = NA) - if (!is.na(envval)) - return(tolower(envval) %in% c("true", "t", "1")) - } - - # enable by default - TRUE - - }) - - # bail if we're not enabled - if (!enabled) { - - # if we're not enabled, we might still need to manually load - # the user profile here - profile <- Sys.getenv("R_PROFILE_USER", unset = "~/.Rprofile") - if (file.exists(profile)) { - cfg <- Sys.getenv("RENV_CONFIG_USER_PROFILE", unset = "TRUE") - if (tolower(cfg) %in% c("true", "t", "1")) - sys.source(profile, envir = globalenv()) - } - - return(FALSE) - - } - - # avoid recursion - if (identical(getOption("renv.autoloader.running"), TRUE)) { - warning("ignoring recursive attempt to run renv autoloader") - return(invisible(TRUE)) - } - - # signal that we're loading renv during R startup - options(renv.autoloader.running = TRUE) - on.exit(options(renv.autoloader.running = NULL), add = TRUE) - - # signal that we've consented to use renv - options(renv.consent = TRUE) - - # load the 'utils' package eagerly -- this ensures that renv shims, which - # mask 'utils' packages, will come first on the search path - library(utils, lib.loc = .Library) - - # unload renv if it's already been loaded - if ("renv" %in% loadedNamespaces()) - unloadNamespace("renv") - - # load bootstrap tools - ansify <- function(text) { - if (renv_ansify_enabled()) - renv_ansify_enhanced(text) - else - renv_ansify_default(text) - } - - renv_ansify_enabled <- function() { - - override <- Sys.getenv("RENV_ANSIFY_ENABLED", unset = NA) - if (!is.na(override)) - return(as.logical(override)) - - pane <- Sys.getenv("RSTUDIO_CHILD_PROCESS_PANE", unset = NA) - if (identical(pane, "build")) - return(FALSE) - - testthat <- Sys.getenv("TESTTHAT", unset = "false") - if (tolower(testthat) %in% "true") - return(FALSE) - - iderun <- Sys.getenv("R_CLI_HAS_HYPERLINK_IDE_RUN", unset = "false") - if (tolower(iderun) %in% "false") - return(FALSE) - - TRUE - - } - - renv_ansify_default <- function(text) { - text - } - - renv_ansify_enhanced <- function(text) { - - # R help links - pattern <- "`\\?(renv::(?:[^`])+)`" - replacement <- "`\033]8;;x-r-help:\\1\a?\\1\033]8;;\a`" - text <- gsub(pattern, replacement, text, perl = TRUE) - - # runnable code - pattern <- "`(renv::(?:[^`])+)`" - replacement <- "`\033]8;;x-r-run:\\1\a\\1\033]8;;\a`" - text <- gsub(pattern, replacement, text, perl = TRUE) - - # return ansified text - text - - } - - renv_ansify_init <- function() { - - envir <- renv_envir_self() - if (renv_ansify_enabled()) - assign("ansify", renv_ansify_enhanced, envir = envir) - else - assign("ansify", renv_ansify_default, envir = envir) - - } - - `%||%` <- function(x, y) { - if (is.null(x)) y else x - } - - catf <- function(fmt, ..., appendLF = TRUE) { - - quiet <- getOption("renv.bootstrap.quiet", default = FALSE) - if (quiet) - return(invisible()) - - msg <- sprintf(fmt, ...) - cat(msg, file = stdout(), sep = if (appendLF) "\n" else "") - - invisible(msg) - - } - - header <- function(label, - ..., - prefix = "#", - suffix = "-", - n = min(getOption("width"), 78)) - { - label <- sprintf(label, ...) - n <- max(n - nchar(label) - nchar(prefix) - 2L, 8L) - if (n <= 0) - return(paste(prefix, label)) - - tail <- paste(rep.int(suffix, n), collapse = "") - paste0(prefix, " ", label, " ", tail) - - } - - heredoc <- function(text, leave = 0) { - - # remove leading, trailing whitespace - trimmed <- gsub("^\\s*\\n|\\n\\s*$", "", text) - - # split into lines - lines <- strsplit(trimmed, "\n", fixed = TRUE)[[1L]] - - # compute common indent - indent <- regexpr("[^[:space:]]", lines) - common <- min(setdiff(indent, -1L)) - leave - text <- paste(substring(lines, common), collapse = "\n") - - # substitute in ANSI links for executable renv code - ansify(text) - - } - - bootstrap <- function(version, library) { - - friendly <- renv_bootstrap_version_friendly(version) - section <- header(sprintf("Bootstrapping renv %s", friendly)) - catf(section) - - # attempt to download renv - catf("- Downloading renv ... ", appendLF = FALSE) - withCallingHandlers( - tarball <- renv_bootstrap_download(version), - error = function(err) { - catf("FAILED") - stop("failed to download:\n", conditionMessage(err)) - } - ) - catf("OK") - on.exit(unlink(tarball), add = TRUE) - - # now attempt to install - catf("- Installing renv ... ", appendLF = FALSE) - withCallingHandlers( - status <- renv_bootstrap_install(version, tarball, library), - error = function(err) { - catf("FAILED") - stop("failed to install:\n", conditionMessage(err)) - } - ) - catf("OK") - - # add empty line to break up bootstrapping from normal output - catf("") - - return(invisible()) - } - - renv_bootstrap_tests_running <- function() { - getOption("renv.tests.running", default = FALSE) - } - - renv_bootstrap_repos <- function() { - - # get CRAN repository - cran <- getOption("renv.repos.cran", "https://cloud.r-project.org") - - # check for repos override - repos <- Sys.getenv("RENV_CONFIG_REPOS_OVERRIDE", unset = NA) - if (!is.na(repos)) { - - # check for RSPM; if set, use a fallback repository for renv - rspm <- Sys.getenv("RSPM", unset = NA) - if (identical(rspm, repos)) - repos <- c(RSPM = rspm, CRAN = cran) - - return(repos) - - } - - # check for lockfile repositories - repos <- tryCatch(renv_bootstrap_repos_lockfile(), error = identity) - if (!inherits(repos, "error") && length(repos)) - return(repos) - - # retrieve current repos - repos <- getOption("repos") - - # ensure @CRAN@ entries are resolved - repos[repos == "@CRAN@"] <- cran - - # add in renv.bootstrap.repos if set - default <- c(FALLBACK = "https://cloud.r-project.org") - extra <- getOption("renv.bootstrap.repos", default = default) - repos <- c(repos, extra) - - # remove duplicates that might've snuck in - dupes <- duplicated(repos) | duplicated(names(repos)) - repos[!dupes] - - } - - renv_bootstrap_repos_lockfile <- function() { - - lockpath <- Sys.getenv("RENV_PATHS_LOCKFILE", unset = "renv.lock") - if (!file.exists(lockpath)) - return(NULL) - - lockfile <- tryCatch(renv_json_read(lockpath), error = identity) - if (inherits(lockfile, "error")) { - warning(lockfile) - return(NULL) - } - - repos <- lockfile$R$Repositories - if (length(repos) == 0) - return(NULL) - - keys <- vapply(repos, `[[`, "Name", FUN.VALUE = character(1)) - vals <- vapply(repos, `[[`, "URL", FUN.VALUE = character(1)) - names(vals) <- keys - - return(vals) - - } - - renv_bootstrap_download <- function(version) { - - sha <- attr(version, "sha", exact = TRUE) - - methods <- if (!is.null(sha)) { - - # attempting to bootstrap a development version of renv - c( - function() renv_bootstrap_download_tarball(sha), - function() renv_bootstrap_download_github(sha) - ) - - } else { - - # attempting to bootstrap a release version of renv - c( - function() renv_bootstrap_download_tarball(version), - function() renv_bootstrap_download_cran_latest(version), - function() renv_bootstrap_download_cran_archive(version) - ) - - } - - for (method in methods) { - path <- tryCatch(method(), error = identity) - if (is.character(path) && file.exists(path)) - return(path) - } - - stop("All download methods failed") - - } - - renv_bootstrap_download_impl <- function(url, destfile) { - - mode <- "wb" - - # https://bugs.r-project.org/bugzilla/show_bug.cgi?id=17715 - fixup <- - Sys.info()[["sysname"]] == "Windows" && - substring(url, 1L, 5L) == "file:" - - if (fixup) - mode <- "w+b" - - args <- list( - url = url, - destfile = destfile, - mode = mode, - quiet = TRUE - ) - - if ("headers" %in% names(formals(utils::download.file))) { - headers <- renv_bootstrap_download_custom_headers(url) - if (length(headers) && is.character(headers)) - args$headers <- headers - } - - do.call(utils::download.file, args) - - } - - renv_bootstrap_download_custom_headers <- function(url) { - - headers <- getOption("renv.download.headers") - if (is.null(headers)) - return(character()) - - if (!is.function(headers)) - stopf("'renv.download.headers' is not a function") - - headers <- headers(url) - if (length(headers) == 0L) - return(character()) - - if (is.list(headers)) - headers <- unlist(headers, recursive = FALSE, use.names = TRUE) - - ok <- - is.character(headers) && - is.character(names(headers)) && - all(nzchar(names(headers))) - - if (!ok) - stop("invocation of 'renv.download.headers' did not return a named character vector") - - headers - - } - - renv_bootstrap_download_cran_latest <- function(version) { - - spec <- renv_bootstrap_download_cran_latest_find(version) - type <- spec$type - repos <- spec$repos - - baseurl <- utils::contrib.url(repos = repos, type = type) - ext <- if (identical(type, "source")) - ".tar.gz" - else if (Sys.info()[["sysname"]] == "Windows") - ".zip" - else - ".tgz" - name <- sprintf("renv_%s%s", version, ext) - url <- paste(baseurl, name, sep = "/") - - destfile <- file.path(tempdir(), name) - status <- tryCatch( - renv_bootstrap_download_impl(url, destfile), - condition = identity - ) - - if (inherits(status, "condition")) - return(FALSE) - - # report success and return - destfile - - } - - renv_bootstrap_download_cran_latest_find <- function(version) { - - # check whether binaries are supported on this system - binary <- - getOption("renv.bootstrap.binary", default = TRUE) && - !identical(.Platform$pkgType, "source") && - !identical(getOption("pkgType"), "source") && - Sys.info()[["sysname"]] %in% c("Darwin", "Windows") - - types <- c(if (binary) "binary", "source") - - # iterate over types + repositories - for (type in types) { - for (repos in renv_bootstrap_repos()) { - - # build arguments for utils::available.packages() call - args <- list(type = type, repos = repos) - - # add custom headers if available -- note that - # utils::available.packages() will pass this to download.file() - if ("headers" %in% names(formals(utils::download.file))) { - headers <- renv_bootstrap_download_custom_headers(repos) - if (length(headers) && is.character(headers)) - args$headers <- headers - } - - # retrieve package database - db <- tryCatch( - as.data.frame( - do.call(utils::available.packages, args), - stringsAsFactors = FALSE - ), - error = identity - ) - - if (inherits(db, "error")) - next - - # check for compatible entry - entry <- db[db$Package %in% "renv" & db$Version %in% version, ] - if (nrow(entry) == 0) - next - - # found it; return spec to caller - spec <- list(entry = entry, type = type, repos = repos) - return(spec) - - } - } - - # if we got here, we failed to find renv - fmt <- "renv %s is not available from your declared package repositories" - stop(sprintf(fmt, version)) - - } - - renv_bootstrap_download_cran_archive <- function(version) { - - name <- sprintf("renv_%s.tar.gz", version) - repos <- renv_bootstrap_repos() - urls <- file.path(repos, "src/contrib/Archive/renv", name) - destfile <- file.path(tempdir(), name) - - for (url in urls) { - - status <- tryCatch( - renv_bootstrap_download_impl(url, destfile), - condition = identity - ) - - if (identical(status, 0L)) - return(destfile) - - } - - return(FALSE) - - } - - renv_bootstrap_download_tarball <- function(version) { - - # if the user has provided the path to a tarball via - # an environment variable, then use it - tarball <- Sys.getenv("RENV_BOOTSTRAP_TARBALL", unset = NA) - if (is.na(tarball)) - return() - - # allow directories - if (dir.exists(tarball)) { - name <- sprintf("renv_%s.tar.gz", version) - tarball <- file.path(tarball, name) - } - - # bail if it doesn't exist - if (!file.exists(tarball)) { - - # let the user know we weren't able to honour their request - fmt <- "- RENV_BOOTSTRAP_TARBALL is set (%s) but does not exist." - msg <- sprintf(fmt, tarball) - warning(msg) - - # bail - return() - - } - - catf("- Using local tarball '%s'.", tarball) - tarball - - } - - renv_bootstrap_github_token <- function() { - for (envvar in c("GITHUB_TOKEN", "GITHUB_PAT", "GH_TOKEN")) { - envval <- Sys.getenv(envvar, unset = NA) - if (!is.na(envval)) - return(envval) - } - } - - renv_bootstrap_download_github <- function(version) { - - enabled <- Sys.getenv("RENV_BOOTSTRAP_FROM_GITHUB", unset = "TRUE") - if (!identical(enabled, "TRUE")) - return(FALSE) - - # prepare download options - token <- renv_bootstrap_github_token() - if (is.null(token)) - token <- "" - - if (nzchar(Sys.which("curl")) && nzchar(token)) { - fmt <- "--location --fail --header \"Authorization: token %s\"" - extra <- sprintf(fmt, token) - saved <- options("download.file.method", "download.file.extra") - options(download.file.method = "curl", download.file.extra = extra) - on.exit(do.call(base::options, saved), add = TRUE) - } else if (nzchar(Sys.which("wget")) && nzchar(token)) { - fmt <- "--header=\"Authorization: token %s\"" - extra <- sprintf(fmt, token) - saved <- options("download.file.method", "download.file.extra") - options(download.file.method = "wget", download.file.extra = extra) - on.exit(do.call(base::options, saved), add = TRUE) - } - - url <- file.path("https://api.github.com/repos/rstudio/renv/tarball", version) - name <- sprintf("renv_%s.tar.gz", version) - destfile <- file.path(tempdir(), name) - - status <- tryCatch( - renv_bootstrap_download_impl(url, destfile), - condition = identity - ) - - if (!identical(status, 0L)) - return(FALSE) - - renv_bootstrap_download_augment(destfile) - - return(destfile) - - } - - # Add Sha to DESCRIPTION. This is stop gap until #890, after which we - # can use renv::install() to fully capture metadata. - renv_bootstrap_download_augment <- function(destfile) { - sha <- renv_bootstrap_git_extract_sha1_tar(destfile) - if (is.null(sha)) { - return() - } - - # Untar - tempdir <- tempfile("renv-github-") - on.exit(unlink(tempdir, recursive = TRUE), add = TRUE) - untar(destfile, exdir = tempdir) - pkgdir <- dir(tempdir, full.names = TRUE)[[1]] - - # Modify description - desc_path <- file.path(pkgdir, "DESCRIPTION") - desc_lines <- readLines(desc_path) - remotes_fields <- c( - "RemoteType: github", - "RemoteHost: api.github.com", - "RemoteRepo: renv", - "RemoteUsername: rstudio", - "RemotePkgRef: rstudio/renv", - paste("RemoteRef: ", sha), - paste("RemoteSha: ", sha) - ) - writeLines(c(desc_lines[desc_lines != ""], remotes_fields), con = desc_path) - - # Re-tar - local({ - old <- setwd(tempdir) - on.exit(setwd(old), add = TRUE) - - tar(destfile, compression = "gzip") - }) - invisible() - } - - # Extract the commit hash from a git archive. Git archives include the SHA1 - # hash as the comment field of the tarball pax extended header - # (see https://www.kernel.org/pub/software/scm/git/docs/git-archive.html) - # For GitHub archives this should be the first header after the default one - # (512 byte) header. - renv_bootstrap_git_extract_sha1_tar <- function(bundle) { - - # open the bundle for reading - # We use gzcon for everything because (from ?gzcon) - # > Reading from a connection which does not supply a 'gzip' magic - # > header is equivalent to reading from the original connection - conn <- gzcon(file(bundle, open = "rb", raw = TRUE)) - on.exit(close(conn)) - - # The default pax header is 512 bytes long and the first pax extended header - # with the comment should be 51 bytes long - # `52 comment=` (11 chars) + 40 byte SHA1 hash - len <- 0x200 + 0x33 - res <- rawToChar(readBin(conn, "raw", n = len)[0x201:len]) - - if (grepl("^52 comment=", res)) { - sub("52 comment=", "", res) - } else { - NULL - } - } - - renv_bootstrap_install <- function(version, tarball, library) { - - # attempt to install it into project library - dir.create(library, showWarnings = FALSE, recursive = TRUE) - output <- renv_bootstrap_install_impl(library, tarball) - - # check for successful install - status <- attr(output, "status") - if (is.null(status) || identical(status, 0L)) - return(status) - - # an error occurred; report it - header <- "installation of renv failed" - lines <- paste(rep.int("=", nchar(header)), collapse = "") - text <- paste(c(header, lines, output), collapse = "\n") - stop(text) - - } - - renv_bootstrap_install_impl <- function(library, tarball) { - - # invoke using system2 so we can capture and report output - bin <- R.home("bin") - exe <- if (Sys.info()[["sysname"]] == "Windows") "R.exe" else "R" - R <- file.path(bin, exe) - - args <- c( - "--vanilla", "CMD", "INSTALL", "--no-multiarch", - "-l", shQuote(path.expand(library)), - shQuote(path.expand(tarball)) - ) - - system2(R, args, stdout = TRUE, stderr = TRUE) - - } - - renv_bootstrap_platform_prefix_default <- function() { - - # read version component - version <- Sys.getenv("RENV_PATHS_VERSION", unset = "R-%v") - - # expand placeholders - placeholders <- list( - list("%v", format(getRversion()[1, 1:2])), - list("%V", format(getRversion()[1, 1:3])) - ) - - for (placeholder in placeholders) - version <- gsub(placeholder[[1L]], placeholder[[2L]], version, fixed = TRUE) - - # include SVN revision for development versions of R - # (to avoid sharing platform-specific artefacts with released versions of R) - devel <- - identical(R.version[["status"]], "Under development (unstable)") || - identical(R.version[["nickname"]], "Unsuffered Consequences") - - if (devel) - version <- paste(version, R.version[["svn rev"]], sep = "-r") - - version - - } - - renv_bootstrap_platform_prefix <- function() { - - # construct version prefix - version <- renv_bootstrap_platform_prefix_default() - - # build list of path components - components <- c(version, R.version$platform) - - # include prefix if provided by user - prefix <- renv_bootstrap_platform_prefix_impl() - if (!is.na(prefix) && nzchar(prefix)) - components <- c(prefix, components) - - # build prefix - paste(components, collapse = "/") - - } - - renv_bootstrap_platform_prefix_impl <- function() { - - # if an explicit prefix has been supplied, use it - prefix <- Sys.getenv("RENV_PATHS_PREFIX", unset = NA) - if (!is.na(prefix)) - return(prefix) - - # if the user has requested an automatic prefix, generate it - auto <- Sys.getenv("RENV_PATHS_PREFIX_AUTO", unset = NA) - if (is.na(auto) && getRversion() >= "4.4.0") - auto <- "TRUE" - - if (auto %in% c("TRUE", "True", "true", "1")) - return(renv_bootstrap_platform_prefix_auto()) - - # empty string on failure - "" - - } - - renv_bootstrap_platform_prefix_auto <- function() { - - prefix <- tryCatch(renv_bootstrap_platform_os(), error = identity) - if (inherits(prefix, "error") || prefix %in% "unknown") { - - msg <- paste( - "failed to infer current operating system", - "please file a bug report at https://github.com/rstudio/renv/issues", - sep = "; " - ) - - warning(msg) - - } - - prefix - - } - - renv_bootstrap_platform_os <- function() { - - sysinfo <- Sys.info() - sysname <- sysinfo[["sysname"]] - - # handle Windows + macOS up front - if (sysname == "Windows") - return("windows") - else if (sysname == "Darwin") - return("macos") - - # check for os-release files - for (file in c("/etc/os-release", "/usr/lib/os-release")) - if (file.exists(file)) - return(renv_bootstrap_platform_os_via_os_release(file, sysinfo)) - - # check for redhat-release files - if (file.exists("/etc/redhat-release")) - return(renv_bootstrap_platform_os_via_redhat_release()) - - "unknown" - - } - - renv_bootstrap_platform_os_via_os_release <- function(file, sysinfo) { - - # read /etc/os-release - release <- utils::read.table( - file = file, - sep = "=", - quote = c("\"", "'"), - col.names = c("Key", "Value"), - comment.char = "#", - stringsAsFactors = FALSE - ) - - vars <- as.list(release$Value) - names(vars) <- release$Key - - # get os name - os <- tolower(sysinfo[["sysname"]]) - - # read id - id <- "unknown" - for (field in c("ID", "ID_LIKE")) { - if (field %in% names(vars) && nzchar(vars[[field]])) { - id <- vars[[field]] - break - } - } - - # read version - version <- "unknown" - for (field in c("UBUNTU_CODENAME", "VERSION_CODENAME", "VERSION_ID", "BUILD_ID")) { - if (field %in% names(vars) && nzchar(vars[[field]])) { - version <- vars[[field]] - break - } - } - - # join together - paste(c(os, id, version), collapse = "-") - - } - - renv_bootstrap_platform_os_via_redhat_release <- function() { - - # read /etc/redhat-release - contents <- readLines("/etc/redhat-release", warn = FALSE) - - # infer id - id <- if (grepl("centos", contents, ignore.case = TRUE)) - "centos" - else if (grepl("redhat", contents, ignore.case = TRUE)) - "redhat" - else - "unknown" - - # try to find a version component (very hacky) - version <- "unknown" - - parts <- strsplit(contents, "[[:space:]]")[[1L]] - for (part in parts) { - - nv <- tryCatch(numeric_version(part), error = identity) - if (inherits(nv, "error")) - next - - version <- nv[1, 1] - break - - } - - paste(c("linux", id, version), collapse = "-") - - } - - renv_bootstrap_library_root_name <- function(project) { - - # use project name as-is if requested - asis <- Sys.getenv("RENV_PATHS_LIBRARY_ROOT_ASIS", unset = "FALSE") - if (asis) - return(basename(project)) - - # otherwise, disambiguate based on project's path - id <- substring(renv_bootstrap_hash_text(project), 1L, 8L) - paste(basename(project), id, sep = "-") - - } - - renv_bootstrap_library_root <- function(project) { - - prefix <- renv_bootstrap_profile_prefix() - - path <- Sys.getenv("RENV_PATHS_LIBRARY", unset = NA) - if (!is.na(path)) - return(paste(c(path, prefix), collapse = "/")) - - path <- renv_bootstrap_library_root_impl(project) - if (!is.null(path)) { - name <- renv_bootstrap_library_root_name(project) - return(paste(c(path, prefix, name), collapse = "/")) - } - - renv_bootstrap_paths_renv("library", project = project) - - } - - renv_bootstrap_library_root_impl <- function(project) { - - root <- Sys.getenv("RENV_PATHS_LIBRARY_ROOT", unset = NA) - if (!is.na(root)) - return(root) - - type <- renv_bootstrap_project_type(project) - if (identical(type, "package")) { - userdir <- renv_bootstrap_user_dir() - return(file.path(userdir, "library")) - } - - } - - renv_bootstrap_validate_version <- function(version, description = NULL) { - - # resolve description file - # - # avoid passing lib.loc to `packageDescription()` below, since R will - # use the loaded version of the package by default anyhow. note that - # this function should only be called after 'renv' is loaded - # https://github.com/rstudio/renv/issues/1625 - description <- description %||% packageDescription("renv") - - # check whether requested version 'version' matches loaded version of renv - sha <- attr(version, "sha", exact = TRUE) - valid <- if (!is.null(sha)) - renv_bootstrap_validate_version_dev(sha, description) - else - renv_bootstrap_validate_version_release(version, description) - - if (valid) - return(TRUE) - - # the loaded version of renv doesn't match the requested version; - # give the user instructions on how to proceed - dev <- identical(description[["RemoteType"]], "github") - remote <- if (dev) - paste("rstudio/renv", description[["RemoteSha"]], sep = "@") - else - paste("renv", description[["Version"]], sep = "@") - - # display both loaded version + sha if available - friendly <- renv_bootstrap_version_friendly( - version = description[["Version"]], - sha = if (dev) description[["RemoteSha"]] - ) - - fmt <- heredoc(" - renv %1$s was loaded from project library, but this project is configured to use renv %2$s. - - Use `renv::record(\"%3$s\")` to record renv %1$s in the lockfile. - - Use `renv::restore(packages = \"renv\")` to install renv %2$s into the project library. - ") - catf(fmt, friendly, renv_bootstrap_version_friendly(version), remote) - - FALSE - - } - - renv_bootstrap_validate_version_dev <- function(version, description) { - - expected <- description[["RemoteSha"]] - if (!is.character(expected)) - return(FALSE) - - pattern <- sprintf("^\\Q%s\\E", version) - grepl(pattern, expected, perl = TRUE) - - } - - renv_bootstrap_validate_version_release <- function(version, description) { - expected <- description[["Version"]] - is.character(expected) && identical(expected, version) - } - - renv_bootstrap_hash_text <- function(text) { - - hashfile <- tempfile("renv-hash-") - on.exit(unlink(hashfile), add = TRUE) - - writeLines(text, con = hashfile) - tools::md5sum(hashfile) - - } - - renv_bootstrap_load <- function(project, libpath, version) { - - # try to load renv from the project library - if (!requireNamespace("renv", lib.loc = libpath, quietly = TRUE)) - return(FALSE) - - # warn if the version of renv loaded does not match - renv_bootstrap_validate_version(version) - - # execute renv load hooks, if any - hooks <- getHook("renv::autoload") - for (hook in hooks) - if (is.function(hook)) - tryCatch(hook(), error = warnify) - - # load the project - renv::load(project) - - TRUE - - } - - renv_bootstrap_profile_load <- function(project) { - - # if RENV_PROFILE is already set, just use that - profile <- Sys.getenv("RENV_PROFILE", unset = NA) - if (!is.na(profile) && nzchar(profile)) - return(profile) - - # check for a profile file (nothing to do if it doesn't exist) - path <- renv_bootstrap_paths_renv("profile", profile = FALSE, project = project) - if (!file.exists(path)) - return(NULL) - - # read the profile, and set it if it exists - contents <- readLines(path, warn = FALSE) - if (length(contents) == 0L) - return(NULL) - - # set RENV_PROFILE - profile <- contents[[1L]] - if (!profile %in% c("", "default")) - Sys.setenv(RENV_PROFILE = profile) - - profile - - } - - renv_bootstrap_profile_prefix <- function() { - profile <- renv_bootstrap_profile_get() - if (!is.null(profile)) - return(file.path("profiles", profile, "renv")) - } - - renv_bootstrap_profile_get <- function() { - profile <- Sys.getenv("RENV_PROFILE", unset = "") - renv_bootstrap_profile_normalize(profile) - } - - renv_bootstrap_profile_set <- function(profile) { - profile <- renv_bootstrap_profile_normalize(profile) - if (is.null(profile)) - Sys.unsetenv("RENV_PROFILE") - else - Sys.setenv(RENV_PROFILE = profile) - } - - renv_bootstrap_profile_normalize <- function(profile) { - - if (is.null(profile) || profile %in% c("", "default")) - return(NULL) - - profile - - } - - renv_bootstrap_path_absolute <- function(path) { - - substr(path, 1L, 1L) %in% c("~", "/", "\\") || ( - substr(path, 1L, 1L) %in% c(letters, LETTERS) && - substr(path, 2L, 3L) %in% c(":/", ":\\") - ) - - } - - renv_bootstrap_paths_renv <- function(..., profile = TRUE, project = NULL) { - renv <- Sys.getenv("RENV_PATHS_RENV", unset = "renv") - root <- if (renv_bootstrap_path_absolute(renv)) NULL else project - prefix <- if (profile) renv_bootstrap_profile_prefix() - components <- c(root, renv, prefix, ...) - paste(components, collapse = "/") - } - - renv_bootstrap_project_type <- function(path) { - - descpath <- file.path(path, "DESCRIPTION") - if (!file.exists(descpath)) - return("unknown") - - desc <- tryCatch( - read.dcf(descpath, all = TRUE), - error = identity - ) - - if (inherits(desc, "error")) - return("unknown") - - type <- desc$Type - if (!is.null(type)) - return(tolower(type)) - - package <- desc$Package - if (!is.null(package)) - return("package") - - "unknown" - - } - - renv_bootstrap_user_dir <- function() { - dir <- renv_bootstrap_user_dir_impl() - path.expand(chartr("\\", "/", dir)) - } - - renv_bootstrap_user_dir_impl <- function() { - - # use local override if set - override <- getOption("renv.userdir.override") - if (!is.null(override)) - return(override) - - # use R_user_dir if available - tools <- asNamespace("tools") - if (is.function(tools$R_user_dir)) - return(tools$R_user_dir("renv", "cache")) - - # try using our own backfill for older versions of R - envvars <- c("R_USER_CACHE_DIR", "XDG_CACHE_HOME") - for (envvar in envvars) { - root <- Sys.getenv(envvar, unset = NA) - if (!is.na(root)) - return(file.path(root, "R/renv")) - } - - # use platform-specific default fallbacks - if (Sys.info()[["sysname"]] == "Windows") - file.path(Sys.getenv("LOCALAPPDATA"), "R/cache/R/renv") - else if (Sys.info()[["sysname"]] == "Darwin") - "~/Library/Caches/org.R-project.R/R/renv" - else - "~/.cache/R/renv" - - } - - renv_bootstrap_version_friendly <- function(version, shafmt = NULL, sha = NULL) { - sha <- sha %||% attr(version, "sha", exact = TRUE) - parts <- c(version, sprintf(shafmt %||% " [sha: %s]", substring(sha, 1L, 7L))) - paste(parts, collapse = "") - } - - renv_bootstrap_exec <- function(project, libpath, version) { - if (!renv_bootstrap_load(project, libpath, version)) - renv_bootstrap_run(project, libpath, version) - } - - renv_bootstrap_run <- function(project, libpath, version) { - - # perform bootstrap - bootstrap(version, libpath) - - # exit early if we're just testing bootstrap - if (!is.na(Sys.getenv("RENV_BOOTSTRAP_INSTALL_ONLY", unset = NA))) - return(TRUE) - - # try again to load - if (requireNamespace("renv", lib.loc = libpath, quietly = TRUE)) { - return(renv::load(project = project)) - } - - # failed to download or load renv; warn the user - msg <- c( - "Failed to find an renv installation: the project will not be loaded.", - "Use `renv::activate()` to re-initialize the project." - ) - - warning(paste(msg, collapse = "\n"), call. = FALSE) - - } - - renv_json_read <- function(file = NULL, text = NULL) { - - jlerr <- NULL - - # if jsonlite is loaded, use that instead - if ("jsonlite" %in% loadedNamespaces()) { - - json <- tryCatch(renv_json_read_jsonlite(file, text), error = identity) - if (!inherits(json, "error")) - return(json) - - jlerr <- json - - } - - # otherwise, fall back to the default JSON reader - json <- tryCatch(renv_json_read_default(file, text), error = identity) - if (!inherits(json, "error")) - return(json) - - # report an error - if (!is.null(jlerr)) - stop(jlerr) - else - stop(json) - - } - - renv_json_read_jsonlite <- function(file = NULL, text = NULL) { - text <- paste(text %||% readLines(file, warn = FALSE), collapse = "\n") - jsonlite::fromJSON(txt = text, simplifyVector = FALSE) - } - - renv_json_read_patterns <- function() { - - list( - - # objects - list("{", "\t\n\tobject(\t\n\t", TRUE), - list("}", "\t\n\t)\t\n\t", TRUE), - - # arrays - list("[", "\t\n\tarray(\t\n\t", TRUE), - list("]", "\n\t\n)\n\t\n", TRUE), - - # maps - list(":", "\t\n\t=\t\n\t", TRUE), - - # newlines - list("\\u000a", "\n", FALSE) - - ) - - } - - renv_json_read_envir <- function() { - - envir <- new.env(parent = emptyenv()) - - envir[["+"]] <- `+` - envir[["-"]] <- `-` - - envir[["object"]] <- function(...) { - result <- list(...) - names(result) <- as.character(names(result)) - result - } - - envir[["array"]] <- list - - envir[["true"]] <- TRUE - envir[["false"]] <- FALSE - envir[["null"]] <- NULL - - envir - - } - - renv_json_read_remap <- function(object, patterns) { - - # repair names if necessary - if (!is.null(names(object))) { - - nms <- names(object) - for (pattern in patterns) - nms <- gsub(pattern[[2L]], pattern[[1L]], nms, fixed = TRUE) - names(object) <- nms - - } - - # repair strings if necessary - if (is.character(object)) { - for (pattern in patterns) - object <- gsub(pattern[[2L]], pattern[[1L]], object, fixed = TRUE) - } - - # recurse for other objects - if (is.recursive(object)) - for (i in seq_along(object)) - object[i] <- list(renv_json_read_remap(object[[i]], patterns)) - - # return remapped object - object - - } - - renv_json_read_default <- function(file = NULL, text = NULL) { - - # read json text - text <- paste(text %||% readLines(file, warn = FALSE), collapse = "\n") - - # convert into something the R parser will understand - patterns <- renv_json_read_patterns() - transformed <- text - for (pattern in patterns) - transformed <- gsub(pattern[[1L]], pattern[[2L]], transformed, fixed = TRUE) - - # parse it - rfile <- tempfile("renv-json-", fileext = ".R") - on.exit(unlink(rfile), add = TRUE) - writeLines(transformed, con = rfile) - json <- parse(rfile, keep.source = FALSE, srcfile = NULL)[[1L]] - - # evaluate in safe environment - result <- eval(json, envir = renv_json_read_envir()) - - # fix up strings if necessary -- do so only with reversible patterns - patterns <- Filter(function(pattern) pattern[[3L]], patterns) - renv_json_read_remap(result, patterns) - - } - - - # load the renv profile, if any - renv_bootstrap_profile_load(project) - - # construct path to library root - root <- renv_bootstrap_library_root(project) - - # construct library prefix for platform - prefix <- renv_bootstrap_platform_prefix() - - # construct full libpath - libpath <- file.path(root, prefix) - - # run bootstrap code - renv_bootstrap_exec(project, libpath, version) - - invisible() - -}) diff --git a/renv/settings.json b/renv/settings.json deleted file mode 100644 index ffdbb32..0000000 --- a/renv/settings.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "bioconductor.version": null, - "external.libraries": [], - "ignored.packages": [], - "package.dependency.fields": [ - "Imports", - "Depends", - "LinkingTo" - ], - "ppm.enabled": null, - "ppm.ignored.urls": [], - "r.version": null, - "snapshot.type": "implicit", - "use.cache": true, - "vcs.ignore.cellar": true, - "vcs.ignore.library": true, - "vcs.ignore.local": true, - "vcs.manage.ignores": true -} diff --git a/scripts/R/pre_commit.R b/scripts/R/pre_commit.R deleted file mode 100644 index d5db43e..0000000 --- a/scripts/R/pre_commit.R +++ /dev/null @@ -1,14 +0,0 @@ -styler_confs <- rlang::exprs( - filetype = c(".R", ".Rmd", ".Rmarkdown", ".Rnw"), - indent_by = 4L, - start_comments_with_one_space = TRUE -) - -do.call(styler::style_pkg, styler_confs) -do.call(styler::style_dir, c(list(path = here::here("scripts")), styler_confs)) -do.call(styler::style_dir, c(list(path = here::here("reference_data")), styler_confs)) -do.call(styler::style_dir, c(list(path = here::here("tests")), styler_confs)) -do.call(styler::style_dir, c(list(path = here::here("tools")), styler_confs)) -source(here::here("reference_data", "build_package_data.R")) -devtools::document() -devtools::test() diff --git a/scripts/R/run_pipeline.R b/scripts/R/run_pipeline.R deleted file mode 100644 index 5c161da..0000000 --- a/scripts/R/run_pipeline.R +++ /dev/null @@ -1,139 +0,0 @@ -download_data <- function(bucket, data_dir) { - print("Start downloading data from GCP Storage") - command <- paste("gsutil -m cp -r", paste0("gs://", bucket), data_dir) - exit_code <- system(command) - if (exit_code != 0) { - paste("Error while executing", command) - stop("Error during downloading data") - } - print("Finished downloading data from GCP Storage") -} - -download_google_sheet <- function(sheet_id, name) { - json_key <- here::here("secrets", "a4d-gcp-sa.json") - - # Authenticate both packages with the same service account - googlesheets4::gs4_auth(path = json_key) - googledrive::drive_auth(path = json_key) - - file <- googledrive::drive_get(googledrive::as_id(sheet_id)) - googledrive::drive_download(file, here::here("reference_data", name), type = "xlsx", overwrite = TRUE) -} - -upload_data <- function(bucket, data_dir) { - print("Start uploading data to GCP Storage") - command <- paste("gsutil -m cp -r", data_dir, paste0("gs://", bucket)) - exit_code <- system(command) - if (exit_code != 0) { - paste("Error while executing", command) - stop("Error during uploading data") - } - print("Finished uploading data to GCP Storage") -} - -ingest_data <- function(project_id, cluster_fields, dataset, table, source) { - print("Deleting old table in GCP Big Query") - command <- paste( - "bq rm", - "-f", - "-t", - paste0(project_id, ":", dataset, ".", table) - ) - cat(command) - exit_code <- system(command) - if (exit_code != 0) { - paste("Error while executing", command) - stop("Error during ingesting data") - } - - print("Ingesting data to GCP Big Query") - command <- paste( - "bq load", - "--source_format=PARQUET", - paste0("--project_id=", project_id), - "--max_bad_records=0", - paste0("--clustering_fields=", cluster_fields), - paste0(dataset, ".", table), - source - ) - cat(command) - exit_code <- system(command) - if (exit_code != 0) { - paste("Error while executing", command) - stop("Error during ingesting data") - } - print("Finished ingesting data to GCP Big Query") -} - -config <- config::get() -data_dir <- config$data_root -output_dir <- file.path(data_dir, "output") -# WARNING: this deletes all tracker files, only run on VM where we download them again! -unlink(file.path(data_dir, "*"), recursive = T, force = T) -unlink(output_dir, recursive = T, force = T) -table_dir <- file.path(output_dir, "tables") - -download_data(bucket = config$download_bucket, data_dir = data_dir) -download_google_sheet("1HOxi0o9fTAoHySjW_M3F-09TRBnUITOzzxGx2HwRMAw", "clinic_data.xlsx") -source("scripts/R/run_script_1_extract_raw_data.R") # creates CSV files in subfolders patient_data_raw and product_data_raw -source("scripts/R/run_script_2_clean_data.R") # creates CSV files in subfolders patient_data_cleaned and product_data_cleaned -source("scripts/R/run_script_3_create_tables.R") # creates final CSV files in subfolder tables -source("scripts/R/run_script_4_create_logs_table.R") -source("scripts/R/run_script_5_create_metadata_table.R") -upload_data(bucket = config$upload_bucket, data_dir = output_dir) -ingest_data( - project_id = config$project_id, - cluster_fields = "clinic_id,patient_id,tracker_date", - dataset = config$dataset, - table = "patient_data_monthly", - source = file.path(table_dir, "patient_data_monthly.parquet") -) -ingest_data( - project_id = config$project_id, - cluster_fields = "patient_id,tracker_date", - dataset = config$dataset, - table = "patient_data_annual", - source = file.path(table_dir, "patient_data_annual.parquet") -) -ingest_data( - project_id = config$project_id, - cluster_fields = "clinic_id,patient_id,tracker_date", - dataset = config$dataset, - table = "patient_data_static", - source = file.path(table_dir, "patient_data_static.parquet") -) -ingest_data( - project_id = config$project_id, - cluster_fields = "clinic_id,patient_id,tracker_date", - dataset = config$dataset, - table = "patient_data_hba1c", - source = file.path(table_dir, "longitudinal_data_hba1c.parquet") -) -ingest_data( - project_id = config$project_id, - cluster_fields = "clinic_id,product_released_to,product_table_year,product_table_month", - dataset = config$dataset, - table = "product_data", - source = file.path(table_dir, "product_data.parquet") -) -ingest_data( - project_id = config$project_id, - cluster_fields = "clinic_id", - dataset = config$dataset, - table = "clinic_data_static", - source = file.path(table_dir, "clinic_data_static.parquet") -) -ingest_data( - project_id = config$project_id, - cluster_fields = "functionName,errorCode,warningCode,fileName", - dataset = config$dataset, - table = "logs", - source = file.path(table_dir, "table_logs.parquet") -) -ingest_data( - project_id = config$project_id, - cluster_fields = "file_name,clinic_code", - dataset = config$dataset, - table = "tracker_metadata", - source = file.path(table_dir, "tracker_metadata.parquet") -) diff --git a/scripts/R/run_script_1_extract_raw_data.R b/scripts/R/run_script_1_extract_raw_data.R deleted file mode 100644 index 4097e1d..0000000 --- a/scripts/R/run_script_1_extract_raw_data.R +++ /dev/null @@ -1,32 +0,0 @@ -options(readxl.show_progress = FALSE) - -main <- function() { - paths <- init_paths(c("patient_data_raw", "product_data_raw"), delete = TRUE) - setup_logger(paths$output_root, "script1") - tracker_files <- get_files(paths$tracker_root) - logInfo( - log_to_json( - "Found {values['len']} xlsx files under {values['root']}.", - values = list(len = length(tracker_files), root = paths$tracker_root), - script = "script1", - file = "run_script_1_extract_raw_data.R", - functionName = "main" - ) - ) - - synonyms <- get_synonyms() - - for (i in seq_along(tracker_files)) { - tracker_file <- tracker_files[i] - tracker_name <- tools::file_path_sans_ext(basename(tracker_file)) - tictoc::tic(paste("Processing tracker file:", tracker_name)) - process_tracker_file(paths, tracker_file, tracker_name, synonyms) - tictoc::toc() - cat(paste("Processed ", i, " of ", length(tracker_files), " (", round(i / length(tracker_files) * 100, 0), "%) tracker files.\n")) - } -} - -# profvis(main()) -main() - -clearLoggers() diff --git a/scripts/R/run_script_2_clean_data.R b/scripts/R/run_script_2_clean_data.R deleted file mode 100644 index 16d42b0..0000000 --- a/scripts/R/run_script_2_clean_data.R +++ /dev/null @@ -1,122 +0,0 @@ -options(readxl.show_progress = FALSE) - -ERROR_VAL_NUMERIC <<- 999999 -ERROR_VAL_CHARACTER <<- "Undefined" -ERROR_VAL_DATE <<- "9999-09-09" - -main <- function() { - paths <- init_paths(c("patient_data_cleaned", "product_data_cleaned"), delete = TRUE) - setup_logger(paths$output_root, "script2") - patient_data_files <- get_files(paths$output_root, pattern = "patient_raw.parquet$") - product_data_files <- get_files(paths$tracker_root, pattern = "product_raw.parquet$") - - logInfo( - log_to_json( - "Found {values['len']} csv files with patient data under {values['root']}. ", - values = list(len = length(patient_data_files), root = paths$output_root), - script = "script2", - file = "run_script2_clean_data.R", - functionName = "main" - ) - ) - logInfo( - log_to_json( - "Found {values['len']} csv files with product data under {values['root']}. ", - values = list(len = length(product_data_files), root = paths$output_root), - script = "script2", - file = "run_script2_clean_data.R", - functionName = "main" - ) - ) - - for (i in seq_along(patient_data_files)) { - patient_file <- patient_data_files[i] - patient_file_name <- tools::file_path_sans_ext(basename(patient_file)) - tictoc::tic(paste("Processing raw patient data:", patient_file_name)) - - logfile <- paste0(patient_file_name) - with_file_logger(logfile, - { - tryCatch( - process_raw_patient_file(paths, patient_file, patient_file_name, paths$patient_data_cleaned), - error = function(e) { - logError( - log_to_json( - "Could not process raw patient data. Error = {values['e']}.", - values = list(e = e$message), - script = "script2", - file = "script2_process_patient_data.R", - errorCode = "critical_abort", - functionName = "process_raw_patient_file" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - "Could not process raw patient data. Warning = {values['w']}.", - values = list(w = w$message), - script = "script2", - file = "script2_process_patient_data.R", - warningCode = "critical_abort", - functionName = "process_raw_patient_file" - ) - ) - } - ) - }, - output_root = paths$output_root - ) - tictoc::toc() - cat(paste("Processed ", i, " of ", length(patient_data_files), " (", round(i / length(patient_data_files) * 100, 0), "%) raw patient files.\n")) - } - - synonyms <- get_synonyms() - synonyms_product <- synonyms$product - - for (i in seq_along(product_data_files)) { - product_file <- product_data_files[i] - product_file_name <- tools::file_path_sans_ext(basename(product_file)) - tictoc::tic(paste("Processing product data:", product_file_name)) - - logfile <- paste0(product_file_name) - with_file_logger(logfile, - { - tryCatch( - process_raw_product_file(paths, product_file, product_file_name, synonyms_product, paths$product_data_cleaned), - error = function(e) { - logError( - log_to_json( - "Could not process raw product data. Error = {values['e']}.", - values = list(e = e$message), - script = "script2", - file = "script2_process_product_data.R", - functionName = "process_raw_product_file", - errorCode = "critical_abort" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - "Could not process raw product data. Warning = {values['w']}.", - values = list(w = w$message), - script = "script2", - file = "script2_process_product_data.R", - functionName = "process_raw_product_file", - warningCode = "critical_abort" - ) - ) - } - ) - }, - output_root = paths$output_root - ) - tictoc::toc() - cat(paste("Processed ", i, " of ", length(product_data_files), " (", round(i / length(product_data_files) * 100, 0), "%) product files.\n")) - } -} - -main() - -clearLoggers() diff --git a/scripts/R/run_script_3_create_tables.R b/scripts/R/run_script_3_create_tables.R deleted file mode 100644 index 8a27014..0000000 --- a/scripts/R/run_script_3_create_tables.R +++ /dev/null @@ -1,293 +0,0 @@ -options(readxl.show_progress = FALSE) - -ERROR_VAL_NUMERIC <<- 999999 -ERROR_VAL_CHARACTER <<- "Undefined" -ERROR_VAL_DATE <<- "9999-09-09" - -main <- function() { - paths <- init_paths(c("tables"), delete = TRUE) - setup_logger(paths$output_root, "script3") - patient_data_files <- get_files(file.path(paths$output_root, "patient_data_cleaned"), pattern = "\\.parquet$") - product_data_files <- get_files(file.path(paths$output_root, "product_data_cleaned"), pattern = "\\.parquet$") - logInfo( - log_to_json( - "Found {values['len']} patient csv files under {values['root']}. ", - values = list(len = length(patient_data_files), root = paths$tracker_root), - script = "script3", - file = "run_script3_create_tables.R", - functionName = "main" - ) - ) - logInfo( - log_to_json( - "Found {values['len']} product csv files under {values['root']}. ", - values = list(len = length(product_data_files), root = paths$tracker_root), - script = "script3", - file = "run_script3_create_tables.R", - functionName = "main" - ) - ) - - logfile <- "table_patient_data_static" - with_file_logger(logfile, - { - tryCatch( - { - create_table_patient_data_static(patient_data_files, file.path(paths$output_root, "patient_data_cleaned"), paths$tables) - }, - error = function(e) { - logError( - log_to_json( - "Could not create table for static patient data. Error = {values['e']}.", - values = list(e = e$message), - script = "script3", - file = "run_script_3_create_tables.R", - errorCode = "critical_abort", - functionName = "create_table_patient_data_static" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - "Could not create table for static patient data. Warning = {values['w']}.", - values = list(w = w$message), - script = "script3", - file = "run_script_3_create_tables.R", - warningCode = "critical_abort", - functionName = "create_table_patient_data_static" - ) - ) - } - ) - }, - output_root = paths$output_root - ) - - logfile <- "table_patient_data_monthly" - with_file_logger(logfile, - { - tryCatch( - { - create_table_patient_data_monthly(patient_data_files, file.path(paths$output_root, "patient_data_cleaned"), paths$tables) - }, - error = function(e) { - logError( - log_to_json( - "Could not create table for monthly patient data. Error = {values['e']}.", - values = list(e = e$message), - script = "script3", - file = "run_script_3_create_tables.R", - errorCode = "critical_abort", - functionName = "create_table_patient_data_monthly" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - "Could not create table for monthly patient data. Warning = {values['w']}.", - values = list(w = w$message), - script = "script3", - file = "run_script_3_create_tables.R", - warningCode = "critical_abort", - functionName = "create_table_patient_data_monthly" - ) - ) - } - ) - }, - output_root = paths$output_root - ) - - logfile <- "table_longitudinal_data_hba1c" - with_file_logger(logfile, - { - tryCatch( - { - create_table_longitudinal_data( - patient_data_files, - file.path(paths$output_root, "patient_data_cleaned"), - paths$tables, - "hba1c_updated", - "hba1c" - ) - }, - error = function(e) { - logError( - log_to_json( - "Could not create table for longitudinal patient data. Error = {values['e']}.", - values = list(e = e$message), - script = "script3", - file = "run_script_3_create_tables.R", - errorCode = "critical_abort", - functionName = "create_table_longitudinal_data" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - "Could not create table for longitudinal patient data. Warning = {values['w']}.", - values = list(w = w$message), - script = "script3", - file = "run_script_3_create_tables.R", - warningCode = "critical_abort", - functionName = "create_table_longitudinal_data" - ) - ) - } - ) - }, - output_root = paths$output_root - ) - - logfile <- "table_patient_data_annual" - with_file_logger(logfile, - { - tryCatch( - { - create_table_patient_data_annual(patient_data_files, file.path(paths$output_root, "patient_data_cleaned"), paths$tables) - }, - error = function(e) { - logError( - log_to_json( - "Could not create table for annual patient data. Error = {values['e']}.", - values = list(e = e$message), - script = "script3", - file = "run_script_3_create_tables.R", - errorCode = "critical_abort", - functionName = "create_table_patient_data_annual" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - "Could not create table for annual patient data. Warning = {values['w']}.", - values = list(w = w$message), - script = "script3", - file = "run_script_3_create_tables.R", - warningCode = "critical_abort", - functionName = "create_table_patient_data_annual" - ) - ) - } - ) - }, - output_root = paths$output_root - ) - - logfile <- "table_product_data" - with_file_logger(logfile, - { - tryCatch( - { - create_table_product_data(file.path(paths$output_root, "product_data_cleaned"), paths$tables) - }, - error = function(e) { - logError( - log_to_json( - "Could not create table for product data. Error = {values['e']}.", - values = list(e = e$message), - script = "script3", - file = "run_script_3_create_tables.R", - errorCode = "critical_abort", - functionName = "create_table_product_data" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - "Could not create table for product data. Warning = {values['w']}.", - values = list(w = w$message), - script = "script3", - file = "run_script_3_create_tables.R", - warningCode = "critical_abort", - functionName = "create_table_product_data" - ) - ) - } - ) - }, - output_root = paths$output_root - ) - - logfile <- "table_clinic_data_static" - with_file_logger(logfile, - { - tryCatch( - { - create_table_clinic_static_data(paths$tables) - }, - error = function(e) { - logError( - log_to_json( - "Could not create table for clinic static data. Error = {values['e']}.", - values = list(e = e$message), - script = "script3", - file = "run_script_3_create_tables.R", - errorCode = "critical_abort", - functionName = "create_table_clinic_static_data" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - "Could not create table for clinic static data. Warning = {values['w']}.", - values = list(w = w$message), - script = "script3", - file = "run_script_3_create_tables.R", - warningCode = "critical_abort", - functionName = "create_table_clinic_static_data" - ) - ) - } - ) - }, - output_root = paths$output_root - ) - - logfile <- "link_product_patient_data" - with_file_logger(logfile, - { - tryCatch( - { - link_product_patient( - file.path(paths$tables, "product_data.parquet"), - file.path(paths$tables, "patient_data_monthly.parquet") - ) - }, - error = function(e) { - logError( - log_to_json( - "Could not link files for product and patient data. Error = {values['e']}.", - values = list(e = e$message), - script = "script3", - file = "run_script_3_create_tables.R", - errorCode = "tryCatch", - functionName = "link_product_patient" - ) - ) - }, - warning = function(w) { - logWarn( - log_to_json( - "Could not link files for product and patient data. Warning = {values['w']}.", - values = list(w = w$message), - script = "script3", - file = "run_script_3_create_tables.R", - warningCode = "tryCatch", - functionName = "link_product_patient" - ) - ) - } - ) - }, - output_root = paths$output_root - ) -} - -main() diff --git a/scripts/R/run_script_4_create_logs_table.R b/scripts/R/run_script_4_create_logs_table.R deleted file mode 100644 index 2287e36..0000000 --- a/scripts/R/run_script_4_create_logs_table.R +++ /dev/null @@ -1,73 +0,0 @@ -# current time -current_timestamp <- Sys.time() - -main <- function() { - paths <- init_paths(c("logs", "tables"), delete = FALSE) - logFiles <- get_files(paths$logs, pattern = "*.log") - allLogs <- tibble::tibble() - - for (file in logFiles) { - tryCatch( - { - filePath <- file.path(paths$logs, file) - fileConnection <- file(filePath, open = "r") - lines <- readLines(fileConnection) - close(fileConnection) - - if (!is.null(lines)) { - result <- parseLines(lines) %>% - dplyr::mutate(fileName = file) - allLogs <- allLogs %>% - dplyr::bind_rows(result) - } - - print(paste("Parsed file:", file)) - }, - error = function(cond) { - message("Error message:") - message(conditionMessage(cond)) - NA - }, - warning = function(cond) { - message("Warning message:") - message(conditionMessage(cond)) - NULL - } - ) - } - - eventLogs <- allLogs %>% - dplyr::mutate(across(where(is.character), as.factor)) %>% - dplyr::mutate(Message = as.character(Message)) %>% - dplyr::filter(substr(Message, 1, 1) == "{") %>% - dplyr::mutate(Message = purrr::map(Message, ~ jsonlite::fromJSON(.) %>% - lapply(., function(x) if (is.null(x)) NA else if (is.list(x)) toString(x) else x) %>% - tibble::as_tibble())) %>% - tidyr::unnest(Message) %>% - dplyr::rename(Message = message) %>% - dplyr::mutate(across(c("file", "errorCode", "warningCode", "functionName"), factor)) - - export_data_as_parquet( - data = eventLogs, - filename = "table_logs", - output_root = paths$tables, - suffix = "" - ) -} - -parseLines <- function(lines) { - rows <- strsplit(lines, "\t") - malformed <- sapply(rows, function(x) length(x) != 6) - rows <- rows[!malformed] - result <- data.frame( - Timestamp = current_timestamp, # current time - # Thread = sapply(rows, function(x) x[2]), - Level = sapply(rows, function(x) x[3]), - # Package = sapply(rows, function(x) x[4]), - # Function = sapply(rows, function(x) x[5]), - Message = sapply(rows, function(x) x[6]) - ) - return(result) -} - -main() diff --git a/scripts/R/run_script_5_create_metadata_table.R b/scripts/R/run_script_5_create_metadata_table.R deleted file mode 100644 index 6b4058b..0000000 --- a/scripts/R/run_script_5_create_metadata_table.R +++ /dev/null @@ -1,57 +0,0 @@ -# current time -current_timestamp <- Sys.time() - -main <- function() { - config <- config::get() - subdirs <- c( - "patient_data_cleaned", - "patient_data_raw", - "product_data_cleaned", - "product_data_raw" - ) - paths <- init_paths(subdirs, delete = FALSE) - - tracker_files <- get_files(paths$tracker_root, pattern = "*.xlsx") - - all_data <- tibble::tibble() - for (file in tracker_files) { - row <- list() - file_path <- file.path(paths$tracker_root, file) - file_name <- tools::file_path_sans_ext(basename(file)) - row$file_name <- file_name - clinic_code <- basename(dirname(file)) - row$clinic_code <- clinic_code - row$md5 <- digest::digest(file_path, algo = "md5", serialize = FALSE) - - for (dir in subdirs) { - matches <- list.files(paths[[dir]], full.names = FALSE) %>% - .[startsWith(., file_name)] - if (length(matches) > 0) { - row[[dir]] <- TRUE - } else { - row[[dir]] <- FALSE - } - } - - - all_data <- all_data %>% - dplyr::bind_rows(tibble::as_tibble(row)) - - print(paste("Parsed file:", file)) - } - - all_data <- all_data %>% - dplyr::mutate( - complete = dplyr::if_all(dplyr::all_of(subdirs), identity), - timestamp = current_timestamp - ) - - export_data_as_parquet( - data = all_data, - filename = "tracker_metadata", - output_root = file.path(paths$output_root, "tables"), - suffix = "" - ) -} - -main() diff --git a/scripts/analyze_logs.sql b/scripts/analyze_logs.sql new file mode 100644 index 0000000..d1f31c8 --- /dev/null +++ b/scripts/analyze_logs.sql @@ -0,0 +1,74 @@ +-- analyze_logs.sql +.mode box.timer on -- Summary Statistics +SELECT + 'Log Summary' as section; + +SELECT + COUNT(*) as total_logs, + COUNT(DISTINCT file_name) as unique_trackers, + MIN(timestamp) as earliest, + MAX(timestamp) as latest +FROM + '/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_python/tables/table_logs.parquet'; + +-- Level Distribution +SELECT + 'Level Distribution' as section; + +SELECT + level, + COUNT(*) as count +FROM + '/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_python/tables/table_logs.parquet' +GROUP BY + level +ORDER BY + count DESC; + +-- Top Errors +SELECT + 'Top 10 Files with Most Errors' as section; + +SELECT + file_name, + COUNT(*) as issues +FROM + '/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_python/tables/table_logs.parquet' +WHERE + level = 'ERROR' +GROUP BY + file_name +ORDER BY + issues DESC +LIMIT + 10; + +SELECT + file_name, + COUNT(*) as issues +FROM + '/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_python/tables/table_logs.parquet' +WHERE + level = 'WARNING' +GROUP BY + file_name +ORDER BY + issues DESC +LIMIT + 10; + +-- Exception Summary +SELECT + 'Exception Types' as section; + +SELECT + exception_type, + COUNT(*) as count +FROM + '/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_python/tables/table_logs.parquet' +WHERE + has_exception = true +GROUP BY + exception_type +ORDER BY + count DESC; \ No newline at end of file diff --git a/scripts/compare_outputs.py b/scripts/compare_outputs.py new file mode 100644 index 0000000..ff67210 --- /dev/null +++ b/scripts/compare_outputs.py @@ -0,0 +1,1354 @@ +#!/usr/bin/env python3 +"""Diff a Python pipeline output directory against the frozen R baseline. + +Migration-only tooling with a defined end-of-life (R's retirement, ticket 12) +-- deliberately not wired into `a4d.cli`. Decoupled from pipeline execution: +takes two existing output directories and diffs them. Logic lives in +`a4d.migration.compare`; this is a thin CLI + report writer. Per stage +(patient/product x raw/cleaned) writes one Excel workbook -- summary sheets +(aggregate counts) plus the actual flagged rows, one sheet per measure. Excel, +not HTML, because triage means loading this as a dataframe, filtering, +sorting, and adding columns -- not just reading a static page. + +Usage: + uv run python scripts/compare_outputs.py \ + --r-dir "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_r" \ + --py-dir "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output_python" \ + --output-dir output/comparison +""" + +import json +import re +import sys +from dataclasses import dataclass +from datetime import UTC, datetime +from enum import Enum, auto +from pathlib import Path +from typing import Annotated + +import polars as pl +import typer +from loguru import logger +from openpyxl import Workbook +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +from a4d.clean.schema import ( + get_date_columns, + get_numeric_columns, +) +from a4d.clean.schema import ( + get_string_columns as get_patient_string_columns, +) +from a4d.clean.validators import load_validation_rules +from a4d.migration.compare import ( + BUDDHIST_ERA_CONVERSION_CLASSIFIERS, + DERIVED_RUNNING_TOTAL_CLASSIFIERS, + EXCEL_FORMULA_ERROR_CLASSIFIERS, + PATIENT_ABSURD_SERIAL_CLASSIFIERS, + PATIENT_AGE_CASCADE_CLASSIFIERS, + PATIENT_AGE_FROM_BARE_YEAR_CLASSIFIERS, + PATIENT_BARE_YEAR_CLASSIFIERS, + PATIENT_BEYOND_TRACKER_YEAR_CLASSIFIERS, + PATIENT_BUDDHIST_ERA_CLASSIFIERS, + PATIENT_CLINICAL_NOTE_CLASSIFIERS, + PATIENT_DIAGNOSIS_AGE_CLASSIFIERS, + PATIENT_FBG_TEXT_CLASSIFIERS, + PATIENT_FUTURE_DATE_CLASSIFIERS, + PATIENT_GLUCOSE_CASCADE_CLASSIFIERS, + PATIENT_GLUCOSE_UNIT_CLASSIFIERS, + PATIENT_HBA1C_RANGE_CLASSIFIERS, + PATIENT_INSULIN_DRUG_NAME_CLASSIFIERS, + PATIENT_INSULIN_SUBTYPE_CLASSIFIERS, + PATIENT_INSULIN_TOTAL_UNITS_CLASSIFIERS, + PATIENT_INSULIN_TYPE_CLASSIFIERS, + PATIENT_JOIN_SUFFIX_COLLISION_CLASSIFIERS, + PATIENT_MERGED_SUBVALUE_TRIM_CLASSIFIERS, + PATIENT_MONTH_NAME_TRUNCATED_CLASSIFIERS, + PATIENT_NA_UNITE_PADDING_CLASSIFIERS, + PATIENT_NON_LATIN_HEADER_CLASSIFIERS, + PATIENT_R_EXTRACTION_GAP_CLASSIFIERS, + PATIENT_RICHTEXT_SPACE_CLASSIFIERS, + PATIENT_SCREENING_SELECTION_CLASSIFIERS, + PATIENT_STATIC_JOIN_CLASSIFIERS, + PATIENT_UNICODE_SANITIZER_CLASSIFIERS, + PATIENT_UNREADABLE_MONTH_CLASSIFIERS, + PATIENT_UNRECONSTRUCTABLE_DATE_CLASSIFIERS, + PATIENT_UNTRIMMED_VALIDATION_CLASSIFIERS, + PATIENT_YMD_FIRST_CLASSIFIERS, + PRODUCT_CATEGORY_CLASSIFIERS, + PRODUCT_ENTRY_DATE_CLASSIFIERS, + PRODUCT_ROW_ORDER_CLASSIFIERS, + PYTHON_CANONICAL_LABEL_CLASSIFIERS, + R_DATE_ERROR_SENTINEL_CLASSIFIERS, + R_NUMERIC_ERROR_SENTINEL_CLASSIFIERS, + ROW_ORDINAL_COL, + STRAY_DATE_CLASSIFIERS, + STRAY_DATE_DROPPED_CLASSIFIERS, + STRAY_DATE_ZEROED_CLASSIFIERS, + WIDE_FORMAT_FRAGMENT_CLASSIFIERS, + Delta, + DirectoryComparison, + FileComparison, + add_row_ordinal, + align_duplicate_rows, + build_mismatch_rows, + build_summary_rows, + compare_directory, + compute_deltas, + load_glucose_unit_swaps, + normalize_boolean_literal_column, + normalize_date_column, + normalize_numeric_column, + normalize_whitespace_column, + numeric_normalize_targets, + snapshot_from_summary, + static_join_missed_ids, + string_numeric_normalize_targets, + summarize_directory, + whitespace_normalize_targets, +) + +# normalize_date_column (via a4d.clean.date_parser.parse_date_flexible) logs +# at DEBUG per parsed value; loguru's default sink has no level filter, so +# without this every date cell would spam the console. This tool never calls +# a4d.logging.setup_logging() (no pipeline run, no output_root to log into). +# +# a4d.clean.* is silenced entirely, not just below WARNING: the date parser +# also warns once per value it cannot parse, which in the *pipeline* is a +# real data-quality signal but here is pure noise -- this tool feeds it raw +# free-text columns precisely to normalize the parseable ones, so +# unparseable cells are expected, not actionable, and at 18 date columns x +# ~245 files they drown the report the tool exists to print. +logger.remove() +logger.add( + sys.stderr, + level="WARNING", + filter=lambda record: not record["name"].startswith("a4d.clean"), +) + +console = Console() +app = typer.Typer() + + +class Sentinel(Enum): + """Marks a normalization that resolves against each frame's own columns.""" + + ALL_RAW_COLUMNS = auto() + ALL_STRING_COLUMNS = auto() + + +ALL_RAW_COLUMNS = Sentinel.ALL_RAW_COLUMNS +ALL_STRING_COLUMNS = Sentinel.ALL_STRING_COLUMNS + +PATIENT_KEY_COLS = ["patient_id", "sheet_name"] +PATIENT_ID_COL = "patient_id" +# Fields with allowed-value validation in reference_data/data_cleaning.yaml -- +# the genuinely categorical patient columns, not free text. +PATIENT_CATEGORICAL_COLS = [ + "analog_insulin_long_acting", + "analog_insulin_rapid_acting", + "clinic_visit", + "complication_screening_eye_exam_value", + "complication_screening_foot_exam_value", + "dm_complication_eye", + "dm_complication_kidney", + "dm_complication_others", + "hospitalisation_cause", + "human_insulin_intermediate_acting", + "human_insulin_pre_mixed", + "human_insulin_short_acting", + "insulin_regimen", + "insulin_type", + "insulin_subtype", + "observations_category", + "patient_consent", + "province", + "remote_followup", + "status", + "support_level", +] + +# Raw-stage-only, derived from the cleaned schema's own pl.Date columns +# (ticket 23): the same representation gap ticket 20 found and fixed for +# product's raw product_entry_date -- R's raw extraction stores the +# unparsed source text (an Excel serial for date-formatted cells) while +# Python's raw extraction already ISO-formats parsed dates. Cleaned stage +# never needs this since both sides are already parsed dates there. +# meter_received_date (ticket 27) is appended by hand: it's a raw-only +# column absent from the cleaned schema entirely (no cleaned-stage +# equivalent), so get_date_columns() can't see it, but it carries the same +# raw-serial-vs-parsed representation gap -- verified against real drive +# data (1,761 mismatches collapsed to the same shape as the derived columns). +# complication_screening_date (ticket 48) is appended for the same reason: the +# cleaned schema splits screening dates per test (eye/foot/kidney/...), so the +# generic raw column has no cleaned-stage equivalent for get_date_columns() to +# find. It only became visible once ticket 48's merged-header fix started +# extracting the column at all. +PATIENT_RAW_DATE_NORMALIZE_COLS = [ + *get_date_columns(), + "meter_received_date", + "complication_screening_date", +] + +# Raw-stage-only (ticket 27, extending ticket 22's product_balance +# precedent to patient): R's own float-to-string conversion rounds a raw +# numeric column's trailing digits differently than Python's -- a +# representation difference cleaning's own type-casting already resolves. +# +# Ticket 43 widened this from get_numeric_columns() to every raw column: that +# list +# is derived, but from the wrong schema -- it cannot name the Patient List +# join's ".static" copies (fbg_baseline_mg.static, fbg_baseline_mmol.static) +# and it types a screening measurement as a string because the column can +# also hold "normal" (complication_screening_*_value). All four hold plain +# floats, and all four were reported as mismatches purely because R and +# Python round a float's string form differently. Measured before widening: +# across all 27,980 raw-stage mismatches, exactly 434 have both sides parsing +# to the same float, and all 434 sit in those four columns -- so this widens +# equality without changing anything else. See numeric_normalize_targets. +PATIENT_RAW_NUMERIC_NORMALIZE_COLS = ALL_RAW_COLUMNS + +# Ordinal position within (clinic_id, product_sheet_name) -- see +# add_row_ordinal's docstring. Replaces the old equi-join key (clinic_id, +# product, product_sheet_name, product_entry_date), which collapsed onto far +# fewer distinct values than rows exist wherever product_entry_date is null +# (ticket 17). +PRODUCT_ORDINAL_GROUP_COLS = ["clinic_id", "product_sheet_name"] +PRODUCT_ID_COL = "product" +PRODUCT_TRACKER_YEAR_COL = "product_table_year" +PRODUCT_CATEGORICAL_COLS = ["product_category", "product_balance_status"] + +# Raw-stage-only (ticket 24, extending ticket 22's product_balance precedent): +# R's own float-to-string conversion rounds a raw numeric column's trailing +# digits differently -- same representation gap product_balance already had, +# just not yet wired for these three quantity columns. +PRODUCT_RAW_NUMERIC_NORMALIZE_COLS = [ + "product_balance", + "product_units_received", + "product_units_released", + "product_received_from", +] + +# Raw-stage-only (ticket 22): readxl's trim_ws=TRUE default strips whitespace +# R-side that openpyxl-based Python extraction preserves as-is. Every raw +# string column except product_entry_date (date-normalized instead) and +# product_balance (numeric-normalized instead). +PRODUCT_RAW_WHITESPACE_NORMALIZE_COLS = [ + "product", + "product_received_from", + "product_released_to", + "product_remarks", + "product_units_received", + "product_units_released", + "product_units_returned", + "product_returned_by", +] + +# ticket 21: unlike the other raw-stage-only normalizations, an embedded +# \r\n-vs-\n line break survives cleaning (step 2.16's str.strip_chars only +# trims ends) and still shows up on `product` at the cleaned stage -- +# verified directly against the real R/Python cleaned output pair (e.g. +# "FastClix \r\nLancets" vs "FastClix \nLancets"). Scoped to `product` alone: +# it's the only cleaned-stage column found carrying multi-line values: rest +# were already fully explained by PRODUCT_ROW_ORDER_CLASSIFIERS. +# ticket 36 extends this to the two identifier columns. Both pipelines now +# trim every string cell; R trims only product_released_to, so R keeps a +# stray space the source really carries (sheet tab "Dec24 ", file +# "... - final .xlsx"). A settled representation difference, so the report +# should stop counting it as divergence. +PRODUCT_CLEANED_WHITESPACE_NORMALIZE_COLS = ["product", "product_sheet_name", "file_name"] + +# ticket 36: patient cleaning now trims too, and sheet_name is half the +# patient row-alignment key -- without this, R's untrimmed "Dec24 " would +# fail to join Python's "Dec24" and drop those rows from the comparison +# entirely rather than showing them as equal. +# Derived from the patient schema rather than hand-listed. R's cleaned patient +# output keeps the trailing whitespace its source cells carry (measured: ~3,750 +# cells across edu_occ, observations, name, family_history and others), which +# Python's cleaning now strips -- a settled representation difference, and the +# same treatment ticket 22 gave product's raw stage. sheet_name matters most: +# it is half the patient row-alignment key, so without this R's untrimmed +# "Dec24 " would fail to join Python's "Dec24" and drop those rows from the +# comparison entirely rather than showing them as equal. +PATIENT_WHITESPACE_NORMALIZE_COLS = get_patient_string_columns() + +# Ticket 49 widens this to every string column for the *raw* stage only, on +# the same grounds ticket 43 widened the numeric list: get_patient_string_columns() +# reads the cleaned schema, which cannot see a raw-only column +# (dm_complications, no cleaned equivalent) and types as Float64 columns the +# raw stage still holds as text (insulin_injections, hba1c_updated). Those +# three were the entire residual of readxl's trim_ws and \r\n line endings at +# the raw stage -- 40 of the 229 unclassified mismatches. The cleaned stage +# keeps the schema-derived list: there the dtypes are real, and a column +# typed Float64 genuinely holds floats rather than text. +PATIENT_RAW_WHITESPACE_NORMALIZE_COLS = ALL_RAW_COLUMNS + +# Ticket 54: cleaning casts its numeric columns, so the cleaned stage needs +# numeric normalization only where the schema keeps a measurement as a string +# (a screening value that can also read "normal"). Resolved per frame against +# the frame's own dtypes -- see string_numeric_normalize_targets. +PATIENT_CLEANED_NUMERIC_NORMALIZE_COLS = ALL_STRING_COLUMNS + +CLASSIFIERS_BY_COLUMN = { + # ticket 28: R's static "Patient List" recruitment-date extraction fails + # to populate recruitment_date for most patients even where the tracker + # plainly records one -- verified against the real source Excel. + "recruitment_date": PATIENT_R_EXTRACTION_GAP_CLASSIFIERS, + # ticket 37: two verified R gaps on the same column -- the leading-space + # defeat of R's own "Updated 2022" header fixup on 2022 trackers, and the + # 2026 template's new "Annual" sheet, which R reads nothing from. Both + # leave R null where Python has a source-verified value, so both land on + # r_extraction_gap; the future-date cause covers the reverse direction. + "edu_occ_updated": PATIENT_FUTURE_DATE_CLASSIFIERS | PATIENT_R_EXTRACTION_GAP_CLASSIFIERS, + # ticket 28: R's own allowed-values validator rejects the multi-insulin + # CSV output R's own derivation logic produces for 2024+ trackers -- + # already documented as a deliberate Python correction in + # _derive_insulin_fields's docstring (src/a4d/clean/patient.py). + "insulin_subtype": PATIENT_INSULIN_SUBTYPE_CLASSIFIERS, + # ticket 42: Python resolves glucose readings recorded under the wrong + # unit's header (a decision taken on A4D's medical advisor's answer); R has + # no unit resolution, so every correction shows as a divergence. The + # baseline-mg join question is a separate, still-open population, which is + # why the classifier never fires on an R-null cell. + # ticket 46 adds the rich-text space cause; it is disjoint from the unit + # correction, which only fires where both sides parse as numbers. + "fbg_updated_mg": PATIENT_GLUCOSE_UNIT_CLASSIFIERS | PATIENT_RICHTEXT_SPACE_CLASSIFIERS, + "fbg_updated_mmol": PATIENT_GLUCOSE_UNIT_CLASSIFIERS, + # ticket 31: the three canonical columns whose source sub-columns both + # pipelines unite into one value. R's tidyr::unite pads absent + # sub-columns with the literal string "NA"; Python skips them. Derived, + # not guessed: these are the only targets that form a duplicate group + # anywhere in the 254-tracker set. + # ticket 50 adds the multi-select cause: where the same merged block holds + # several "(Select)" sub-columns, R's duplicate-name suffixing keeps only + # the first. Checked after the padding cause, which is the more specific of + # the two (the shapes are disjoint -- R's value carries no NA token here). + "complication_screening": ( + PATIENT_NA_UNITE_PADDING_CLASSIFIERS | PATIENT_SCREENING_SELECTION_CLASSIFIERS + ), + "latest_complication_screenning": PATIENT_NA_UNITE_PADDING_CLASSIFIERS, + # ticket 46: two whitespace causes in opposite directions -- readxl + # dropping a whitespace-only rich-text run (verified in the source XML of + # 2017 Yangon and 2022 Mahosot) and Python's own trimming of each + # sub-value before the merge ticket 31 added. Both are direction-scoped, + # so their order relative to each other does not matter. + "observations": ( + PATIENT_NA_UNITE_PADDING_CLASSIFIERS + | PATIENT_RICHTEXT_SPACE_CLASSIFIERS + | PATIENT_MERGED_SUBVALUE_TRIM_CLASSIFIERS + ), + "hba1c_updated": PATIENT_RICHTEXT_SPACE_CLASSIFIERS, + # ticket 46: R reads no header at all for the 2021 Kantha Bopha column + # holding the regimen (verified: Mar21!Q, both header rows empty, data + # "Self-mixed BD"), so it drops the column; Python recovers the name from + # the sibling sheets that label it (ticket 30). Scoped here rather than + # left to the generic gap classifier because that one file is the whole + # of the column's residual. + "insulin_regimen": PATIENT_R_EXTRACTION_GAP_CLASSIFIERS, + # ticket 37: the sibling insulin_type column has a different cause -- + # R's ifelse propagates NA from a blank human-insulin column and loses + # a type the analog columns plainly state. + "insulin_type": PATIENT_INSULIN_TYPE_CLASSIFIERS, + # ticket 36: R rejects a source value for its trailing whitespace alone + # and sentinels it; Python now trims before validating and keeps it. + "sex": PATIENT_UNTRIMMED_VALIDATION_CLASSIFIERS, + # ticket 27: bmi and t1d_diagnosis_age are formula-derived in the source + # trackers -- a source formula error (height 0, an unparseable date) + # leaves R's raw extraction holding the literal Excel error string, + # while Python's raw extraction (openpyxl, data_only) has no cached + # value to carry and returns null. age shares the same shape on a + # handful of rows. Verified against the real drive data. + "bmi": EXCEL_FORMULA_ERROR_CLASSIFIERS, + # ticket 50 adds the stray-date cause: two trackers record a diagnosis + # *date* in this numeric column. Disjoint from the formula-error cause, + # which needs an Excel error string on R's side. + "t1d_diagnosis_age": EXCEL_FORMULA_ERROR_CLASSIFIERS | STRAY_DATE_CLASSIFIERS, + "age": EXCEL_FORMULA_ERROR_CLASSIFIERS, + # ticket 27: source-data typos where a clinician entered a Thai + # Buddhist-Era year into a Gregorian date cell -- see + # PATIENT_BUDDHIST_ERA_CLASSIFIERS's docstring. Every raw-stage date + # column found carrying this pattern against the real drive data. + "bmi_date": PATIENT_BUDDHIST_ERA_CLASSIFIERS, + "hba1c_updated_date": PATIENT_BUDDHIST_ERA_CLASSIFIERS, + # ticket 37: also the R extraction gap on the 2026 template's new + # "Annual" sheet -- checked after the Buddhist-era cause, which is the + # more specific of the two (the shapes are disjoint either way). + "blood_pressure_updated": ( + PATIENT_BUDDHIST_ERA_CLASSIFIERS | PATIENT_R_EXTRACTION_GAP_CLASSIFIERS + ), + # ticket 50: 2019 Preah Kossamak's Jul19/Aug19 sheets lost the header merge + # that qualifies the FBG "Date" sub-header, so R is left with a bare `date` + # column it cannot map -- see _is_r_extraction_gap. That one file is the + # whole of the column's residual. + "fbg_updated_date": (PATIENT_BUDDHIST_ERA_CLASSIFIERS | PATIENT_R_EXTRACTION_GAP_CLASSIFIERS), + # ticket 49: 2022 Mukdahan appends Thai translations to both headers, which + # R's Unicode-aware sanitizer keeps and its exact match then misses -- see + # _is_r_non_latin_header_miss. Python recovers 44 real dates R drops. + "last_clinic_visit_date": ( + PATIENT_BUDDHIST_ERA_CLASSIFIERS | PATIENT_NON_LATIN_HEADER_CLASSIFIERS + ), + "last_remote_followup_date": PATIENT_NON_LATIN_HEADER_CLASSIFIERS, + # ticket 49, extending ticket 37's verified 2026 finding to the columns it + # did not reach: the 2026 template carries these on its new "Annual" sheet, + # which R reads no values from. Verified per column against the real source + # Excel (2026 ISDFI, Annual!E32/H32/I32/AD32 for PH_IS022: "college + # graduate", 80, 50, "for cataract surgery" -- exactly Python's values); + # every affected row across all five files is a 2026 tracker. + "blood_pressure_sys_mmhg": PATIENT_R_EXTRACTION_GAP_CLASSIFIERS, + "blood_pressure_dias_mmhg": PATIENT_R_EXTRACTION_GAP_CLASSIFIERS, + "edu_occ": PATIENT_R_EXTRACTION_GAP_CLASSIFIERS, + "other_issues": PATIENT_R_EXTRACTION_GAP_CLASSIFIERS, + "status": PATIENT_R_EXTRACTION_GAP_CLASSIFIERS, + # ticket 50: the 2022 Kantha Bopha header opens with thirteen spaces, which + # defeats R's name sanitizer the way ticket 37's "Updated 2022" leading + # space did -- both columns of the block, so both carry the gap cause. + # ticket 39: the clinical-note causes go last, so a cell explained by a + # specific mechanism above is not swallowed by the broad "R sentinels a + # note it cannot read" shape. + "hospitalisation_date": ( + PATIENT_BUDDHIST_ERA_CLASSIFIERS + | PATIENT_R_EXTRACTION_GAP_CLASSIFIERS + | PATIENT_CLINICAL_NOTE_CLASSIFIERS + ), + "hospitalisation_cause": PATIENT_R_EXTRACTION_GAP_CLASSIFIERS, + # ticket 50: 2025 LWCH leaves the current-month visit column unheaded in + # both header rows, so R drops it and Python recovers the name from the + # sibling month sheets (ticket 30). Disjoint from ticket 49's boolean + # spelling fold, which never leaves R null. + "clinic_visit": PATIENT_R_EXTRACTION_GAP_CLASSIFIERS, + # ticket 50: a date typed into a non-date column. R's readxl coerces the + # whole column to its majority numeric type and carries the raw Excel + # serial; Python's openpyxl honours the cell's own date format. Verified + # source-side per column (2023 Chiang Mai Patient List!H14 and 2023 Yangon + # General Patient List!H76, both `mmm-yy`/`d-mmm-yyyy` formatted; 2020 + # Mahosot Nov20!R152; 2021 Khon Kaen Mar21!P60) -- the same + # openpyxl_date_typed_stray_cell ticket 24 root-caused on the product arm, + # and a source defect reported for ticket 40 rather than a pipeline change. + "blood_pressure_mmhg": STRAY_DATE_CLASSIFIERS, + "testing_frequency": STRAY_DATE_CLASSIFIERS, + "complication_screening_kidney_test_date": PATIENT_BUDDHIST_ERA_CLASSIFIERS, + "complication_screening_lipid_profile_date": PATIENT_BUDDHIST_ERA_CLASSIFIERS, + "complication_screening_thyroid_test_date": PATIENT_BUDDHIST_ERA_CLASSIFIERS, + "complication_screening_eye_exam_date": PATIENT_BUDDHIST_ERA_CLASSIFIERS, + # ticket 36: the entry-date-specific causes are checked first (they are + # source-verified and stage-specific), then the row-order fallback every + # positionally-aligned product column carries. R's readxl nulls the + # *text*-formatted cells of a mixed-type Entry Date column (verified: 2022 + # Vietnam National Children's Hospital, Apr22, where "20/04/2022" is + # stored as text and "2022-04-01" as a datetime), which drops R back to + # input-order sorting -- so both sides hold a real, different date and + # `r_value_missing` never fires. + # ticket 61 leads: a Buddhist-era entry date the cleaned stage converts is + # otherwise claimed by python_out_of_window_date_preserved, whose test the + # R side genuinely passes. + "product_entry_date": ( + BUDDHIST_ERA_CONVERSION_CLASSIFIERS + | PRODUCT_ENTRY_DATE_CLASSIFIERS + | PRODUCT_ROW_ORDER_CLASSIFIERS + ), + "product_category": PRODUCT_CATEGORY_CLASSIFIERS | PRODUCT_ROW_ORDER_CLASSIFIERS, + # ticket 21: cleaned-stage columns whose mismatches are dominated by a + # within-(clinic, sheet) sort-order divergence, not genuine content + # differences -- see PRODUCT_ROW_ORDER_CLASSIFIERS's docstring. + # ticket 36: the endpoint test comes first -- it is the order-independent + # evidence for a derived running total, where the membership heuristic + # under-detects by construction (27.9% caught, ticket 21's "future work"). + "product_balance": DERIVED_RUNNING_TOTAL_CLASSIFIERS | PRODUCT_ROW_ORDER_CLASSIFIERS, + # ticket 24: also carries the raw-stage stray-date-typed-cell cause -- + # merged since CLASSIFIERS_BY_COLUMN holds one registry per column across + # every stage. Stray-date checked first: row_order_candidate is a loose + # "this value appears elsewhere in the group" heuristic that false-fires + # on common small values like "0", so it must not shadow the more + # specific, source-verified stray-date match. + "product_received_from": STRAY_DATE_CLASSIFIERS | PRODUCT_ROW_ORDER_CLASSIFIERS, + "product_released_to": PRODUCT_ROW_ORDER_CLASSIFIERS, + "product_remarks": PRODUCT_ROW_ORDER_CLASSIFIERS, + # ticket 36: STRAY_DATE_ZEROED_CLASSIFIERS is the cleaned-stage face of the + # same cause and must, like the raw-stage one, precede the loose row-order + # heuristic -- "0" appears all over a group, so row_order_candidate + # false-fires on it. + "product_units_received": STRAY_DATE_CLASSIFIERS + | STRAY_DATE_ZEROED_CLASSIFIERS + | PRODUCT_ROW_ORDER_CLASSIFIERS, + "product": PRODUCT_ROW_ORDER_CLASSIFIERS, + # ticket 36: the remaining schema columns, wired so no positionally + # aligned column is left without the row-order fallback. + "product_units_notes": PRODUCT_ROW_ORDER_CLASSIFIERS, + "product_units_returned": STRAY_DATE_CLASSIFIERS | PRODUCT_ROW_ORDER_CLASSIFIERS, + "product_returned_by": PRODUCT_ROW_ORDER_CLASSIFIERS, + "product_balance_status": PRODUCT_ROW_ORDER_CLASSIFIERS, + "orig_product_released_to": PRODUCT_ROW_ORDER_CLASSIFIERS, + "product_unit_capacity": PRODUCT_ROW_ORDER_CLASSIFIERS, + # ticket 24: raw-stage wide-format comma/hyphen split ambiguity (2017-2019 + # Mandalay files). ticket 25 adds the row-order cause for the cleaned + # stage, which this column shares with every sibling above -- it was + # omitted when ticket 21 named six columns and not this one, leaving + # 2,144 cleaned-stage mismatches reported as unclassified. Wide-format + # first (it is source-verified and stage-specific); measured against the + # real drive data, it matches nothing at the cleaned stage, so the order + # only guards the raw stage's existing classification. + "product_units_released": WIDE_FORMAT_FRAGMENT_CLASSIFIERS | PRODUCT_ROW_ORDER_CLASSIFIERS, + # ticket 29: R's insulin-column dedup grep deletes "TOTAL Insulin Units" + # from every 2024+ tracker before it is read, so R's column is null on all + # 81,859 rows while Python reads it correctly. + "insulin_total_units": PATIENT_INSULIN_TOTAL_UNITS_CLASSIFIERS, + # ticket 29: R's Patient List join suffixes both sides on a name + # collision, so R's cleaning finds no unsuffixed column and leaves these + # null for the whole file (21 files / 3 files respectively). + # ticket 42 is checked first and is disjoint from the join question: it + # never fires on an R-null cell, which is the whole of the join shape. + "fbg_baseline_mg": ( + PATIENT_GLUCOSE_UNIT_CLASSIFIERS | PATIENT_JOIN_SUFFIX_COLLISION_CLASSIFIERS + ), + "fbg_baseline_mmol": ( + PATIENT_GLUCOSE_UNIT_CLASSIFIERS | PATIENT_JOIN_SUFFIX_COLLISION_CLASSIFIERS + ), +} + +# ticket 29: every column whose validation rules declare canonical-label +# aliases diverges from R by design -- R's config lists the retired spelling +# as an allowed value and its first-match lookup collapses onto it, while +# Python folds retired spellings into one canonical label. Derived from the +# same config that produces the values, so a future alias needs no edit here. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | PYTHON_CANONICAL_LABEL_CLASSIFIERS + for col, spec in load_validation_rules().items() + if isinstance(spec, dict) and spec.get("aliases") +} + +# ticket 29: R's 999999 numeric sentinel is not a property of any one column +# -- R stamps it on any numeric cell whose source text failed to parse -- so +# it is derived from the patient schema's own numeric column list rather than +# hand-listed. Appended after each column's specific causes so a +# source-verified, column-specific classifier still wins the first match. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | R_NUMERIC_ERROR_SENTINEL_CLASSIFIERS + for col in get_numeric_columns() +} + +# ticket 38: the same argument on the date path -- R stamps 9999-09-09 on any +# date cell that recorded an absence its is.na() check does not recognize +# (literal "NA", "-", "Nil"), so like its numeric twin the cause belongs to +# every date column rather than to any one of them. Derived from the schema's +# own date column list, and appended last so a column-specific, +# source-verified cause still wins the first match. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | R_DATE_ERROR_SENTINEL_CLASSIFIERS + for col in get_date_columns() +} + +# ticket 51: R's ymd-before-dmy parse order, which mis-reads any D.M.YY source +# string. Same argument again -- parse_date_string is R's one date entry point, +# so the cause belongs to every date column, not to the five that happen to +# carry it in the current tracker set. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | PATIENT_YMD_FIRST_CLASSIFIERS + for col in get_date_columns() +} + +# ticket 60: a junk Excel serial published as a year-3000-plus date is not +# evidence about R -- it was reaching r_parse_order_cannot_read_cell, which +# asserts R's parse orders failed, when R had simply carried the serial +# through as a string. Prepended rather than appended because the cause it +# must out-rank is itself schema-derived; safe to run first because its own +# Buddhist-era band check declines a genuine BE year. +CLASSIFIERS_BY_COLUMN |= { + col: PATIENT_ABSURD_SERIAL_CLASSIFIERS | CLASSIFIERS_BY_COLUMN.get(col, {}) + for col in get_date_columns() +} + +# ticket 52: a bare year typed into a date cell, which Python now resolves as +# the year and R still reads as a 1905 Excel serial. parse_date_flexible is +# Python's one date entry point, so the cause belongs to every date column +# rather than to the two that carry it in the current tracker set. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | PATIENT_BARE_YEAR_CLASSIFIERS + for col in get_date_columns() +} + +# ticket 52: the two ages derived from those dates. Named explicitly rather +# than derived from a list: these are exactly the columns _fix_age and +# _fix_t1d_diagnosis_age compute from dob and t1d_diagnosis_date. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | PATIENT_AGE_FROM_BARE_YEAR_CLASSIFIERS + for col in ("age", "t1d_diagnosis_age") +} + +# ticket 51: Python's own beyond-tracker-year guard (_validate_dates), which +# runs over get_date_columns() itself -- so the cause has exactly that domain +# by construction, not by observation. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | PATIENT_BEYOND_TRACKER_YEAR_CLASSIFIERS + for col in get_date_columns() +} + +# ticket 54: R never derives a diagnosis age at all (its call site is +# commented out), and a date typed into that column reaches R as a serial. +# Scoped to the one column _fix_t1d_diagnosis_age computes, and appended after +# PATIENT_AGE_FROM_BARE_YEAR_CLASSIFIERS so the bare-year subset -- which is +# the same mechanism already decided under a narrower, row-flagged test -- +# keeps its own name rather than being absorbed by the general one. +CLASSIFIERS_BY_COLUMN["t1d_diagnosis_age"] = ( + CLASSIFIERS_BY_COLUMN.get("t1d_diagnosis_age", {}) | PATIENT_DIAGNOSIS_AGE_CLASSIFIERS +) + +# ticket 55: the 2024+ insulin tick boxes that one clinic ticks by naming the +# drug. Appended after the existing insulin_subtype causes so the narrower +# multi-value shape keeps its own name. +CLASSIFIERS_BY_COLUMN["insulin_subtype"] = ( + CLASSIFIERS_BY_COLUMN.get("insulin_subtype", {}) | PATIENT_INSULIN_DRUG_NAME_CLASSIFIERS +) + +# ticket 55: R's fix_fbg manufactures a reading from text and cannot read a +# reading that carries its unit. Scoped to the two mg/dL columns fix_fbg runs +# over -- the mmol pair is ticket 44's, and its R-null shape is a different +# mechanism. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | PATIENT_FBG_TEXT_CLASSIFIERS + for col in ("fbg_updated_mg", "fbg_baseline_mg") +} + +# ticket 44: the mmol columns are derived from their mg siblings, so an mg +# cell's divergence restates itself next door. Appended last of all the glucose +# causes, because it names a cascade rather than a mechanism -- anything that +# can explain the mmol cell in its own right must claim it first. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | PATIENT_GLUCOSE_CASCADE_CLASSIFIERS + for col in ("fbg_updated_mmol", "fbg_baseline_mmol") +} + +# ticket 56: R's own date entry point again -- the destructive month-name +# truncation in parse_dates and the order list it falls through afterwards. Like +# every other parse_dates cause the domain is the whole date family, not the +# columns that happen to carry it today. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | PATIENT_MONTH_NAME_TRUNCATED_CLASSIFIERS + for col in get_date_columns() +} + +# ticket 56: the unreadable-month half of the same mechanism. hospitalisation_date +# is excluded by name: round 6 measured its residual as 100% clinical notes, so +# an R sentinel there means R could not read a sentence -- ticket 39's open +# question, and a different cause wearing the same shape. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | PATIENT_UNREADABLE_MONTH_CLASSIFIERS + for col in get_date_columns() + if col != "hospitalisation_date" +} + +# ticket 57: the other face of the same parse_dates mechanism -- R publishing a +# date from a token no reading can be recovered from, where Python declines. +# Same domain argument and the same hospitalisation_date carve-out, and +# appended after every column-specific and every reconstructable cause, because +# its test is direction alone: it must only ever see what nothing narrower +# claimed. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | PATIENT_UNRECONSTRUCTABLE_DATE_CLASSIFIERS + for col in get_date_columns() + if col != "hospitalisation_date" +} + +# ticket 57: a fasting-glucose reading typed into the HbA1c column, which R has +# no range check to catch. Scoped to the two columns validation_rules.yaml +# declares an HbA1c range for -- the classifier reads that same config, so the +# two cannot drift. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | PATIENT_HBA1C_RANGE_CLASSIFIERS + for col in ("hba1c_baseline", "hba1c_updated") +} + +# ticket 57: the third face of the stray-date cause (ticket 24's raw shape, +# ticket 36's zeroed shape). Where the column's cleaned type is an integer, +# Python's failed cast leaves null rather than 0, which neither existing test +# can see. +CLASSIFIERS_BY_COLUMN["testing_frequency"] = ( + CLASSIFIERS_BY_COLUMN.get("testing_frequency", {}) | STRAY_DATE_DROPPED_CLASSIFIERS +) + +# ticket 51: the two sanitizers disagree on accented letters, so the cause +# belongs to every allowed-value column rather than to province alone -- it is +# only province today because province is the one whose canonical values carry +# accents. Appended last so a column-specific cause still wins the first match. +CLASSIFIERS_BY_COLUMN |= { + col: CLASSIFIERS_BY_COLUMN.get(col, {}) | PATIENT_UNICODE_SANITIZER_CLASSIFIERS + for col in PATIENT_CATEGORICAL_COLS +} + + +# ticket 61: the cleaned stage's Buddhist-era conversion, which runs over +# get_date_columns() itself -- so, like ticket 51's future-date guard, the +# cause has exactly that domain by construction. Prepended rather than +# appended: its test is the tightest of any date cause (an exact 543-year +# shift, same month and day), and several broader causes -- R sentinelling, +# a note read differently -- match a converted cell too. +CLASSIFIERS_BY_COLUMN |= { + col: BUDDHIST_ERA_CONVERSION_CLASSIFIERS | CLASSIFIERS_BY_COLUMN.get(col, {}) + for col in get_date_columns() +} + +# ticket 58: Python now attaches the Patient List and Annual sheets on the +# normalized ID; R still joins on the raw one, so a month row spelled +# LA-MH056 against a Patient List entry of LA_MH056 comes out null on R's side +# for every static column at once. Prepended, because r_extraction_gap's +# column-naming shape matches these cells too and would otherwise absorb them +# under the wrong reason. Declines on the cleaned stage by construction: both +# pipelines normalize patient_id there, so the discriminator is gone. +# +# The column list is explicit rather than "every patient column". Wiring it +# corpus-wide was tried and measured first, and it over-claimed on exactly the +# three columns the month sheets *also* carry: t1d_diagnosis_age (1,069 rows +# against a join-miss population of 679), bmi (144) and age (1). For those, R's +# null is not necessarily the join's fault -- R read the month sheet too -- so +# they keep their existing causes and the residue is ticket 63's. Every column +# below fires on exactly the join-miss population, matching the measured +# per-column delta cell for cell. +PATIENT_STATIC_JOIN_COLUMNS = [ + "blood_pressure_dias_mmhg", + "blood_pressure_sys_mmhg", + "blood_pressure_updated", + "dob", + "edu_occ", + "edu_occ_updated", + "fbg_baseline_mg", + "fbg_baseline_mmol", + "hba1c_baseline", + "lost_date", + "other_issues", + "patient_consent", + "province", + "recruitment_date", + "sex", + "status_out", + "t1d_diagnosis_date", + "t1d_diagnosis_with_dka", +] +CLASSIFIERS_BY_COLUMN |= { + col: PATIENT_STATIC_JOIN_CLASSIFIERS | CLASSIFIERS_BY_COLUMN.get(col, {}) + for col in PATIENT_STATIC_JOIN_COLUMNS +} + +# ticket 63: age is the one column ticket 58 measured as over-claimed and left +# out of the list above, because the month sheets carry an Age cell of their +# own -- so R's value is not necessarily the join's fault. Its own cause goes +# on separately, and *appended* rather than prepended: age already has +# python_age_from_bare_year, whose mechanism (a bare year read as a date) is +# the tighter test and must keep winning where both apply. +CLASSIFIERS_BY_COLUMN["age"] = ( + CLASSIFIERS_BY_COLUMN.get("age", {}) | PATIENT_AGE_CASCADE_CLASSIFIERS +) + + +class RowAlignment(Enum): + """How an arm's R and Python rows are paired for cell-by-cell comparison.""" + + # Product (ticket 17): no usable natural key -- product_entry_date is null + # on many rows -- so a row is identified purely by its position within its + # sheet. Nothing checks that the two paired rows describe the same thing. + POSITIONAL = auto() + # Patient (ticket 45): patient_id is the real key and sheet_name is the + # data's monthly granularity, so rows pair only when both agree. The + # ordinal breaks ties in the handful of sheets that list a patient twice, + # and the tie is broken by content rather than position, because within + # one patient-and-sheet group the row order carries no meaning. + IDENTITY = auto() + + +@dataclass(frozen=True) +class Stage: + """One comparable output directory and how its rows are aligned. + + Every field is named rather than positional (ticket 45): how an arm aligns + its rows decides whether the comparison still checks identity at all, and + that is too load-bearing to read off tuple position. + """ + + label: str + subdir: str + id_col: str + categorical_cols: list[str] + alignment: RowAlignment + # The columns add_row_ordinal groups by before appending a within-group + # ordinal: a whole sheet for POSITIONAL, the identity key itself for + # IDENTITY. + ordinal_group_cols: list[str] + date_normalize_cols: list[str] | None = None + numeric_normalize_cols: list[str] | Sentinel | None = None + whitespace_normalize_cols: list[str] | Sentinel | None = None + # Raw-stage-only (ticket 49): readxl writes an Excel boolean as FALSE, + # openpyxl as False. Cleaning already canonicalizes both, so the cleaned + # stage never sees it. + normalize_boolean_literals: bool = False + # Cleaned-stage-only (ticket 44): resolve_glucose_units runs during + # cleaning, so a swapped column is a fact about cleaned output. The raw + # stage holds the workbook's own headers, where nothing has moved yet. + load_glucose_unit_swaps: bool = False + # Cleaned-stage-only (ticket 63): the raw stage tells a join-missed row + # apart by its own ID spelling, which cleaning normalizes away. Naming + # the raw subdir here rather than deriving it from `subdir` keeps the + # dependency between two stages explicit instead of implied by a suffix. + static_join_missed_ids_from: str | None = None + # The column holding the tracker's own calendar year, if this stage has one + # (ticket 32). It lets a classifier judge whether Python's own value is + # plausible for the tracker, not just how the two sides differ. + tracker_year_col: str | None = None + + @property + def detect_row_order_divergence(self) -> bool: + """compare_cells' order_group_cols diagnostic (ticket 21): does a + mismatched value appear elsewhere in its group -- i.e. are R and Python + ordering the same rows differently rather than holding different data? + Only meaningful for a positional key over a whole sheet; under IDENTITY + a group is one patient on one sheet, where the same test would label + noise as understood.""" + return self.alignment is RowAlignment.POSITIONAL + + +# Raw and cleaned are compared separately so a divergence can be localized to +# extraction vs. cleaning; categorical columns listed here that don't exist yet +# at the raw stage are silently skipped by compare_categorical_overlap. +# date_normalize_cols is raw-stage-only (ticket 20): R's raw extraction +# stores unparsed source date text (an Excel serial for date-formatted +# cells) while Python's raw extraction already ISO-formats parsed dates -- +# a representation difference, not a real divergence, that the cleaned +# stage never has since both sides are already parsed dates there. +# numeric_normalize_cols is raw-stage-only (ticket 22): R's own +# float-to-string conversion rounds a raw numeric column's trailing digits +# differently, a representation difference cleaning's own type-casting (step +# 2.16) already resolves. whitespace_normalize_cols mostly is too (readxl's +# trim_ws=TRUE trims R-side what Python's raw extraction preserves), but an +# embedded \r\n-vs-\n line break isn't touched by cleaning's end-trim-only +# str.strip_chars and survives into `product` at the cleaned stage (ticket 21). +STAGES = [ + Stage( + label="Patient (raw)", + subdir="patient_data_raw", + id_col=PATIENT_ID_COL, + categorical_cols=PATIENT_CATEGORICAL_COLS, + alignment=RowAlignment.IDENTITY, + ordinal_group_cols=PATIENT_KEY_COLS, + date_normalize_cols=PATIENT_RAW_DATE_NORMALIZE_COLS, + numeric_normalize_cols=PATIENT_RAW_NUMERIC_NORMALIZE_COLS, + whitespace_normalize_cols=PATIENT_RAW_WHITESPACE_NORMALIZE_COLS, + normalize_boolean_literals=True, + ), + Stage( + label="Patient (cleaned)", + subdir="patient_data_cleaned", + id_col=PATIENT_ID_COL, + categorical_cols=PATIENT_CATEGORICAL_COLS, + alignment=RowAlignment.IDENTITY, + ordinal_group_cols=PATIENT_KEY_COLS, + whitespace_normalize_cols=PATIENT_WHITESPACE_NORMALIZE_COLS, + numeric_normalize_cols=PATIENT_CLEANED_NUMERIC_NORMALIZE_COLS, + load_glucose_unit_swaps=True, + static_join_missed_ids_from="patient_data_raw", + ), + Stage( + label="Product (raw)", + subdir="product_data_raw", + id_col=PRODUCT_ID_COL, + categorical_cols=PRODUCT_CATEGORICAL_COLS, + alignment=RowAlignment.POSITIONAL, + ordinal_group_cols=PRODUCT_ORDINAL_GROUP_COLS, + date_normalize_cols=["product_entry_date"], + tracker_year_col=PRODUCT_TRACKER_YEAR_COL, + numeric_normalize_cols=PRODUCT_RAW_NUMERIC_NORMALIZE_COLS, + whitespace_normalize_cols=PRODUCT_RAW_WHITESPACE_NORMALIZE_COLS, + ), + Stage( + label="Product (cleaned)", + subdir="product_data_cleaned", + id_col=PRODUCT_ID_COL, + categorical_cols=PRODUCT_CATEGORICAL_COLS, + alignment=RowAlignment.POSITIONAL, + ordinal_group_cols=PRODUCT_ORDINAL_GROUP_COLS, + tracker_year_col=PRODUCT_TRACKER_YEAR_COL, + whitespace_normalize_cols=PRODUCT_CLEANED_WHITESPACE_NORMALIZE_COLS, + ), +] + + +def _load_parquet_dir(directory: Path) -> dict[str, pl.DataFrame]: + return {path.name: pl.read_parquet(path) for path in sorted(directory.glob("*.parquet"))} + + +def _numeric_cols(frames: dict[str, pl.DataFrame]) -> list[str]: + if not frames: + return [] + sample = next(iter(frames.values())) + return [name for name, dtype in sample.schema.items() if dtype.is_numeric()] + + +ERRORS_TABLE = Path("tables") / "table_errors.parquet" + + +def _unit_swaps_by_frame(py_root: Path, py_frames: dict[str, pl.DataFrame]) -> dict[str, set[str]]: + """Re-key the run's glucose-unit swaps from tracker name to parquet name. + + The error table names a tracker; the comparison names a parquet file. The + bridge is each frame's own ``file_name`` column rather than string surgery + on the parquet's name, so a change to the output naming convention cannot + silently drop the context. + """ + errors_path = py_root / ERRORS_TABLE + if not errors_path.exists(): + console.print( + f"[yellow]No {ERRORS_TABLE} under {py_root} -- glucose unit-swap context " + "unavailable, so swapped columns will not be classified.[/yellow]" + ) + return {} + swaps = load_glucose_unit_swaps(pl.read_parquet(errors_path)) + by_frame = {} + for name, df in py_frames.items(): + if "file_name" not in df.columns or not len(df): + continue + columns = swaps.get(df["file_name"][0]) + if columns: + by_frame[name] = columns + return by_frame + + +def _static_join_missed_ids_by_frame( + py_root: Path, stage: Stage, py_frames: dict[str, pl.DataFrame] +) -> dict[str, set[str]]: + """Per cleaned frame, the identities whose raw spelling missed the static join. + + Read from this run's own *raw* output, which is the last place the two + spellings coexist (ticket 63). Re-keyed from raw parquet name to cleaned + parquet name through each frame's ``file_name`` column, for the reason + ``_unit_swaps_by_frame`` gives: the two directories suffix their files + differently, and string surgery on the name would break silently. + """ + assert stage.static_join_missed_ids_from is not None + raw_dir = py_root / stage.static_join_missed_ids_from + if not raw_dir.exists(): + console.print( + f"[yellow]No {stage.static_join_missed_ids_from} under {py_root} -- respelled-ID " + "context unavailable, so the static-join cause cannot fire on this stage.[/yellow]" + ) + return {} + by_tracker: dict[str, set[str]] = {} + for path in sorted(raw_dir.glob("*.parquet")): + schema = pl.scan_parquet(path).collect_schema() + if "patient_id" not in schema.names() or "file_name" not in schema.names(): + continue + raw_df = pl.read_parquet(path, columns=["patient_id", "file_name"]) + missed = static_join_missed_ids(raw_df) + if missed and len(raw_df): + by_tracker[raw_df["file_name"][0]] = missed + by_frame = {} + for name, df in py_frames.items(): + if "file_name" not in df.columns or not len(df): + continue + missed = by_tracker.get(df["file_name"][0]) + if missed: + by_frame[name] = missed + return by_frame + + +def _compare_arm(py_root: Path, r_dir: Path, py_dir: Path, stage: Stage) -> DirectoryComparison: + r_frames = _load_parquet_dir(r_dir) + py_frames = _load_parquet_dir(py_dir) + unit_swapped_columns = ( + _unit_swaps_by_frame(py_root, py_frames) if stage.load_glucose_unit_swaps else {} + ) + static_join_missed_ids_by_frame = ( + _static_join_missed_ids_by_frame(py_root, stage, py_frames) + if stage.static_join_missed_ids_from + else {} + ) + for column in stage.date_normalize_cols or []: + for frames in (r_frames, py_frames): + for name, df in frames.items(): + frames[name] = normalize_date_column(df, column) + # Whitespace before numeric (ticket 24): normalize_numeric_column widens a + # column to `pl.Object` (mixed float/str) the moment any value parses, + # which breaks normalize_whitespace_column's `.str.*` expressions on the + # non-numeric residual if applied after -- stripping first keeps the + # column `pl.String` for numeric normalization to then act on. + for frames in (r_frames, py_frames): + for name, df in frames.items(): + if stage.whitespace_normalize_cols is ALL_RAW_COLUMNS: + # Resolved per frame, not once: raw parquets differ in which + # columns they carry, and a 2026 tracker holds columns a 2017 + # one never had. + columns = whitespace_normalize_targets(df, exclude=[]) + else: + columns = stage.whitespace_normalize_cols or [] + for column in columns: + frames[name] = normalize_whitespace_column(frames[name], column) + # Boolean literals share whitespace's scope and must run before + # numeric normalization for the same reason: once a column widens + # to pl.Object the `.str.*` expressions no longer apply. + if stage.normalize_boolean_literals: + for column in columns: + frames[name] = normalize_boolean_literal_column(frames[name], column) + for frames in (r_frames, py_frames): + for name, df in frames.items(): + if stage.numeric_normalize_cols is ALL_STRING_COLUMNS: + columns = string_numeric_normalize_targets(df, exclude=stage.ordinal_group_cols) + elif stage.numeric_normalize_cols is ALL_RAW_COLUMNS: + # The ordinal's group columns are excluded because + # normalize_numeric_column widens a column to pl.Object the + # moment any value parses, which would stop add_row_ordinal + # whitespace-normalizing it as a string. + columns = numeric_normalize_targets(df, exclude=stage.ordinal_group_cols) + else: + columns = stage.numeric_normalize_cols or [] + for column in columns: + frames[name] = normalize_numeric_column(frames[name], column) + key_cols = None + if stage.alignment is RowAlignment.IDENTITY: + # Both sides at once: the tie-break between two rows sharing an + # identity key is decided by which pairing agrees best, which cannot be + # computed from one frame alone. + for name in r_frames.keys() & py_frames.keys(): + r_frames[name], py_frames[name], key_cols = align_duplicate_rows( + r_frames[name], py_frames[name], stage.ordinal_group_cols + ) + for frames in (r_frames, py_frames): + for name, df in frames.items(): + if ROW_ORDINAL_COL in df.columns: + continue + frames[name], key_cols = add_row_ordinal(df, stage.ordinal_group_cols) + assert key_cols is not None + # Everything add_row_ordinal returned except the trailing ordinal itself -- + # the group a within-group sort-order divergence (ticket 21) is scoped to. + order_group_cols = key_cols[:-1] if stage.detect_row_order_divergence else None + # __-prefixed helper columns from add_row_ordinal are join-key-only synthetic + # data (an ordinal counter, normalized group values) -- summing/diffing them + # as if they were real output columns would be noise, not signal. + numeric_cols = [c for c in _numeric_cols(r_frames) if not c.startswith("__")] + # compare_directory/compare_categorical_overlap skip a column not present on + # both sides of a given file -- raw output isn't schema-normalized like cleaned + # output, so which columns exist can vary file by file, not just by directory. + return compare_directory( + r_frames, + py_frames, + key_cols=key_cols, + numeric_cols=numeric_cols, + id_col=stage.id_col, + categorical_cols=stage.categorical_cols, + order_group_cols=order_group_cols, + unit_swapped_columns=unit_swapped_columns, + tracker_year_col=stage.tracker_year_col, + static_join_missed_ids_by_frame=static_join_missed_ids_by_frame, + ) + + +LEGEND = """\ +[bold]Shape match[/bold] -- do R and Python have the same row count for this file? A \ +coarse structural check: matching shape says nothing about whether individual cell values agree. + +[bold]ID divergence[/bold] -- identities (patient_id for patient, product name for \ +product) present on only one side, regardless of row count. Independent of the row-alignment \ +key used for row-key/cell divergence below -- catches a patient or product dropped entirely, \ +even when that key is unreliable. + +[bold]Column divergence[/bold] -- columns present on only one side, plus columns present on \ +both sides but with a different dtype. Structural, like shape -- no values are compared. + +[bold]Categorical divergence[/bold] -- of the file's categorical/label columns (allowed-value \ +fields for patient; product_category/product_balance_status for product), how many have a \ +label value on one side that never appears on the other? Also independent of the \ +row-alignment key -- a distinct-value-set check per column, not tied to row identity. + +[bold]Totals divergence[/bold] -- of the file's numeric columns, how many have a column-sum \ +that differs beyond a float tolerance? The first check that actually compares values, at the \ +coarsest (whole-column) granularity. + +[bold]Row-key divergence[/bold] -- rows whose full row-alignment key (all of it, not just the \ +single identity column above) found no partner on the other side at all, counted per row, not \ +per distinct key. A repeated key on one side with no matching repeat on the other also counts \ +here -- that's the fan-out failure mode. This is what tells you whether Cell divergence below \ +is measuring real disagreement or just has nothing to compare. + +[bold]Cell divergence[/bold] -- rows matched across R and Python (via the arm's \ +row-alignment key), diffed value by value. The most granular value comparison -- but it's only \ +meaningful once Row-key divergence above confirms the key actually paired the rows; 0 here can \ +mean "everything agreed" or "nothing was paired to compare" (ticket 15/17 on the wayfinder map).\ +""" + + +def _print_legend() -> None: + console.print(Panel(LEGEND, title="What these columns measure", expand=False)) + + +YEAR_PREFIX = re.compile(r"^(\d{4})_") + + +def _file_year(file_name: str) -> int | None: + match = YEAR_PREFIX.match(file_name) + return int(match.group(1)) if match else None + + +def _has_any_mismatch(file_comparison: FileComparison) -> bool: + return ( + not file_comparison.shape.match + or bool(file_comparison.columns.only_in_r) + or bool(file_comparison.columns.only_in_py) + or bool(file_comparison.columns.dtype_mismatches) + or bool(file_comparison.totals) + or bool(file_comparison.categorical_overlap) + or bool(file_comparison.cell_mismatches) + or file_comparison.row_key_overlap.r_unmatched > 0 + or file_comparison.row_key_overlap.py_unmatched > 0 + or ( + file_comparison.id_overlap is not None + and bool(file_comparison.id_overlap.only_in_r or file_comparison.id_overlap.only_in_py) + ) + ) + + +def _print_summary(arm: str, comparison: DirectoryComparison, only_mismatches: bool) -> None: + files = ( + [f for f in comparison.files if _has_any_mismatch(f)] + if only_mismatches + else comparison.files + ) + if only_mismatches and not files: + console.print(f"[green]{arm}: no mismatches in any file.[/green]") + return + + by_year: dict[int | None, list] = {} + for file_comparison in files: + by_year.setdefault(_file_year(file_comparison.file_name), []).append(file_comparison) + + for year in sorted(by_year, key=lambda y: (y is None, -(y or 0))): + title = f"{arm} — {year}" if year is not None else f"{arm} — unknown year" + table = Table(title=title) + table.add_column("File") + table.add_column("Shape match") + table.add_column("ID divergence (R-only / Py-only)") + table.add_column("Column divergence", justify="right") + table.add_column("Categorical divergence", justify="right") + table.add_column("Totals divergence", justify="right") + table.add_column("Row-key divergence (R-only / Py-only)") + table.add_column("Cell divergence", justify="right") + + for file_comparison in by_year[year]: + column_divergence = ( + len(file_comparison.columns.only_in_r) + + len(file_comparison.columns.only_in_py) + + len(file_comparison.columns.dtype_mismatches) + ) + if file_comparison.id_overlap is None: + id_divergence_str = "n/a" + else: + id_divergence_str = ( + f"{len(file_comparison.id_overlap.only_in_r)} / " + f"{len(file_comparison.id_overlap.only_in_py)}" + ) + row_key_str = ( + f"{file_comparison.row_key_overlap.r_unmatched} / " + f"{file_comparison.row_key_overlap.py_unmatched}" + ) + table.add_row( + file_comparison.file_name, + "✓" if file_comparison.shape.match else "✗", + id_divergence_str, + str(column_divergence), + str(len(file_comparison.categorical_overlap)), + str(len(file_comparison.totals)), + row_key_str, + str(len(file_comparison.cell_mismatches)), + ) + console.print(table) + + _render_arm_totals(arm, comparison) + + if comparison.only_in_r: + console.print( + f"[yellow]Only in R ({len(comparison.only_in_r)}):[/yellow] " + f"{', '.join(comparison.only_in_r)}" + ) + if comparison.only_in_py: + console.print( + f"[yellow]Only in Python ({len(comparison.only_in_py)}):[/yellow] " + f"{', '.join(comparison.only_in_py)}" + ) + + +def _render_arm_totals(arm: str, comparison: DirectoryComparison) -> None: + """Whole-arm rollup, printed after the per-year tables. + + The per-year tables say *where* a divergence is; without this you can + only get *how much* there is by scrolling and adding up every year. + """ + summary = summarize_directory(comparison) + + table = Table(title=f"{arm} — totals across all {summary.files_compared} files compared") + table.add_column("Measure") + table.add_column("Files affected", justify="right") + table.add_column("Total", justify="right") + + def add(measure: str, value: tuple[int, int]) -> None: + files_affected, total = value + style = "" if total == 0 else "yellow" + table.add_row(measure, str(files_affected), str(total), style=style) + + shape_style = "" if summary.shape_mismatch_files == 0 else "yellow" + table.add_row("Shape mismatch", str(summary.shape_mismatch_files), "—", style=shape_style) + add("ID divergence", summary.id_divergence) + add("Column divergence", summary.column_divergence) + add("Categorical divergence", summary.categorical_divergence) + add("Totals divergence", summary.totals_divergence) + add("Row-key divergence", summary.row_key_divergence) + add("Cell divergence", summary.cell_divergence) + + console.print(table) + + if summary.only_in_r or summary.only_in_py: + console.print( + f"[yellow]Files on only one side:[/yellow] " + f"{summary.only_in_r} R-only, {summary.only_in_py} Python-only " + f"(not included in the counts above — nothing to compare them against)" + ) + + +def _write_excel_report(path: Path, rows_by_sheet: dict[str, list[dict]]) -> None: + workbook = Workbook() + workbook.remove(workbook.active) + for sheet_name, rows in rows_by_sheet.items(): + worksheet = workbook.create_sheet(sheet_name[:31]) + if not rows: + worksheet.append(["(no mismatches)"]) + continue + headers = list(rows[0].keys()) + worksheet.append(headers) + for row in rows: + worksheet.append([row[h] for h in headers]) + workbook.save(path) + + +def _previous_run_dirs(output_dir: Path, current_run_dir: Path) -> list[Path]: + # Run directories are named by ISO-8601 timestamp, so lexicographic sort is + # chronological -- most recent first, excluding the run just created. + if not output_dir.exists(): + return [] + run_dirs = [p for p in output_dir.iterdir() if p.is_dir() and p != current_run_dir] + return sorted(run_dirs, reverse=True) + + +def _load_latest_snapshot( + output_dir: Path, current_run_dir: Path, subdir: str +) -> dict[str, dict[str, int]] | None: + # A stage can be missing from an older run (e.g. it errored, or the stage + # set changed) -- walk back further rather than only checking the single + # most recent run directory. + for run_dir in _previous_run_dirs(output_dir, current_run_dir): + snapshot_path = run_dir / f"snapshot_{subdir}.json" + if snapshot_path.exists(): + return json.loads(snapshot_path.read_text()) + return None + + +def _write_snapshot(run_dir: Path, subdir: str, snapshot: dict[str, dict[str, int]]) -> Path: + path = run_dir / f"snapshot_{subdir}.json" + path.write_text(json.dumps(snapshot, indent=2, sort_keys=True)) + return path + + +def _print_deltas(label: str, column_deltas: list[Delta], cause_deltas: list[Delta]) -> None: + if not column_deltas and not cause_deltas: + console.print(f"[dim]{label}: no change from the previous run.[/dim]") + return + + table = Table(title=f"{label} — change since previous run") + table.add_column("Column") + table.add_column("Previous", justify="right") + table.add_column("Current", justify="right") + table.add_column("Diff", justify="right") + for delta in column_deltas: + diff = delta.current - delta.previous + style = "red" if diff > 0 else "green" + table.add_row( + delta.key, str(delta.previous), str(delta.current), f"[{style}]{diff:+d}[/{style}]" + ) + console.print(table) + + +def _delta_rows(deltas: list[Delta]) -> list[dict]: + return [ + { + "key": delta.key, + "previous": delta.previous, + "current": delta.current, + "diff": delta.current - delta.previous, + } + for delta in deltas + ] + + +@app.command() +def compare( + r_dir: Annotated[Path, typer.Option("--r-dir", help="Root of the frozen R output directory")], + py_dir: Annotated[Path, typer.Option("--py-dir", help="Root of the Python output directory")], + output_dir: Annotated[ + Path, + typer.Option( + "--output-dir", help="Directory each run's own timestamped subfolder is created under" + ), + ] = Path("output/comparison"), + only_mismatches: Annotated[ + bool, + typer.Option( + "--only-mismatches", help="Only print files with at least one measure flagged" + ), + ] = False, +) -> None: + _print_legend() + # Every stage of this run shares one timestamped subfolder, so a run is a + # single self-contained unit on disk -- its reports and snapshots (for the + # *next* run's delta) live together, and nothing gets overwritten by a + # later run. + run_dir = output_dir / datetime.now(UTC).strftime("%Y-%m-%dT%H%M%SZ") + run_dir.mkdir(parents=True, exist_ok=True) + + for stage in STAGES: + label, subdir = stage.label, stage.subdir + comparison = _compare_arm(py_dir, r_dir / subdir, py_dir / subdir, stage) + _print_summary(label, comparison, only_mismatches) + + # One workbook per stage -- combining raw and cleaned into one per-column + # aggregate would sum e.g. product_entry_date mismatches from both stages + # into a single count, defeating the point of separating them. + stage_report_out = run_dir / f"compare_report_{subdir}.xlsx" + summary_sheets = build_summary_rows(comparison, classifiers_by_column=CLASSIFIERS_BY_COLUMN) + detail_sheets = build_mismatch_rows(comparison, classifiers_by_column=CLASSIFIERS_BY_COLUMN) + + # Run-over-run history (ticket 19): compare this run's per-column/per-cause + # counts against the most recent prior run's snapshot for this stage, so a + # triage fix's actual effect (or a regression) is visible directly rather + # than only ever seeing the current numbers. + current_snapshot = snapshot_from_summary(summary_sheets) + previous_snapshot = _load_latest_snapshot(output_dir, run_dir, subdir) + column_deltas = ( + compute_deltas(previous_snapshot["per_column"], current_snapshot["per_column"]) + if previous_snapshot + else [] + ) + cause_deltas = ( + compute_deltas(previous_snapshot["per_cause"], current_snapshot["per_cause"]) + if previous_snapshot + else [] + ) + if previous_snapshot is None: + console.print(f"[dim]{label}: no previous run to compare against.[/dim]") + else: + _print_deltas(label, column_deltas, cause_deltas) + history_sheets = { + "history_column_deltas": _delta_rows(column_deltas), + "history_cause_deltas": _delta_rows(cause_deltas), + } + _write_snapshot(run_dir, subdir, current_snapshot) + + _write_excel_report(stage_report_out, {**summary_sheets, **detail_sheets, **history_sheets}) + + console.print(f"[bold green]{label} report written to {stage_report_out}[/bold green]") + + +if __name__ == "__main__": + app() diff --git a/scripts/gcp/copy_data_back_to_storage.sh b/scripts/gcp/copy_data_back_to_storage.sh deleted file mode 100644 index f666f6b..0000000 --- a/scripts/gcp/copy_data_back_to_storage.sh +++ /dev/null @@ -1,2 +0,0 @@ -# user does not matter? -gsutil -m cp -r /home/rstudio/data/a4dphase2_upload/output/ gs://a4dphase2_output diff --git a/scripts/gcp/copy_data_from_storage.sh b/scripts/gcp/copy_data_from_storage.sh deleted file mode 100644 index fd08b68..0000000 --- a/scripts/gcp/copy_data_from_storage.sh +++ /dev/null @@ -1,4 +0,0 @@ -# as sudo -sudo -u rstudio gsutil -m cp -r gs://a4dphase2_upload /home/rstudio/data -# as rstudio user (from Terminal in RStudio) -gsutil -m cp -r gs://a4dphase2_upload /home/rstudio/data diff --git a/scripts/gcp/ingest_data.sh b/scripts/gcp/ingest_data.sh deleted file mode 100644 index 270e01d..0000000 --- a/scripts/gcp/ingest_data.sh +++ /dev/null @@ -1,4 +0,0 @@ -bq load --source_format PARQUET --clustering_fields clinic_code,id,tracker_year,tracker_month --project_id=a4d-315220 --max_bad_records 0 --replace=true tracker.patient_data_monthly "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output/tables/patient_data_monthly.parquet" -bq load --source_format PARQUET --clustering_fields product_hospital,product_released_to,product_table_year,product_table_month --project_id=a4d-315220 --max_bad_records 0 --replace=true tracker.product_data "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output/tables/product_data.parquet" -bq load --source_format PARQUET --clustering_fields id,tracker_year,tracker_month --project_id=a4d-315220 --max_bad_records 0 --replace=true tracker.patient_data_static "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output/tables/patient_data_static.parquet" -bq load --source_format PARQUET --clustering_fields clinic_code,id,tracker_year,tracker_month --project_id=a4d-315220 --max_bad_records 0 --replace=true tracker.patient_data_hba1c "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/output/tables/longitudinal_data_hba1c.parquet" diff --git a/scripts/gcp/setup.sh b/scripts/gcp/setup.sh deleted file mode 100644 index 42661b3..0000000 --- a/scripts/gcp/setup.sh +++ /dev/null @@ -1,14 +0,0 @@ -# https://posit.co/download/rstudio-server/ -sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 -sudo add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/' -sudo apt update -sudo apt -y install r-base -sudo apt -y install libssl-dev libfontconfig1-dev libharfbuzz-dev libfribidi-dev \ - libfreetype6-dev libpng-dev libtiff5-dev libjpeg-dev libxml2-dev libcurl4-openssl-dev - -sudo apt -y install gdebi-core -wget https://download2.rstudio.org/server/focal/amd64/rstudio-server-2023.06.1-524-amd64.deb -sudo gdebi -n rstudio-server-2023.06.1-524-amd64.deb - -sudo useradd rstudio -m -sudo passwd rstudio diff --git a/scripts/logs.ipynb b/scripts/logs.ipynb deleted file mode 100644 index ce9d225..0000000 --- a/scripts/logs.ipynb +++ /dev/null @@ -1,3296 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "\n", - "import pandas as pd\n", - "from collections import defaultdict\n", - "\n", - "pd.set_option('display.max_colwidth', None)\n", - "pd.set_option(\"display.max_rows\", 1000)\n", - "pd.set_option('display.width', None)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "dir = \"/Volumes/USB SanDisk 3.2Gen1 Media/a4d/a4dphase2_upload/output/logs\"" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "220" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "files = list(Path(dir).glob(\"*.log\"))\n", - "len(files)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "data = defaultdict(list)\n", - "for file in files:\n", - " with open(file, \"r\", encoding=\"utf-8\") as fstream:\n", - " for line in fstream.read().splitlines():\n", - " error = None\n", - " warn = None\n", - " if \"ERROR\" in line:\n", - " error = line.split(\"\\t\")[-1]\n", - " \n", - " if \"WARN\" in line:\n", - " warn = line.split(\"\\t\")[-1]\n", - " \n", - " if error or warn: \n", - " data[\"file\"].append(file.name)\n", - " data[\"error\"].append(error)\n", - " data[\"warning\"].append(warn)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "df = pd.DataFrame(data)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
fileerrorwarning
02022_Children's Hospital 2 A4D Tracker_patient.logNoneNon-matching column names found: meterissued.
12022_Children's Hospital 2 A4D Tracker_patient.logNoneNon-matching column names found: instantmeterreceived.
22022_Children's Hospital 2 A4D Tracker_patient.logNoneNon-matching column names found: instantmeterreceived.
32022_Children's Hospital 2 A4D Tracker_patient.logNoneNon-matching column names found: instantmeterreceived.
42022_Children's Hospital 2 A4D Tracker_patient.logNoneNon-matching column names found: instantmeterreceived.
\n", - "
" - ], - "text/plain": [ - " file error \\\n", - "0 2022_Children's Hospital 2 A4D Tracker_patient.log None \n", - "1 2022_Children's Hospital 2 A4D Tracker_patient.log None \n", - "2 2022_Children's Hospital 2 A4D Tracker_patient.log None \n", - "3 2022_Children's Hospital 2 A4D Tracker_patient.log None \n", - "4 2022_Children's Hospital 2 A4D Tracker_patient.log None \n", - "\n", - " warning \n", - "0 Non-matching column names found: meterissued. \n", - "1 Non-matching column names found: instantmeterreceived. \n", - "2 Non-matching column names found: instantmeterreceived. \n", - "3 Non-matching column names found: instantmeterreceived. \n", - "4 Non-matching column names found: instantmeterreceived. " - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "df.to_csv(Path(dir) / \"error_report.csv\", index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "145" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.file.nunique()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([\"2022_Children's Hospital 2 A4D Tracker_patient.log\",\n", - " '2020_Penang General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2019_Sultanah Bahiyah Hospital A4D Tracker_DC_product.log',\n", - " '2020_Mahosot Hospital A4D Tracker_DC_product_raw.log',\n", - " \"2020_Vietnam National Children's Hospital A4D Tracker_product.log\",\n", - " '2021_Putrajaya Hospital A4D Tracker_DC_patient.log',\n", - " '2022_Vietnam National Children_s Hospital A4D Tracker_product_raw.log',\n", - " '2019_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_Siriraj Hospital A4D Tracker_product.log',\n", - " '2022_Sultanah Bahiyah A4D Tracker_DC_patient_raw.log',\n", - " '2022_Preah Kossamak Hospital A4D Tracker_product.log',\n", - " '2021_Vietnam National Children_s Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Penang General Hospital A4D Tracker_DC_patient.log',\n", - " '2022_Sultanah Bahiyah A4D Tracker_DC_product.log',\n", - " '2018_Mahosot Hospital A4D Tracker_product.log',\n", - " '2022_Khon Kaen Hospital A4D Tracker_patient_raw.log',\n", - " '2020_Lao Friends Hospital for Children A4D Tracker_DC_product.log',\n", - " '2020_Lao Friends Hospital for Children A4D Tracker_DC_product_raw.log',\n", - " \"2017_Vietnam National Children's Hospital A4D Tracker_patient_raw.log\",\n", - " '2019_Lao Friends Hospital for Children A4D Tracker_DC_patient_raw.log',\n", - " '2022_Sarawak General Hospital A4D Tracker_DC_product_raw.log',\n", - " '2021_Mahosot Hospital A4D Tracker_DC_product_raw.log',\n", - " \"2021_Likas Women & Children's Hospital A4D Tracker_DC_product.log\",\n", - " '2019_Penang General Hospital A4D Tracker_DC_patient.log',\n", - " '2018_Vietnam National Children_s Hospital A4D Tracker_product_raw.log',\n", - " '2022_Surat Thani A4D Tracker_product_raw.log',\n", - " '2022_Chulalongkorn Hospital A4D Tracker_patient_raw.log',\n", - " '2019_Penang General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2019_Sultanah Bahiyah Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_Sunprasitthiprasong Hospital A4D Tracker_product.log',\n", - " '2020_Sarawak General Hospital A4D Tracker_DC_product_raw.log',\n", - " '2020_Mahosot Hospital A4D Tracker_DC_product.log',\n", - " \"2022_Mandalay Children's Hospital A4D Tracker_product_raw.log\",\n", - " '2022_Uni Med Centre A4D Tracker_patient.log',\n", - " '2019_Mahosot Hospital A4D Tracker_patient_raw.log',\n", - " '2021_Sarawak General Hospital A4D Tracker_DC_patient.log',\n", - " '2021_Penang General Hospital A4D Tracker_DC_patient.log',\n", - " '2021_Sultanah Bahiyah Hospital A4D Tracker_DC_patient.log',\n", - " '2021_Uni Med Center A4D Tracker_patient.log',\n", - " '2022_Sarawak General Hospital A4D Tracker_DC_product.log',\n", - " '2019_Lao Friends Hospital for Children A4D Tracker_DC_product.log',\n", - " '2021_Penang General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2020_Penang General Hospital A4D Tracker_DC_product.log',\n", - " '2022_Surat Thani A4D Tracker_product.log',\n", - " '2020_Lao Friends Hospital for Children A4D Tracker_DC_patient_raw.log',\n", - " '2022_Siriraj Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Putrajaya Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2018_Penang General Hospital A4D Tracker_DC_product.log',\n", - " '2022_Khon Kaen Hospital A4D Tracker_product_raw.log',\n", - " '2022_Jayavarman VII Hospital A4D Tracker_patient_raw.log',\n", - " '2021_Vietnam National Children_s Hospital A4D Tracker_product_raw.log',\n", - " '2022_Jayavarman VII Hospital A4D Tracker_patient.log',\n", - " '2022_Chiang Mai Maharaj Nakorn A4D Tracker_patient.log',\n", - " '2019_Sultanah Bahiyah Hospital A4D Tracker_DC_product_raw.log',\n", - " '2019_Vietnam National Children_s Hospital A4D Tracker_product.log',\n", - " '2019_Penang General Hospital A4D Tracker_DC_product_raw.log',\n", - " '2022_Khon Kaen Hospital A4D Tracker_product.log',\n", - " '2018_Vietnam National Children_s Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Surat Thani A4D Tracker_patient_raw.log',\n", - " '2022_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2021_Mahosot Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2020_Sultanah Bahiyah Hospital A4D Tracker_DC_product.log',\n", - " '2018_Penang General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_NPH A4D Tracker_product.log',\n", - " '2022_Vietnam National Children_s Hospital A4D Tracker_patient.log',\n", - " '2022_Mukdahan Hospital A4D Tracker_patient.log',\n", - " '2022_Mahosot Hosptial A4D Tracker_DC_product.log',\n", - " '2021_Putrajaya Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2019_Mahosot Hospital A4D Tracker_product_raw.log',\n", - " \"2022_Mandalay Children's Hospital A4D Tracker_patient_raw.log\",\n", - " '2020_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2021_Mahosot Hospital A4D Tracker_DC_product.log',\n", - " \"2021_Children's Hospital 2 A4D Tracker_patient_raw.log\",\n", - " '2021_NPH A4D Tracker_patient_raw.log',\n", - " '2022_Mahosot Hosptial A4D Tracker_DC_product_raw.log',\n", - " \"2022_Likas Women & Children's Hospital A4D Tracker_DC_product_raw.log\",\n", - " '2021_NPH A4D Tracker_product.log',\n", - " '2022_Vietnam National Children_s Hospital A4D Tracker_product.log',\n", - " '2022_Sunprasitthiprasong Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Mukdahan Hospital A4D Tracker_product.log',\n", - " \"2022_Mandalay Children's Hospital A4D Tracker_product.log\",\n", - " '2018_Mahosot Hospital A4D Tracker_product_raw.log',\n", - " '2021_Uni Med Center A4D Tracker_patient_raw.log',\n", - " '2022_Preah Kossamak Hospital A4D Tracker_patient_raw.log',\n", - " \"2021_Children's Hospital 2 A4D Tracker_product.log\",\n", - " '2022_Chiang Mai Maharaj Nakorn A4D Tracker_product_raw.log',\n", - " '2019_Sarawak General Hospital A4D Tracker_DC_product.log',\n", - " '2022_Kantha Bopha Hospital A4D Tracker_patient.log',\n", - " '2021_Lao Friends Hospital for Children A4D Tracker_DC_product.log',\n", - " '2022_Khon Kaen Hospital A4D Tracker_patient.log',\n", - " '2022_Phattalung Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Kantha Bopha Hospital A4D Tracker_product_raw.log',\n", - " \"2021_Likas Women & Children's Hospital A4D Tracker_DC_patient_raw.log\",\n", - " '2021_Lao Friends Hospital for Children A4D Tracker_DC_patient_raw.log',\n", - " \"2020_Vietnam National Children's Hospital A4D Tracker_patient_raw.log\",\n", - " '2022_Chiang Mai Maharaj Nakorn A4D Tracker_product.log',\n", - " '2019_Mahosot Hospital A4D Tracker_product.log',\n", - " '2022_Jayavarman VII Hospital A4D Tracker_product.log',\n", - " '2022_Uni Med Centre A4D Tracker_patient_raw.log',\n", - " \"2022_Children's Hospital 2 A4D Tracker_patient_raw.log\",\n", - " '2019_Vietnam National Children_s Hospital A4D Tracker_patient_raw.log',\n", - " '2020_Sultanah Bahiyah Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_NPH A4D Tracker_product_raw.log',\n", - " '2020_Penang General Hospital A4D Tracker_DC_patient.log',\n", - " '2022_CDA A4D Tracker_product_raw.log',\n", - " '2022_Pahol Polpayuhasena A4D Tracker_patient_raw.log',\n", - " '2017_Mahosot Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Udon Thani Hospital A4D Tracker_patient_raw.log',\n", - " '2021_Vietnam National Children_s Hospital A4D Tracker_product.log',\n", - " '2021_Uni Med Center A4D Tracker_product.log',\n", - " '2022_Hat Yai Hospital A4D Tracker_product_raw.log',\n", - " '2021_Sultanah Bahiyah Hospital A4D Tracker_DC_product.log',\n", - " '2022_Mukdahan Hospital A4D Tracker_patient_raw.log',\n", - " '2021_Sarawak General Hospital A4D Tracker_DC_product.log',\n", - " '2021_Penang General Hospital A4D Tracker_DC_product.log',\n", - " '2022_Uni Med Centre A4D Tracker_product.log',\n", - " '2022_Lao Friends Hospital for Children A4D Tracker_DC_patient_raw.log',\n", - " \"2022_Likas Women & Children's Hospital A4D Tracker_DC_patient_raw.log\",\n", - " '2022_Mahosot Hosptial A4D Tracker_DC_patient_raw.log',\n", - " '2021_Sultanah Bahiyah Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2020_Mahosot Hospital A4D Tracker_DC_patient.log',\n", - " \"2020_Vietnam National Children's Hospital A4D Tracker_product_raw.log\",\n", - " '2022_Phattalung Hospital A4D Tracker_product_raw.log',\n", - " '2020_Sarawak General Hospital A4D Tracker_DC_product.log',\n", - " '2022_Kantha Bopha Hospital A4D Tracker_patient_raw.log',\n", - " \"2021_Likas Women & Children's Hospital A4D Tracker_DC_patient.log\",\n", - " '2019_Penang General Hospital A4D Tracker_DC_product.log',\n", - " '2022_Chiang Mai Maharaj Nakorn A4D Tracker_patient_raw.log',\n", - " '2018_Mahosot Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Preah Kossamak Hospital A4D Tracker_product_raw.log',\n", - " '2022_NPH A4D Tracker_patient_raw.log',\n", - " '2020_Sultanah Bahiyah Hospital A4D Tracker_DC_product_raw.log',\n", - " '2019_Vietnam National Children_s Hospital A4D Tracker_product_raw.log',\n", - " '2022_Uni Med Centre A4D Tracker_product_raw.log',\n", - " '2022_Pahol Polpayuhasena A4D Tracker_product.log',\n", - " '2022_Penang General Hospital A4D Tracker_DC_product.log',\n", - " '2022_Preah Kossamak Hospital A4D Tracker_patient.log',\n", - " '2018_Lao Friends Hospital for Children A4D Tracker_DC_patient_raw.log',\n", - " '2022_Hat Yai Hospital A4D Tracker_patient_raw.log',\n", - " \"2017_Vietnam National Children's Hospital A4D Tracker_product.log\",\n", - " '2021_Putrajaya Hospital A4D Tracker_DC_product.log',\n", - " '2021_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2017_Mahosot Hospital A4D Tracker_product_raw.log',\n", - " '2022_CDA A4D Tracker_patient_raw.log',\n", - " \"2022_Children's Hospital 2 A4D Tracker_product.log\"], dtype=object)" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.file.unique()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Error" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
fileerrorwarning
9962022_Penang General Hospital A4D Tracker_DC_patient.logCould not process patient data. Error = Error in if (header_cols[2] == header_cols_2[2]) {: missing value where TRUE/FALSE needed .None
17942022_Sunprasitthiprasong Hospital A4D Tracker_product.logCould not process product data. Error = Error in start_df_msd:end_df_msd: argument of length 0 .None
17952020_Sarawak General Hospital A4D Tracker_DC_product_raw.logCould not process raw product data. Error = Error in `dplyr::mutate()`: ℹ In argument: `product_received_from = case_when(...)`. Caused by error in `case_when()`: ! Can't combine `..1 (right)` <double> and `..2 (right)` <character>. .None
107232021_Children's Hospital 2 A4D Tracker_patient_raw.logCould not process raw patient data. Error = Error in `mutate()`: ℹ In argument: `t1d_diagnosis_age = fix_t1d_diagnosis_age(t1d_diagnosis_age, id)`. ℹ In row 1. Caused by error in `case_when()` at R/helper_patient_data_fix.R:319:4: ! Can't combine `..1 (right)` <character> and `..3 (right)` <integer>. .None
115862021_Uni Med Center A4D Tracker_patient_raw.logCould not process raw patient data. Error = Error in `mutate()`: ℹ In argument: `t1d_diagnosis_age = fix_t1d_diagnosis_age(t1d_diagnosis_age, id)`. ℹ In row 1. Caused by error in `case_when()` at R/helper_patient_data_fix.R:319:4: ! Can't combine `..1 (right)` <character> and `..3 (right)` <integer>. .None
270972017_Vietnam National Children's Hospital A4D Tracker_product.logCould not process product data. Error = Error in `filter()`: ℹ In argument: `!is.na(product_entry_date) & !grepl(\"^[0-9]+$\", product_entry_date)`. Caused by error: ! object 'product_entry_date' not found .None
272112017_Mahosot Hospital A4D Tracker_product_raw.logCould not process raw product data. Error = Error in `[<-` at R/helper_product_data.R:483:12: ! Assigned data `value` must be compatible with existing data. ℹ Error occurred for column `product_units_released`. Caused by error in `vec_assign()`: ! Can't convert <double> to <character>. .None
\n", - "
" - ], - "text/plain": [ - " file \\\n", - "996 2022_Penang General Hospital A4D Tracker_DC_patient.log \n", - "1794 2022_Sunprasitthiprasong Hospital A4D Tracker_product.log \n", - "1795 2020_Sarawak General Hospital A4D Tracker_DC_product_raw.log \n", - "10723 2021_Children's Hospital 2 A4D Tracker_patient_raw.log \n", - "11586 2021_Uni Med Center A4D Tracker_patient_raw.log \n", - "27097 2017_Vietnam National Children's Hospital A4D Tracker_product.log \n", - "27211 2017_Mahosot Hospital A4D Tracker_product_raw.log \n", - "\n", - " error \\\n", - "996 Could not process patient data. Error = Error in if (header_cols[2] == header_cols_2[2]) {: missing value where TRUE/FALSE needed . \n", - "1794 Could not process product data. Error = Error in start_df_msd:end_df_msd: argument of length 0 . \n", - "1795 Could not process raw product data. Error = Error in `dplyr::mutate()`: ℹ In argument: `product_received_from = case_when(...)`. Caused by error in `case_when()`: ! Can't combine `..1 (right)` and `..2 (right)` . . \n", - "10723 Could not process raw patient data. Error = Error in `mutate()`: ℹ In argument: `t1d_diagnosis_age = fix_t1d_diagnosis_age(t1d_diagnosis_age, id)`. ℹ In row 1. Caused by error in `case_when()` at R/helper_patient_data_fix.R:319:4: ! Can't combine `..1 (right)` and `..3 (right)` . . \n", - "11586 Could not process raw patient data. Error = Error in `mutate()`: ℹ In argument: `t1d_diagnosis_age = fix_t1d_diagnosis_age(t1d_diagnosis_age, id)`. ℹ In row 1. Caused by error in `case_when()` at R/helper_patient_data_fix.R:319:4: ! Can't combine `..1 (right)` and `..3 (right)` . . \n", - "27097 Could not process product data. Error = Error in `filter()`: ℹ In argument: `!is.na(product_entry_date) & !grepl(\"^[0-9]+$\", product_entry_date)`. Caused by error: ! object 'product_entry_date' not found . \n", - "27211 Could not process raw product data. Error = Error in `[<-` at R/helper_product_data.R:483:12: ! Assigned data `value` must be compatible with existing data. ℹ Error occurred for column `product_units_released`. Caused by error in `vec_assign()`: ! Can't convert to . . \n", - "\n", - " warning \n", - "996 None \n", - "1794 None \n", - "1795 None \n", - "10723 None \n", - "11586 None \n", - "27097 None \n", - "27211 None " - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df[~df.error.isna()]" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "3.1818181818181817" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# how many files could not be processed in %?\n", - "len(df[~df.error.isna()]) / len(files) * 100\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Error analysis" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Missing or invalid patient id" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
fileerrorwarning
\n", - "
" - ], - "text/plain": [ - "Empty DataFrame\n", - "Columns: [file, error, warning]\n", - "Index: []" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# missing patient id\n", - "df[(~df.error.isna()) & (df.error.str.contains(\"id is not valid\"))].drop_duplicates()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### missing month" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# month list is empty?\n", - "print(\"\\n\".join(l.split(\".\")[0] for l in df[(~df.error.isna()) & (df.error.str.contains(\"month_list\"))].file.tolist()))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### missing patient data" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[]\n", - "[]\n", - "[]\n", - "[]\n", - "[]\n", - "[]\n" - ] - } - ], - "source": [ - "# cannot find patient data\n", - "for year in range(2017,2023):\n", - " subdf = df[(~df.error.isna()) & (df.error.str.contains(\"readxl::cell_limits\"))]\n", - " print(subdf[subdf.file.str.startswith(str(year))].file.tolist())" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
fileerrorwarning
\n", - "
" - ], - "text/plain": [ - "Empty DataFrame\n", - "Columns: [file, error, warning]\n", - "Index: []" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# check problem with num_na_rows\n", - "subdf = df[(~df.error.isna()) & (df.error.str.contains(\"num_na_rows\"))]\n", - "subdf" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['2022_Sunprasitthiprasong Hospital A4D Tracker_product.log']" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# check start_df_msd:end_df_msd\n", - "subdf = df[(~df.error.isna()) & (df.error.str.contains(\"start_df_msd\"))]\n", - "subdf.file.tolist()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Warnings" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
fileerrorwarning
02022_Children's Hospital 2 A4D Tracker_patient.logNoneNon-matching column names found: meterissued.
12022_Children's Hospital 2 A4D Tracker_patient.logNoneNon-matching column names found: instantmeterreceived.
122020_Penang General Hospital A4D Tracker_DC_patient_raw.logNoneFound invalid values for column blood_pressure_mmhg that do not follow the format X/Y. Values were replaced with 999999.
132020_Penang General Hospital A4D Tracker_DC_patient_raw.logNoneExtra columns in patient data: admissiontohospitaldatemmmyy, observations1, admissiontohospitaldatemmyy
142020_Penang General Hospital A4D Tracker_DC_patient_raw.logNoneMissing columns in patient data: clinic_code, country_code, fbg_baseline_mg, fbg_updated_mg, hospitalisation_cause, hospitalisation_date, last_remote_followup_date, lost_date, observations, observations_category, status_out, t1d_diagnosis_date, t1d_diagnosis_with_dka, updated_2022_date
............
274412022_CDA A4D Tracker_patient_raw.logNonePatient KH_CD015: Value Self-mixed BD,Basal-bolus (AN) for column insulin_regimen is not in the list of allowed values.
274422022_CDA A4D Tracker_patient_raw.logNonePatient KH_CD015: Value Basal-bolus MDI (AN/HI),Basal-bolus (AN) for column insulin_regimen is not in the list of allowed values.
274462022_CDA A4D Tracker_patient_raw.logNonePatient KH_CD015: Value Premixed 30/70 BD,Basal-bolus (AN) for column insulin_regimen is not in the list of allowed values.
274532022_Children's Hospital 2 A4D Tracker_product.logNoneJul'22 number of dates in product_entry_date that don't match the month/year on the sheet is 4: 2020-07-27, 2020-07-29, 2020-07-27, 2020-07-29
274542022_Children's Hospital 2 A4D Tracker_product.logNoneNov'22 number of rows with non-processed dates in product_entry_date is 1: 25/22/2022
\n", - "

3597 rows × 3 columns

\n", - "
" - ], - "text/plain": [ - " file error \\\n", - "0 2022_Children's Hospital 2 A4D Tracker_patient.log None \n", - "1 2022_Children's Hospital 2 A4D Tracker_patient.log None \n", - "12 2020_Penang General Hospital A4D Tracker_DC_patient_raw.log None \n", - "13 2020_Penang General Hospital A4D Tracker_DC_patient_raw.log None \n", - "14 2020_Penang General Hospital A4D Tracker_DC_patient_raw.log None \n", - "... ... ... \n", - "27441 2022_CDA A4D Tracker_patient_raw.log None \n", - "27442 2022_CDA A4D Tracker_patient_raw.log None \n", - "27446 2022_CDA A4D Tracker_patient_raw.log None \n", - "27453 2022_Children's Hospital 2 A4D Tracker_product.log None \n", - "27454 2022_Children's Hospital 2 A4D Tracker_product.log None \n", - "\n", - " warning \n", - "0 Non-matching column names found: meterissued. \n", - "1 Non-matching column names found: instantmeterreceived. \n", - "12 Found invalid values for column blood_pressure_mmhg that do not follow the format X/Y. Values were replaced with 999999. \n", - "13 Extra columns in patient data: admissiontohospitaldatemmmyy, observations1, admissiontohospitaldatemmyy \n", - "14 Missing columns in patient data: clinic_code, country_code, fbg_baseline_mg, fbg_updated_mg, hospitalisation_cause, hospitalisation_date, last_remote_followup_date, lost_date, observations, observations_category, status_out, t1d_diagnosis_date, t1d_diagnosis_with_dka, updated_2022_date \n", - "... ... \n", - "27441 Patient KH_CD015: Value Self-mixed BD,Basal-bolus (AN) for column insulin_regimen is not in the list of allowed values. \n", - "27442 Patient KH_CD015: Value Basal-bolus MDI (AN/HI),Basal-bolus (AN) for column insulin_regimen is not in the list of allowed values. \n", - "27446 Patient KH_CD015: Value Premixed 30/70 BD,Basal-bolus (AN) for column insulin_regimen is not in the list of allowed values. \n", - "27453 Jul'22 number of dates in product_entry_date that don't match the month/year on the sheet is 4: 2020-07-27, 2020-07-29, 2020-07-27, 2020-07-29 \n", - "27454 Nov'22 number of rows with non-processed dates in product_entry_date is 1: 25/22/2022 \n", - "\n", - "[3597 rows x 3 columns]" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df[~df.warning.isna()].drop_duplicates()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Warning analysis" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### extra columns" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{' activeremoteupdatedate',\n", - " ' admissiontohospitaldatemmmyy',\n", - " ' admissiontohospitaldatemmyy',\n", - " ' complication_screening',\n", - " ' complication_screening_',\n", - " ' complication_screening_date',\n", - " ' complication_screening_results',\n", - " ' est_strips_pmoth',\n", - " ' family_support_',\n", - " ' family_support_scale',\n", - " ' ge100',\n", - " ' hospitalisation_date1',\n", - " ' instantmeterreceiv',\n", - " ' instantmeterreceiveddate',\n", - " ' insulin_required_month',\n", - " ' insulin_required_y',\n", - " ' insurancestatusnanssfeqeqpending',\n", - " ' lastclinicvisitไปโรงพยาบาลdate',\n", - " ' lastremotefollowupสายเขา',\n", - " ' lastremotefollowupโดยโทรศัพทdate',\n", - " ' meter',\n", - " ' meter_received_',\n", - " ' meter_received_date',\n", - " ' meterissued',\n", - " ' new',\n", - " ' observations1',\n", - " ' patientcontactnumber',\n", - " ' remote_followup',\n", - " ' remotefu',\n", - " ' virtualremotecontactdate',\n", - " ' virtualremotecontactyn',\n", - " '_dosage',\n", - " '_screening',\n", - " '_screening_results',\n", - " '_strips_pmoth',\n", - " '_visit',\n", - " 'hospitaldatemmmyy',\n", - " 'veremoteupdateyn'}" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "missed_names = df[(~df.warning.isna()) & (df.warning.str.contains(\"Extra\"))].warning.str.strip(\"Extra columns in patient data:\").drop_duplicates().to_list()\n", - "\n", - "names = set()\n", - "for x in missed_names:\n", - " names.update(x for x in x.split(\",\") if x)\n", - " \n", - "names" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### invalid values" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['- in column bmi for patient: MY_PN004',\n", - " '- in column fbg_baseline_mmol for patient: MY_PN004',\n", - " '0ct/19 in column bmi_date for patient: MY_PN011',\n", - " 'il in column hba1c_baseline for patient: MY_SW008',\n", - " 'NA or Hospitalisation Date in column hospitalisation_date for patient: MY_SW001',\n", - " 'NA or Hospitalisation Date in column hospitalisation_date for patient: MY_SW002',\n", - " 'NA or Hospitalisation Date in column hospitalisation_date for patient: MY_SW003',\n", - " 'NA or Hospitalisation Date in column hospitalisation_date for patient: MY_SW004',\n", - " 'NA or Hospitalisation Date in column hospitalisation_date for patient: MY_SW005',\n", - " 'NA or Hospitalisation Date in column hospitalisation_date for patient: MY_SW009',\n", - " 'NA or Hospitalisation Date in column hospitalisation_date for patient: MY_SW010',\n", - " 'NA or Hospitalisation Date in column hospitalisation_date for patient: MY_SW011',\n", - " 'Nil in column last_clinic_visit_date for patient: MY_SW008',\n", - " '25-29/7/22 in column hospitalisation_date for patient: MY_SB001',\n", - " '29/5-11/6/22 in column hospitalisation_date for patient: MY_SB005',\n", - " '>49 in column fbg_baseline_mg for patient: VN_VC034',\n", - " '- in column fbg_updated_mg for patient: VN_VC011',\n", - " '- in column hba1c_updated for patient: VN_VC017',\n", - " '21 da in column t1d_diagnosis_age for patient: VN_VC005',\n", - " '. in column bmi_date for patient: VN_VC044',\n", - " '- in column fbg_updated_date for patient: VN_VC011',\n", - " '- in column fbg_updated_date for patient: VN_VC017',\n", - " '- in column hba1c_updated_date for patient: VN_VC017',\n", - " 'April-19: high HbA1c in column hospitalisation_date for patient: VN_VC013',\n", - " 'April 19 (due to very high Hba1c) in column hospitalisation_date for patient: VN_VC013',\n", - " '- in column last_clinic_visit_date for patient: VN_VC011',\n", - " '- in column last_clinic_visit_date for patient: VN_VC017',\n", - " '19.51.20 in column weight for patient: TH_KN009',\n", - " 'irregular in column testing_frequency for patient: VN_VC001',\n", - " '5. irregualr in column testing_frequency for patient: VN_VC002',\n", - " '5. irregular in column testing_frequency for patient: VN_VC003',\n", - " '5 irregular in column testing_frequency for patient: VN_VC004',\n", - " '5 irregular in column testing_frequency for patient: VN_VC005',\n", - " '5 irregular in column testing_frequency for patient: VN_VC006',\n", - " '5 irregular in column testing_frequency for patient: VN_VC007',\n", - " '3 irregular in column testing_frequency for patient: VN_VC008',\n", - " '5 irregular in column testing_frequency for patient: VN_VC009',\n", - " '5 irregular in column testing_frequency for patient: VN_VC010',\n", - " '3 irregular in column testing_frequency for patient: VN_VC011',\n", - " '5 irregular in column testing_frequency for patient: VN_VC012',\n", - " '1.2 irregualar in column testing_frequency for patient: VN_VC013',\n", - " '5 Irregular in column testing_frequency for patient: VN_VC014',\n", - " 'irregular in column testing_frequency for patient: VN_VC015',\n", - " '1 in column hospitalisation_date for patient: LA_LF001',\n", - " '- in column fbg_updated_mmol for patient: MY_PN004',\n", - " '- in column hba1c_updated for patient: MY_PN004',\n", - " '- in column height for patient: MY_PN004',\n", - " '- in column weight for patient: MY_PN004',\n", - " '- in column bmi_date for patient: MY_PN004',\n", - " '- in column fbg_updated_date for patient: MY_PN004',\n", - " '- in column hba1c_updated_date for patient: MY_PN004',\n", - " '24..6 in column weight for patient: MY_SB001',\n", - " '71-134 in column fbg_baseline_mg for patient: LA_MH002',\n", - " '110-150 in column fbg_baseline_mg for patient: LA_MH003',\n", - " '80-120 in column fbg_baseline_mg for patient: LA_MH004',\n", - " '67-272 in column fbg_baseline_mg for patient: LA_MH004',\n", - " '234-240 in column fbg_baseline_mg for patient: LA_MH005',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH001',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH002',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH003',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH004',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH006',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH008',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH009',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH010',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH011',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH012',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH013',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH014',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH015',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH016',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH017',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH018',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH019',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH020',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH021',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH022',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH023',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH024',\n", - " 'Insert Date or NA in column hospitalisation_date for patient: LA_MH025',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH026',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH027',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH028',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH029',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH030',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH031',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH032',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH033',\n", - " 'Insert Date in column hospitalisation_date for patient: LA_MH034',\n", - " '- in column fbg_baseline_mg for patient: MY_PN004',\n", - " '<100 in column fbg_updated_mg for patient: LA_LF001',\n", - " '- in column height for patient: TH_SJ007',\n", - " '- in column weight for patient: TH_SJ007',\n", - " '- in column bmi_date for patient: TH_SJ007',\n", - " '14/6/2022 - 21/6/2022 in column hospitalisation_date for patient: TH_SJ025',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV001',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV003',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV006',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV007',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV008',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV009',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV010',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV013',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV016',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV019',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV023',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV024',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV025',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV027',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV029',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV033',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV034',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV035',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV036',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV037',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV043',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV049',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV051',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV060',\n", - " '≥15 in column hba1c_baseline for patient: KH_JV065',\n", - " '≥15 in column hba1c_updated for patient: KH_JV065',\n", - " '≥15 in column hba1c_updated for patient: KH_JV069',\n", - " '≥15 in column hba1c_updated for patient: KH_JV070',\n", - " '≥15 in column hba1c_updated for patient: KH_JV076',\n", - " '≥15 in column hba1c_updated for patient: KH_JV084',\n", - " '≥15 in column hba1c_updated for patient: KH_JV090',\n", - " '≥15 in column hba1c_updated for patient: KH_JV091',\n", - " '≥15 in column hba1c_updated for patient: KH_JV097',\n", - " '14-05>27-06-22 in column hospitalisation_date for patient: KH_JV086',\n", - " '>49 in column fbg_baseline_mmol for patient: VN_VC034',\n", - " 'Not Yet in column fbg_updated_mmol for patient: VN_VC026',\n", - " 'Not Yet in column fbg_updated_mmol for patient: VN_VC029',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC001',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC005',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC006',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC007',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC009',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC010',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC012',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC013',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC014',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC016',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC017',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC018',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC019',\n", - " 'mmm-yy in column fbg_updated_date for patient: VN_VC020',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH001',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH002',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH003',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH004',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH006',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH007',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH008',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH009',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH010',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH011',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH012',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH013',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH014',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH015',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH016',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH017',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH018',\n", - " 'Modified conventional TID in column testing_frequency for patient: LA_MH019',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH020',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH021',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH022',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH023',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH024',\n", - " 'Modified conventional TID in column testing_frequency for patient: LA_MH025',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH026',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH027',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH028',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH029',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH030',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH031',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH032',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH033',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH034',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH035',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH036',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH037',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH038',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH039',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH040',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH041',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH042',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH043',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH044',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH045',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH046',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH047',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH048',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH049',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH050',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH051',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH052',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH053',\n", - " 'NPH in column testing_frequency for patient: LA_MH054',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA_MH055',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA-MH056',\n", - " 'Premixed 30/70 BD in column testing_frequency for patient: LA-MH057',\n", - " 'Skin infection / hyperglycemia in column hospitalisation_date for patient: LA_MH031',\n", - " 'Hyperglycemia + mild DKA in column hospitalisation_date for patient: LA_MH032',\n", - " '.g. xxx in column fbg_updated_mmol for patient: MY_PN001',\n", - " '.g. xxx in column hba1c_updated for patient: MY_PN001',\n", - " '- in column fbg_updated_mg for patient: MM_MD022',\n", - " '_ in column fbg_updated_mg for patient: MM_MD071',\n", - " 'Hi in column hba1c_updated for patient: MM_MD071',\n", - " '- in column fbg_baseline_mmol for patient: MY_SW005',\n", - " '- in column fbg_updated_mmol for patient: MY_SW005',\n", - " '- in column fbg_updated_date for patient: MY_SW005',\n", - " '- in column hba1c_updated_date for patient: MY_SW005',\n", - " 'May 2019, Dec 2019 DKA in column hospitalisation_date for patient: MY_SW006',\n", - " 'Dec 2019, Mar 2020 DKA in column hospitalisation_date for patient: MY_SW007',\n", - " '14/02/2019 and 05/07/2021 in column hospitalisation_date for patient: KH_NP004',\n", - " '01/02/2021 and 21/11/2021 in column hospitalisation_date for patient: KH_NP014',\n", - " '68.4 - 131.4 in column fbg_baseline_mg for patient: TH_SP001',\n", - " '239.4 - 300.6 in column fbg_baseline_mg for patient: TH_SP002_SA',\n", - " '68.4 - 214.2 in column fbg_baseline_mg for patient: TH_SP003_SA',\n", - " '39.6 - 199.8 in column fbg_baseline_mg for patient: TH_SP004',\n", - " '99.0 - 199.8 in column fbg_baseline_mg for patient: TH_SP005',\n", - " '59.4 - 259.2 in column fbg_baseline_mg for patient: TH_SP006',\n", - " '100.8 - 426.6 in column fbg_baseline_mg for patient: TH_SP007',\n", - " '104.4 - 185.4 in column fbg_baseline_mg for patient: TH_SP008_SA',\n", - " '102.6 - 293.4 in column fbg_baseline_mg for patient: TH_SP009',\n", - " '79 - 95 in column fbg_baseline_mg for patient: TH_SP011',\n", - " '200-300 in column fbg_baseline_mg for patient: TH_SP012',\n", - " '62-398 in column fbg_baseline_mg for patient: TH_SP013',\n", - " '200-400 in column fbg_baseline_mg for patient: TH_SP014',\n", - " '150-300 in column fbg_baseline_mg for patient: TH_SP015',\n", - " '70-331 in column fbg_baseline_mg for patient: TH_SP016',\n", - " '96-350 in column fbg_baseline_mg for patient: TH_SP018_SA',\n", - " '60-300 in column fbg_baseline_mg for patient: TH_SP019_SA',\n", - " '63-288 in column fbg_baseline_mg for patient: TH_SP020_SA',\n", - " '77-350 in column fbg_baseline_mg for patient: TH_SP021',\n", - " '65-275 in column fbg_baseline_mg for patient: TH_SP022',\n", - " '133-319 in column fbg_baseline_mg for patient: TH_SP023_SA',\n", - " '- in column height for patient: TH_SP005',\n", - " '- in column weight for patient: TH_SP005',\n", - " '- in column fbg_baseline_mg for patient: KH_PK004',\n", - " '- in column fbg_updated_mg for patient: KH_PK006',\n", - " '- in column hba1c_baseline for patient: KH_PK003',\n", - " '- in column hba1c_baseline for patient: KH_PK004',\n", - " '- in column hba1c_baseline for patient: KH_PK005',\n", - " '- in column hba1c_baseline for patient: KH_PK006',\n", - " '- in column hba1c_baseline for patient: KH_PK007',\n", - " '- in column hba1c_baseline for patient: KH_PK008',\n", - " '- in column hba1c_baseline for patient: KH_PK009',\n", - " '- in column hba1c_baseline for patient: KH_PK014',\n", - " '- in column hba1c_baseline for patient: KH_PK017',\n", - " '- in column hba1c_baseline for patient: KH_PK019',\n", - " '- in column hba1c_baseline for patient: KH_PK021',\n", - " '- in column hba1c_updated for patient: KH_PK001',\n", - " '- in column hba1c_updated for patient: KH_PK003',\n", - " '- in column hba1c_updated for patient: KH_PK004',\n", - " '- in column hba1c_updated for patient: KH_PK005',\n", - " '- in column hba1c_updated for patient: KH_PK006',\n", - " '- in column hba1c_updated for patient: KH_PK008',\n", - " '- in column hba1c_updated for patient: KH_PK009',\n", - " '- in column hba1c_updated for patient: KH_PK014',\n", - " '- in column hba1c_updated for patient: KH_PK015',\n", - " '- in column height for patient: KH_PK001',\n", - " '- in column height for patient: KH_PK003',\n", - " '- in column height for patient: KH_PK004',\n", - " '- in column height for patient: KH_PK005',\n", - " '- in column height for patient: KH_PK006',\n", - " '- in column height for patient: KH_PK008',\n", - " '- in column height for patient: KH_PK009',\n", - " '- in column height for patient: KH_PK014',\n", - " '- in column height for patient: KH_PK015',\n", - " '- in column weight for patient: KH_PK001',\n", - " '- in column weight for patient: KH_PK003',\n", - " '- in column weight for patient: KH_PK004',\n", - " '- in column weight for patient: KH_PK005',\n", - " '- in column weight for patient: KH_PK006',\n", - " '- in column weight for patient: KH_PK008',\n", - " '- in column weight for patient: KH_PK009',\n", - " '- in column weight for patient: KH_PK014',\n", - " '- in column weight for patient: KH_PK015',\n", - " '- in column bmi_date for patient: KH_PK001',\n", - " '- in column bmi_date for patient: KH_PK003',\n", - " '- in column bmi_date for patient: KH_PK004',\n", - " '- in column bmi_date for patient: KH_PK005',\n", - " '- in column bmi_date for patient: KH_PK006',\n", - " '- in column bmi_date for patient: KH_PK008',\n", - " '- in column bmi_date for patient: KH_PK009',\n", - " '- in column bmi_date for patient: KH_PK014',\n", - " '- in column bmi_date for patient: KH_PK015',\n", - " '12/6/202 in column fbg_updated_date for patient: KH_KB039_PK',\n", - " '- in column fbg_updated_date for patient: KH_PK001',\n", - " '- in column fbg_updated_date for patient: KH_PK003',\n", - " '- in column fbg_updated_date for patient: KH_PK004',\n", - " '- in column fbg_updated_date for patient: KH_PK005',\n", - " '- in column fbg_updated_date for patient: KH_PK006',\n", - " '- in column fbg_updated_date for patient: KH_PK008',\n", - " '- in column fbg_updated_date for patient: KH_PK014',\n", - " '- in column fbg_updated_date for patient: KH_PK015',\n", - " '- in column hba1c_updated_date for patient: KH_PK001',\n", - " '- in column hba1c_updated_date for patient: KH_PK003',\n", - " '- in column hba1c_updated_date for patient: KH_PK004',\n", - " '- in column hba1c_updated_date for patient: KH_PK005',\n", - " '- in column hba1c_updated_date for patient: KH_PK006',\n", - " '- in column hba1c_updated_date for patient: KH_PK008',\n", - " '- in column hba1c_updated_date for patient: KH_PK009',\n", - " '- in column hba1c_updated_date for patient: KH_PK014',\n", - " '- in column hba1c_updated_date for patient: KH_PK015',\n", - " '- in column last_clinic_visit_date for patient: KH_PK003',\n", - " '- in column last_clinic_visit_date for patient: KH_PK005',\n", - " '- in column last_clinic_visit_date for patient: KH_PK006',\n", - " '- in column last_clinic_visit_date for patient: KH_PK008',\n", - " '- in column last_clinic_visit_date for patient: KH_PK014',\n", - " '- in column last_clinic_visit_date for patient: KH_PK015',\n", - " '- in column lost_date for patient: KH_PK001',\n", - " '- in column lost_date for patient: KH_PK003',\n", - " '- in column lost_date for patient: KH_PK005',\n", - " '- in column lost_date for patient: KH_PK006',\n", - " '- in column lost_date for patient: KH_PK008',\n", - " '- in column lost_date for patient: KH_PK014',\n", - " '20-Mei-21 in column hba1c_updated_date for patient: MY_LW010',\n", - " '18-Mei-21 in column hba1c_updated_date for patient: MY_LW011',\n", - " '13..4 in column fbg_baseline_mmol for patient: VN_VC056',\n", - " '- in column fbg_updated_mmol for patient: VN_VC011',\n", - " '100-130 in column fbg_updated_mg for patient: VN_UM001',\n", - " '110-125 in column fbg_updated_mg for patient: VN_UM001',\n", - " '130-150 in column fbg_updated_mg for patient: VN_UM001',\n", - " '130-160 in column fbg_updated_mg for patient: VN_UM001',\n", - " '110-150 in column fbg_updated_mg for patient: VN_UM001',\n", - " '90-200 in column fbg_updated_mg for patient: VN_UM001',\n", - " '130-150 in column fbg_updated_mg for patient: VN_UM002',\n", - " '120-180 in column fbg_updated_mg for patient: VN_UM002',\n", - " '120-200 in column fbg_updated_mg for patient: VN_UM002',\n", - " '100-170 in column fbg_updated_mg for patient: VN_UM002',\n", - " '120-200 in column fbg_updated_mg for patient: VN_UM003',\n", - " '100-300 in column fbg_updated_mg for patient: VN_UM003',\n", - " '100-180 in column fbg_updated_mg for patient: VN_UM003',\n", - " '100-140 in column fbg_updated_mg for patient: VN_UM003',\n", - " '190-220 in column fbg_updated_mg for patient: VN_UM008',\n", - " '180-200 in column fbg_updated_mg for patient: VN_UM008',\n", - " '150-200 in column fbg_updated_mg for patient: VN_UM008',\n", - " '1401-160 in column fbg_updated_mg for patient: VN_UM008',\n", - " '140-160 in column fbg_updated_mg for patient: VN_UM008',\n", - " '120-180 in column fbg_updated_mg for patient: VN_UM008',\n", - " '110-180 in column fbg_updated_mg for patient: VN_UM008',\n", - " '100-150 in column fbg_updated_mg for patient: VN_UM009',\n", - " '90-110 in column fbg_updated_mg for patient: VN_UM009',\n", - " '90-120 in column fbg_updated_mg for patient: VN_UM009',\n", - " '90-150 in column fbg_updated_mg for patient: VN_UM009',\n", - " '100-160 in column fbg_updated_mg for patient: VN_UM009',\n", - " '150-200 in column fbg_updated_mg for patient: VN_UM010',\n", - " '150-170 in column fbg_updated_mg for patient: VN_UM010',\n", - " '130-180 in column fbg_updated_mg for patient: VN_UM010',\n", - " '120-260 in column fbg_updated_mg for patient: VN_UM010',\n", - " '90-220 in column fbg_updated_mg for patient: VN_UM010',\n", - " '100-230 in column fbg_updated_mg for patient: VN_UM010',\n", - " '100-220 in column fbg_updated_mg for patient: VN_UM011',\n", - " '100-200 in column fbg_updated_mg for patient: VN_UM011',\n", - " '100-230 in column fbg_updated_mg for patient: VN_UM011',\n", - " '150-250 in column fbg_updated_mg for patient: VN_UM012',\n", - " '100-200 in column fbg_updated_mg for patient: VN_UM012',\n", - " '80-120 in column fbg_updated_mg for patient: VN_UM012',\n", - " '90-220 in column fbg_updated_mg for patient: VN_UM012',\n", - " 'GM in column testing_frequency for patient: VN_CH001',\n", - " 'GM in column testing_frequency for patient: VN_CH012',\n", - " 'GM in column testing_frequency for patient: VN_CH013',\n", - " '11/7/2022 - 19/7/22 in column hospitalisation_date for patient: VN_CH003',\n", - " 'N/A in column hospitalisation_date for patient: VN_VC007',\n", - " 'N/A in column hospitalisation_date for patient: VN_VC008',\n", - " 'N/A in column hospitalisation_date for patient: VN_VC046',\n", - " 'N/A in column hospitalisation_date for patient: VN_VC047',\n", - " 'N/A in column hospitalisation_date for patient: VN_VC048',\n", - " '80-340 in column fbg_baseline_mg for patient: TH_PP001_PA',\n", - " '105 - 234 in column fbg_baseline_mg for patient: TH_PP003',\n", - " '114 - 361 in column fbg_baseline_mg for patient: TH_PP004_PA',\n", - " '158 - 293 in column fbg_baseline_mg for patient: TH_PP005',\n", - " '50-220 in column fbg_baseline_mg for patient: TH_PP006',\n", - " '65-270 in column fbg_baseline_mg for patient: TH_PP007',\n", - " '80-157 in column fbg_baseline_mg for patient: TH_PP008',\n", - " '105-280 in column fbg_baseline_mg for patient: TH_PP009',\n", - " '90-200 in column fbg_baseline_mg for patient: TH_PP009',\n", - " '100-240 in column fbg_baseline_mg for patient: TH_PP010',\n", - " '60-320 in column fbg_baseline_mg for patient: TH_PP011',\n", - " '80-285 in column fbg_baseline_mg for patient: TH_PP012',\n", - " '64-280 in column fbg_baseline_mg for patient: TH_PP013',\n", - " '60-290 in column fbg_baseline_mg for patient: TH_PP013',\n", - " '80-130 in column fbg_baseline_mg for patient: TH_PP013',\n", - " '60-430 in column fbg_baseline_mg for patient: TH_PP014',\n", - " '98-500 in column fbg_baseline_mg for patient: TH_PP015',\n", - " '70-250 in column fbg_baseline_mg for patient: TH_PP016',\n", - " '80-180 in column fbg_baseline_mg for patient: TH_PP017',\n", - " '130-250 in column fbg_baseline_mg for patient: TH_PP017',\n", - " '80-250 in column fbg_baseline_mg for patient: TH_PP018',\n", - " '90-350 in column fbg_baseline_mg for patient: TH_PP019',\n", - " '100-350 in column fbg_baseline_mg for patient: TH_PP019',\n", - " '- in column fbg_updated_mg for patient: TH_PP019',\n", - " '8.1 - 10.7 in column hba1c_baseline for patient: TH_PP001_PA',\n", - " '7.6 - 9.2 in column hba1c_baseline for patient: TH_PP002_PA',\n", - " '8.5 - 10.7 in column hba1c_baseline for patient: TH_PP003',\n", - " '10.5 - 12.2 in column hba1c_baseline for patient: TH_PP004_PA',\n", - " '10.4 - 13.1 in column hba1c_baseline for patient: TH_PP005',\n", - " '7.1-13 in column hba1c_baseline for patient: TH_PP011',\n", - " '71-294 in column fbg_baseline_mg for patient: LA_MH002',\n", - " 'DKA in column fbg_baseline_mg for patient: LA_MH003',\n", - " '80-130 in column fbg_baseline_mg for patient: LA_MH004',\n", - " 'Aug-7 in column fbg_updated_date for patient: LA_MH002',\n", - " 'Aug-1 in column fbg_updated_date for patient: LA_MH008',\n", - " '156-333 in column fbg_baseline_mg for patient: TH_UT001',\n", - " '66-435 in column fbg_baseline_mg for patient: TH_UT002',\n", - " '67-390 in column fbg_baseline_mg for patient: TH_UT004',\n", - " '65-250 in column fbg_baseline_mg for patient: TH_UT005',\n", - " '80-325 in column fbg_baseline_mg for patient: TH_UT006',\n", - " '83-269 in column fbg_baseline_mg for patient: TH_UT007',\n", - " '78-388 in column fbg_baseline_mg for patient: TH_UT008',\n", - " 'known in column fbg_baseline_mg for patient: TH_UT009',\n", - " '110-280 in column fbg_baseline_mg for patient: TH_UT010',\n", - " '85-438 in column fbg_baseline_mg for patient: TH_UT011',\n", - " '46-349 in column fbg_baseline_mg for patient: TH_UT012',\n", - " '130-552 in column fbg_baseline_mg for patient: TH_UT013',\n", - " '130-532 in column fbg_baseline_mg for patient: TH_UT013',\n", - " '195-427 in column fbg_updated_mg for patient: TH_UT004',\n", - " 'in in column t1d_diagnosis_date for patient: TH_UT007',\n", - " 'in in column t1d_diagnosis_date for patient: TH_UT008',\n", - " 'in in column t1d_diagnosis_date for patient: TH_UT012',\n", - " 'in in column t1d_diagnosis_date for patient: TH_UT013',\n", - " '67-390 in column fbg_baseline_mg for patient: TH_UT004_MH',\n", - " 'N/A in column fbg_baseline_mg for patient: KH_JV026_KB',\n", - " 'N/A in column hba1c_baseline for patient: KH_JV026_KB',\n", - " 'stamlor 5mg in column hospitalisation_date for patient: KH_KB089',\n", - " '300-400 in column fbg_baseline_mg for patient: TH_CP003',\n", - " '300-400 in column fbg_baseline_mg for patient: TH_CP004',\n", - " '54-484 in column fbg_baseline_mg for patient: TH_CP020',\n", - " '122-278 in column fbg_baseline_mg for patient: TH_CP021',\n", - " '100-546 in column fbg_baseline_mg for patient: TH_CP022',\n", - " '47-562 in column fbg_baseline_mg for patient: TH_CP023',\n", - " 'MBG35-562 in column fbg_baseline_mg for patient: TH_CP026',\n", - " 'SMBG 52-442 in column fbg_updated_mg for patient: TH_CP001',\n", - " 'SMBG 60-398 in column fbg_updated_mg for patient: TH_CP001',\n", - " 'SMBG 49-355 in column fbg_updated_mg for patient: TH_CP001',\n", - " 'SMBG 53-476 in column fbg_updated_mg for patient: TH_CP001',\n", - " 'SMBG 55-345 in column fbg_updated_mg for patient: TH_CP001',\n", - " 'SMBG 47-579 in column fbg_updated_mg for patient: TH_CP002',\n", - " 'SMBG 44-597 in column fbg_updated_mg for patient: TH_CP002',\n", - " 'SMBG 49-593 in column fbg_updated_mg for patient: TH_CP002',\n", - " 'SMBG 55-551 in column fbg_updated_mg for patient: TH_CP002',\n", - " 'SMBG 52-572 in column fbg_updated_mg for patient: TH_CP002',\n", - " 'BG 37-373 in column fbg_updated_mg for patient: TH_CP003',\n", - " 'SMBG 131-410 in column fbg_updated_mg for patient: TH_CP004',\n", - " 'BG 36-336 in column fbg_updated_mg for patient: TH_CP005',\n", - " 'BG 46-298 in column fbg_updated_mg for patient: TH_CP005',\n", - " 'SMBG 42-595 in column fbg_updated_mg for patient: TH_CP005',\n", - " 'SMBG 38-585 in column fbg_updated_mg for patient: TH_CP005',\n", - " 'BS 450 in column fbg_updated_mg for patient: TH_CP005',\n", - " 'BG 30-500 in column fbg_updated_mg for patient: TH_CP005',\n", - " 'BG 60-500 in column fbg_updated_mg for patient: TH_CP005',\n", - " 'SMBG 113-508 in column fbg_updated_mg for patient: TH_CP006',\n", - " 'SMBG 70-300 in column fbg_updated_mg for patient: TH_CP006',\n", - " 'SMBG 150-400 in column fbg_updated_mg for patient: TH_CP006',\n", - " 'SMBG 67-525 in column fbg_updated_mg for patient: TH_CP006',\n", - " 'SMBG 56-594 in column fbg_updated_mg for patient: TH_CP006',\n", - " 'SMBG 44-507 in column fbg_updated_mg for patient: TH_CP006',\n", - " 'SMBG 41-598 in column fbg_updated_mg for patient: TH_CP006',\n", - " 'SMBG 70-498 in column fbg_updated_mg for patient: TH_CP006',\n", - " 'FBS 151 in column fbg_updated_mg for patient: TH_CP007',\n", - " 'SMBG50-391 in column fbg_updated_mg for patient: TH_CP008',\n", - " 'SMBG 89-392 in column fbg_updated_mg for patient: TH_CP009',\n", - " 'SMBG 53-401 in column fbg_updated_mg for patient: TH_CP009',\n", - " 'SMBG 61-379 in column fbg_updated_mg for patient: TH_CP009',\n", - " 'SMBG 52-387 in column fbg_updated_mg for patient: TH_CP009',\n", - " 'SMBG 66-502 in column fbg_updated_mg for patient: TH_CP010',\n", - " 'SMBG 97-544 in column fbg_updated_mg for patient: TH_CP010',\n", - " 'SMBG 53-540 in column fbg_updated_mg for patient: TH_CP010',\n", - " 'SMBG 60-470 in column fbg_updated_mg for patient: TH_CP010',\n", - " 'SMBG 57-544 in column fbg_updated_mg for patient: TH_CP010',\n", - " 'DSMP 65-423 in column fbg_updated_mg for patient: TH_CP011',\n", - " 'DSMP 136-516 in column fbg_updated_mg for patient: TH_CP011',\n", - " 'SMBG 52-325 in column fbg_updated_mg for patient: TH_CP012',\n", - " 'SMBG 5116-124 in column fbg_updated_mg for patient: TH_CP012',\n", - " 'SMBG50-353 in column fbg_updated_mg for patient: TH_CP012',\n", - " 'SMBG 28-556 in column fbg_updated_mg for patient: TH_CP013',\n", - " 'SMBG 52-568 in column fbg_updated_mg for patient: TH_CP013',\n", - " 'SMBG 47-524 in column fbg_updated_mg for patient: TH_CP013',\n", - " 'SMBG 69-570 in column fbg_updated_mg for patient: TH_CP013',\n", - " 'SMBG 73-512 in column fbg_updated_mg for patient: TH_CP013',\n", - " 'SMBG 96-690 in column fbg_updated_mg for patient: TH_CP013',\n", - " 'SMBG 96-512 in column fbg_updated_mg for patient: TH_CP014',\n", - " 'BG 50-429 in column fbg_updated_mg for patient: TH_CP015',\n", - " 'BG 43-468 in column fbg_updated_mg for patient: TH_CP015',\n", - " 'BG 36-501 in column fbg_updated_mg for patient: TH_CP015',\n", - " 'BG 55-436 in column fbg_updated_mg for patient: TH_CP015',\n", - " 'BG 62-396 in column fbg_updated_mg for patient: TH_CP015',\n", - " 'BG51-455 in column fbg_updated_mg for patient: TH_CP015',\n", - " 'BG 33-535 in column fbg_updated_mg for patient: TH_CP015',\n", - " 'BG 43-542 in column fbg_updated_mg for patient: TH_CP015',\n", - " 'BG 48-554 in column fbg_updated_mg for patient: TH_CP015',\n", - " 'SMBG 60-345 in column fbg_updated_mg for patient: TH_CP016',\n", - " 'SMBG 87-300 in column fbg_updated_mg for patient: TH_CP016',\n", - " 'SMBG 56-356 in column fbg_updated_mg for patient: TH_CP016',\n", - " 'SMBG 58-350 in column fbg_updated_mg for patient: TH_CP016',\n", - " 'SMBG 42-375 in column fbg_updated_mg for patient: TH_CP016',\n", - " 'SMBG 67-482 in column fbg_updated_mg for patient: TH_CP016',\n", - " 'SMBG 40-378 in column fbg_updated_mg for patient: TH_CP016',\n", - " 'SMBG 80-543 in column fbg_updated_mg for patient: TH_CP017',\n", - " 'SMBG 44-593 in column fbg_updated_mg for patient: TH_CP018',\n", - " 'SMBG 161-471 in column fbg_updated_mg for patient: TH_CP019',\n", - " 'SMBG 93-373 in column fbg_updated_mg for patient: TH_CP019',\n", - " 'SMBG 208-536 in column fbg_updated_mg for patient: TH_CP019',\n", - " 'SMBG 128-466 in column fbg_updated_mg for patient: TH_CP019',\n", - " 'SMBG 61-299 in column fbg_updated_mg for patient: TH_CP019',\n", - " 'SMBG 59-559 in column fbg_updated_mg for patient: TH_CP019',\n", - " 'SMBG 80-436 in column fbg_updated_mg for patient: TH_CP020',\n", - " 'SMBG 68-339 in column fbg_updated_mg for patient: TH_CP021',\n", - " 'SMBG 95-500 in column fbg_updated_mg for patient: TH_CP022',\n", - " 'SMBG44-514 in column fbg_updated_mg for patient: TH_CP022',\n", - " 'SMBG 26-583 in column fbg_updated_mg for patient: TH_CP022',\n", - " 'SMBG 50-400 in column fbg_updated_mg for patient: TH_CP022',\n", - " 'SMBG 66-598 in column fbg_updated_mg for patient: TH_CP022',\n", - " 'SMBG 45-498 in column fbg_updated_mg for patient: TH_CP023',\n", - " 'SMBG 78-466 in column fbg_updated_mg for patient: TH_CP023',\n", - " 'SMBG 80-477 in column fbg_updated_mg for patient: TH_CP023',\n", - " 'SMBG 64-173 in column fbg_updated_mg for patient: TH_CP023',\n", - " 'SMBG 60-427 in column fbg_updated_mg for patient: TH_CP023',\n", - " 'SMBG 47-555 in column fbg_updated_mg for patient: TH_CP023',\n", - " 'SMBG 45-564 in column fbg_updated_mg for patient: TH_CP023',\n", - " 'SMBG 90-561 in column fbg_updated_mg for patient: TH_CP023',\n", - " 'SMBG 45-588 in column fbg_updated_mg for patient: TH_CP023',\n", - " 'SMBG 43-466 in column fbg_updated_mg for patient: TH_CP023',\n", - " 'MBG 103-172 in column fbg_updated_mg for patient: TH_CP024_CA',\n", - " 'SMBG 62-290 in column fbg_updated_mg for patient: TH_CP024_CA',\n", - " 'SMBG 86-297 in column fbg_updated_mg for patient: TH_CP024_CA',\n", - " 'SMBG 89-293 in column fbg_updated_mg for patient: TH_CP024_CA',\n", - " 'MBG 86-290 in column fbg_updated_mg for patient: TH_CP024_CA',\n", - " 'SMBG 61-448 in column fbg_updated_mg for patient: TH_CP025',\n", - " 'SMBG 61-416 in column fbg_updated_mg for patient: TH_CP025',\n", - " 'SMBG 57-394 in column fbg_updated_mg for patient: TH_CP025',\n", - " 'SMBG 57-309 in column fbg_updated_mg for patient: TH_CP025',\n", - " 'SMBG 46-426 in column fbg_updated_mg for patient: TH_CP025',\n", - " 'SMBG 52-386 in column fbg_updated_mg for patient: TH_CP025',\n", - " 'SMBG 68-314 in column fbg_updated_mg for patient: TH_CP026',\n", - " 'SMBG 62-479 in column fbg_updated_mg for patient: TH_CP026',\n", - " 'SMBG 59-432 in column fbg_updated_mg for patient: TH_CP026',\n", - " 'SMBG 64-379 in column fbg_updated_mg for patient: TH_CP026',\n", - " 'SMBG 58-470 in column fbg_updated_mg for patient: TH_CP026',\n", - " 'SMBG 49-357 in column fbg_updated_mg for patient: TH_CP026',\n", - " 'SMBG 81-222 in column fbg_updated_mg for patient: TH_CP027',\n", - " 'SMBG 118-199 in column fbg_updated_mg for patient: TH_CP027',\n", - " 'SMBG 200-400 in column fbg_updated_mg for patient: TH_CP028_CA',\n", - " 'SMBG 77-394 in column fbg_updated_mg for patient: TH_CP028_CA',\n", - " 'SMBG 80-300 in column fbg_updated_mg for patient: TH_CP028_CA',\n", - " 'SMBG 59-562 in column fbg_updated_mg for patient: TH_CP030',\n", - " 'SMBG 110-325 in column fbg_updated_mg for patient: TH_CP031_CA',\n", - " '19/5/2022-08/06/2022 in column hospitalisation_date for patient: TH_CP005',\n", - " '19/4/022-5/5/2022 in column hospitalisation_date for patient: TH_CP005',\n", - " '5/8/2022-23/8/2022 in column hospitalisation_date for patient: TH_CP005',\n", - " '25-26/10/2022 in column hospitalisation_date for patient: TH_CP005',\n", - " '20/12/2022-06/01/2023 in column hospitalisation_date for patient: TH_CP005',\n", - " '16/6/2022-23/6/2022 in column hospitalisation_date for patient: TH_CP005',\n", - " '25-27/7/2022 in column hospitalisation_date for patient: TH_CP005',\n", - " '25-30/4/2022 in column hospitalisation_date for patient: TH_CP006',\n", - " '8/8/2022-15/8/2022 in column hospitalisation_date for patient: TH_CP006',\n", - " '3-13/10/2022 in column hospitalisation_date for patient: TH_CP006',\n", - " '25/7/2022-2/8/2022 in column hospitalisation_date for patient: TH_CP010',\n", - " '24/8/2022-30/8/2022 in column hospitalisation_date for patient: TH_CP013',\n", - " '26-28/10/2022 in column hospitalisation_date for patient: TH_CP015',\n", - " '25-29/4/2022 in column hospitalisation_date for patient: TH_CP019',\n", - " '18-23/4/2022 in column hospitalisation_date for patient: TH_CP022',\n", - " '26-30/10/2022 in column hospitalisation_date for patient: TH_CP026',\n", - " 'URRENT in column last_clinic_visit_date for patient: TH_CP012',\n", - " 'Unknown in column t1d_diagnosis_date for patient: TH_CP027',\n", - " '31 oct 222 in column fbg_updated_date for patient: KH_NP019',\n", - " 'N/A in column hba1c_updated for patient: LA_LF001',\n", - " 'May 2019, Dec 2019 DKA, August 2020 DKA, DKA June 2021 in column hospitalisation_date for patient: MY_SW006',\n", - " '- in column height for patient: KH_CD003',\n", - " '- in column weight for patient: KH_CD003',\n", - " '- in column bmi_date for patient: KH_CD003',\n", - " '39 Aug 2022 in column last_clinic_visit_date for patient: KH_CD008']" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "subdf = df[(~df.warning.isna()) & (df.warning.str.contains(\"Could not convert value\", regex=False))]\n", - "\n", - "subdf.warning.str.strip(\"Could not convert value \").unique().tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['2020_Penang General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2019_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_Sultanah Bahiyah A4D Tracker_DC_patient_raw.log',\n", - " '2021_Vietnam National Children_s Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Khon Kaen Hospital A4D Tracker_patient_raw.log',\n", - " \"2017_Vietnam National Children's Hospital A4D Tracker_patient_raw.log\",\n", - " '2019_Lao Friends Hospital for Children A4D Tracker_DC_patient_raw.log',\n", - " '2019_Penang General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2019_Sultanah Bahiyah Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2019_Mahosot Hospital A4D Tracker_patient_raw.log',\n", - " '2021_Penang General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2020_Lao Friends Hospital for Children A4D Tracker_DC_patient_raw.log',\n", - " '2022_Siriraj Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Jayavarman VII Hospital A4D Tracker_patient_raw.log',\n", - " '2018_Vietnam National Children_s Hospital A4D Tracker_patient_raw.log',\n", - " '2021_Mahosot Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2018_Penang General Hospital A4D Tracker_DC_patient_raw.log',\n", - " \"2022_Mandalay Children's Hospital A4D Tracker_patient_raw.log\",\n", - " '2020_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2021_NPH A4D Tracker_patient_raw.log',\n", - " '2022_Sunprasitthiprasong Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Preah Kossamak Hospital A4D Tracker_patient_raw.log',\n", - " \"2021_Likas Women & Children's Hospital A4D Tracker_DC_patient_raw.log\",\n", - " \"2020_Vietnam National Children's Hospital A4D Tracker_patient_raw.log\",\n", - " '2022_Uni Med Centre A4D Tracker_patient_raw.log',\n", - " \"2022_Children's Hospital 2 A4D Tracker_patient_raw.log\",\n", - " '2019_Vietnam National Children_s Hospital A4D Tracker_patient_raw.log',\n", - " '2020_Sultanah Bahiyah Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_Pahol Polpayuhasena A4D Tracker_patient_raw.log',\n", - " '2017_Mahosot Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Udon Thani Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Mahosot Hosptial A4D Tracker_DC_patient_raw.log',\n", - " '2022_Kantha Bopha Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Chiang Mai Maharaj Nakorn A4D Tracker_patient_raw.log',\n", - " '2018_Mahosot Hospital A4D Tracker_patient_raw.log',\n", - " '2022_NPH A4D Tracker_patient_raw.log',\n", - " '2018_Lao Friends Hospital for Children A4D Tracker_DC_patient_raw.log',\n", - " '2021_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_CDA A4D Tracker_patient_raw.log']" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "subdf.file.unique().tolist()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### invalid values (outside range)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
03
12sblood_pressure_mmhg
18618.5hba1c_baseline
190200fbg_baseline_mmol
66231age
66332age
6743.9hba1c_baseline
167911blood_pressure_sys_mmhg
168312blood_pressure_sys_mmhg
169615blood_pressure_sys_mmhg
170017blood_pressure_sys_mmhg
170319blood_pressure_sys_mmhg
17062blood_pressure_dias_mmhg
17103blood_pressure_dias_mmhg
17197blood_pressure_dias_mmhg
17236blood_pressure_dias_mmhg
172713blood_pressure_dias_mmhg
210013height
245926age
246230age
24740hba1c_baseline
3618762weight
362090.5387052965143bmi
364929age
365328age
366127age
36620.099hba1c_baseline
367510blood_pressure_dias_mmhg
6568122age
658118.9hba1c_baseline
862126hba1c_baseline
86333.4hba1c_baseline
864519.3hba1c_baseline
11054695blood_pressure_dias_mmhg
144796.9height
1476710.4height
165660.1238hba1c_baseline
1826319hba1c_baseline
272080.15hba1c_baseline
\n", - "
" - ], - "text/plain": [ - " 0 3\n", - "12 s blood_pressure_mmhg\n", - "186 18.5 hba1c_baseline\n", - "190 200 fbg_baseline_mmol\n", - "662 31 age\n", - "663 32 age\n", - "674 3.9 hba1c_baseline\n", - "1679 11 blood_pressure_sys_mmhg\n", - "1683 12 blood_pressure_sys_mmhg\n", - "1696 15 blood_pressure_sys_mmhg\n", - "1700 17 blood_pressure_sys_mmhg\n", - "1703 19 blood_pressure_sys_mmhg\n", - "1706 2 blood_pressure_dias_mmhg\n", - "1710 3 blood_pressure_dias_mmhg\n", - "1719 7 blood_pressure_dias_mmhg\n", - "1723 6 blood_pressure_dias_mmhg\n", - "1727 13 blood_pressure_dias_mmhg\n", - "2100 13 height\n", - "2459 26 age\n", - "2462 30 age\n", - "2474 0 hba1c_baseline\n", - "3618 762 weight\n", - "3620 90.5387052965143 bmi\n", - "3649 29 age\n", - "3653 28 age\n", - "3661 27 age\n", - "3662 0.099 hba1c_baseline\n", - "3675 10 blood_pressure_dias_mmhg\n", - "6568 122 age\n", - "6581 18.9 hba1c_baseline\n", - "8621 26 hba1c_baseline\n", - "8633 3.4 hba1c_baseline\n", - "8645 19.3 hba1c_baseline\n", - "11054 695 blood_pressure_dias_mmhg\n", - "14479 6.9 height\n", - "14767 10.4 height\n", - "16566 0.1238 hba1c_baseline\n", - "18263 19 hba1c_baseline\n", - "27208 0.15 hba1c_baseline" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "subdf = df[(~df.warning.isna()) & (df.warning.str.contains(\"Found invalid value\", regex=False))]\n", - "\n", - "subdf.warning.str.strip(\"Found invalid value \").str.split(expand=True)[[0,3]].drop_duplicates()" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['2020_Penang General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2019_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2021_Vietnam National Children_s Hospital A4D Tracker_patient_raw.log',\n", - " \"2017_Vietnam National Children's Hospital A4D Tracker_patient_raw.log\",\n", - " '2019_Penang General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2019_Mahosot Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Siriraj Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Jayavarman VII Hospital A4D Tracker_patient_raw.log',\n", - " '2018_Vietnam National Children_s Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2021_Mahosot Hospital A4D Tracker_DC_patient_raw.log',\n", - " \"2022_Mandalay Children's Hospital A4D Tracker_patient_raw.log\",\n", - " '2020_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_Sunprasitthiprasong Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Preah Kossamak Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Phattalung Hospital A4D Tracker_patient_raw.log',\n", - " \"2021_Likas Women & Children's Hospital A4D Tracker_DC_patient_raw.log\",\n", - " \"2020_Vietnam National Children's Hospital A4D Tracker_patient_raw.log\",\n", - " \"2022_Children's Hospital 2 A4D Tracker_patient_raw.log\",\n", - " '2019_Vietnam National Children_s Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Pahol Polpayuhasena A4D Tracker_patient_raw.log',\n", - " '2022_Udon Thani Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Mukdahan Hospital A4D Tracker_patient_raw.log',\n", - " \"2022_Likas Women & Children's Hospital A4D Tracker_DC_patient_raw.log\",\n", - " '2022_Mahosot Hosptial A4D Tracker_DC_patient_raw.log',\n", - " '2022_Kantha Bopha Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Hat Yai Hospital A4D Tracker_patient_raw.log',\n", - " '2021_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_CDA A4D Tracker_patient_raw.log']" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "subdf.file.unique().tolist()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### invalid values (not allowed)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
36
234Basal-bolus,NAinsulin_regimen
243Basal-bolusfor
246NA,NAinsulin_regimen
318Pendang,column
330Kedah,column
342Jitracolumn
354Jitra,column
366Alorfor
378Changloon,column
686Namdinhprovince
698Langsonprovince
722Hoabinhprovince
734Hanoiprovince
758Hungyenprovince
770Hagiangprovince
782Nghecolumn
794Thaicolumn
806Caocolumn
818Yencolumn
830Hungcolumn
914Bắccolumn
950Tháicolumn
962Hàcolumn
1018Modifiedconventional
1030Self-mixedfor
1054Basal-bolus,Basal-bolusinsulin_regimen
1138Self-mixedcolumn
1336SMBGsupport_from_a4d
1341glucometer,trips,lancets
1433Novorapid,column
1434Premixedcolumn
1436Mixtardinsulin_regimen
1440Actrapid,column
1442Basalinsulin_regimen
1445lantusinsulin_regimen
1454mixtardinsulin_regimen
1459novorapid,column
1465Novomix:insulin_regimen
1468Premixedbolus
1625Other,NAinsulin_regimen
1793Kedah,Malaysiaprovince
2109Salavanhprovince
2121Attapeurprovince
2155Savannaketprovince
2179Xeingcolumn
2191Champhasackprovince
2263Vientianecolumn
2264LA_MH007salyprovince
2274Hauphanhprovince
2293Bolikhamxayprovince
2374Basal-bolosinsulin_regimen
2395Othersinsulin_regimen
2546Modifiedfor
3002Pathumtaniprovince
3014Ayutthayaprovince
3026Chumpornprovince
3038Nakornsawanprovince
3050Suphanburiprovince
3062Ubolratchathaniprovince
3074Choburiprovince
3244basalcolumn
5832Mixcolumn
5842Mix-bolusinsulin_regimen
6314basal-bolus,basal-bolusinsulin_regimen
6629Premixedfor
6797Basal-bolusMDI
73552insulin_regimen
74913insulin_regimen
7765NPHinsulin_regimen
77684insulin_regimen
7782Basalcolumn
7784Veintianecolumn
8156Vientainecolumn
8288Khammouanprovince
8298Xaicolumn
8317Laungnamthaprovince
8569Kualacolumn
8685Basal-boluscolumn
9033Others,NAinsulin_regimen
9633Myincolumn
9645Mandalayprovince
9657Pyinfor
9669Myingprovince
9693Ngacolumn
9705Matefor
9717Chaungcolumn
9729Moncolumn
9741Sintcolumn
9753Kyoutcolumn
9765Tafor
9777Lashioprovince
9801Palateprovince
9813Kanmaprovince
9837Taungcolumn
9873Kantfor
9885Myitcolumn
9897Thafor
9945Nayfor
9981Pinlebuprovince
10005Pwintcolumn
10017Moegokprovince
10029Nyoungcolumn
10041Moecolumn
10053Madayaprovince
10065Pokakkuprovince
10077Kyaukseprovince
10113Wattcolumn
10125Shwecolumn
10137Yafor
10161Naungcolumn
10185Kalayprovince
10197Ywarcolumn
10209Myitfor
10245Hanfor
10257Pacolumn
10305Pakokuprovince
10329Sagaingprovince
10341Sacolumn
10353Kyaukmeprovince
10389Yayfor
10413Myitthaprovince
10461MaHlaingprovince
10473Meikhtilaprovince
10509Kathaprovince
10533Tangyanprovince
10545Myocolumn
10568Natogyiprovince
10595Monywaprovince
10702-insulin_regimen
10738diabetesglucose
107411stto
10745Kompungcolumn
11057Basal-Bolus,NAinsulin_regimen
11333Srisaketprovince
11345Laosprovince
11393Yasotornprovince
13342-,NAinsulin_regimen
13630Takeoprovince
13642Kompongcolumn
13666Krochasprovince
13714Kompotprovince
13738Precolumn
14486SevereAKI
14487Bongawanprovince
14494Likasprovince
14501Penampangprovince
14508Kudatprovince
14522Sipitangprovince
14536Tauranprovince
14543Kotacolumn
14550Sepanggarprovince
15889Other,Otherinsulin_regimen
15901Premixed30/70
15999Binhcolumn
16011Hofor
16035Vinhcolumn
16071Cancolumn
16128Basal-bolusBD
16290Dongcolumn
16302Vungcolumn
16326TPcolumn
17793Insulin,&
17804Insulin,HbA1c
18011Vientianefor
18033Luangphabangprovince
18088savannaketprovince
18391Loeyprovince
19605NPH,NAinsulin_regimen
20690Basal-Bolus,Basal-Bolusinsulin_regimen
22678Tbongcolumn
22690Sihanoukvilleprovince
22726Kampongsoamprovince
22738Kratieprovince
22750Kampongspeuprovince
22786Kampongthomprovince
22846Porsatprovince
22858Kampongchnangprovince
22906Kampongchamprovince
23002Battambongprovince
24091Lumphangprovince
24103Phrayowprovince
24127Chiangmaiprovince
24139Lumphunprovince
24163Phayoaprovince
24175Lumpangprovince
24199Phrayawprovince
24831Sponsorfor
25660Others,Othersinsulin_regimen
25662Other,Othersinsulin_regimen
25913Basalfor
26124Other,Basal-bolusinsulin_regimen
26481Songkhlaprovince
26529Patthalungprovince
26721Yalaprovince
26769Narathiwatprovince
26793Pattalungprovince
26841Satunprovince
26961Nakornsrithammaratprovince
27273Basal-bolus(AN/HI)
27285Basal-bolus(AN)
27417Basal-bolusconventional
27446Premixed(AN)
\n", - "
" - ], - "text/plain": [ - " 3 6\n", - "234 Basal-bolus,NA insulin_regimen\n", - "243 Basal-bolus for\n", - "246 NA,NA insulin_regimen\n", - "318 Pendang, column\n", - "330 Kedah, column\n", - "342 Jitra column\n", - "354 Jitra, column\n", - "366 Alor for\n", - "378 Changloon, column\n", - "686 Namdinh province\n", - "698 Langson province\n", - "722 Hoabinh province\n", - "734 Hanoi province\n", - "758 Hungyen province\n", - "770 Hagiang province\n", - "782 Nghe column\n", - "794 Thai column\n", - "806 Cao column\n", - "818 Yen column\n", - "830 Hung column\n", - "914 Bắc column\n", - "950 Thái column\n", - "962 Hà column\n", - "1018 Modified conventional\n", - "1030 Self-mixed for\n", - "1054 Basal-bolus,Basal-bolus insulin_regimen\n", - "1138 Self-mixed column\n", - "1336 SMBG support_from_a4d\n", - "1341 glucometer, trips,lancets\n", - "1433 Novorapid, column\n", - "1434 Premixed column\n", - "1436 Mixtard insulin_regimen\n", - "1440 Actrapid, column\n", - "1442 Basal insulin_regimen\n", - "1445 lantus insulin_regimen\n", - "1454 mixtard insulin_regimen\n", - "1459 novorapid, column\n", - "1465 Novomix: insulin_regimen\n", - "1468 Premixed bolus\n", - "1625 Other,NA insulin_regimen\n", - "1793 Kedah,Malaysia province\n", - "2109 Salavanh province\n", - "2121 Attapeur province\n", - "2155 Savannaket province\n", - "2179 Xeing column\n", - "2191 Champhasack province\n", - "2263 Vientiane column\n", - "2264 LA_MH007saly province\n", - "2274 Hauphanh province\n", - "2293 Bolikhamxay province\n", - "2374 Basal-bolos insulin_regimen\n", - "2395 Others insulin_regimen\n", - "2546 Modified for\n", - "3002 Pathumtani province\n", - "3014 Ayutthaya province\n", - "3026 Chumporn province\n", - "3038 Nakornsawan province\n", - "3050 Suphanburi province\n", - "3062 Ubolratchathani province\n", - "3074 Choburi province\n", - "3244 basal column\n", - "5832 Mix column\n", - "5842 Mix-bolus insulin_regimen\n", - "6314 basal-bolus,basal-bolus insulin_regimen\n", - "6629 Premixed for\n", - "6797 Basal-bolus MDI\n", - "7355 2 insulin_regimen\n", - "7491 3 insulin_regimen\n", - "7765 NPH insulin_regimen\n", - "7768 4 insulin_regimen\n", - "7782 Basal column\n", - "7784 Veintiane column\n", - "8156 Vientaine column\n", - "8288 Khammouan province\n", - "8298 Xai column\n", - "8317 Laungnamtha province\n", - "8569 Kuala column\n", - "8685 Basal-bolus column\n", - "9033 Others,NA insulin_regimen\n", - "9633 Myin column\n", - "9645 Mandalay province\n", - "9657 Pyin for\n", - "9669 Mying province\n", - "9693 Nga column\n", - "9705 Mate for\n", - "9717 Chaung column\n", - "9729 Mon column\n", - "9741 Sint column\n", - "9753 Kyout column\n", - "9765 Ta for\n", - "9777 Lashio province\n", - "9801 Palate province\n", - "9813 Kanma province\n", - "9837 Taung column\n", - "9873 Kant for\n", - "9885 Myit column\n", - "9897 Tha for\n", - "9945 Nay for\n", - "9981 Pinlebu province\n", - "10005 Pwint column\n", - "10017 Moegok province\n", - "10029 Nyoung column\n", - "10041 Moe column\n", - "10053 Madaya province\n", - "10065 Pokakku province\n", - "10077 Kyaukse province\n", - "10113 Watt column\n", - "10125 Shwe column\n", - "10137 Ya for\n", - "10161 Naung column\n", - "10185 Kalay province\n", - "10197 Ywar column\n", - "10209 Myit for\n", - "10245 Han for\n", - "10257 Pa column\n", - "10305 Pakoku province\n", - "10329 Sagaing province\n", - "10341 Sa column\n", - "10353 Kyaukme province\n", - "10389 Yay for\n", - "10413 Myittha province\n", - "10461 MaHlaing province\n", - "10473 Meikhtila province\n", - "10509 Katha province\n", - "10533 Tangyan province\n", - "10545 Myo column\n", - "10568 Natogyi province\n", - "10595 Monywa province\n", - "10702 - insulin_regimen\n", - "10738 diabetes glucose\n", - "10741 1st to\n", - "10745 Kompung column\n", - "11057 Basal-Bolus,NA insulin_regimen\n", - "11333 Srisaket province\n", - "11345 Laos province\n", - "11393 Yasotorn province\n", - "13342 -,NA insulin_regimen\n", - "13630 Takeo province\n", - "13642 Kompong column\n", - "13666 Krochas province\n", - "13714 Kompot province\n", - "13738 Pre column\n", - "14486 Severe AKI\n", - "14487 Bongawan province\n", - "14494 Likas province\n", - "14501 Penampang province\n", - "14508 Kudat province\n", - "14522 Sipitang province\n", - "14536 Tauran province\n", - "14543 Kota column\n", - "14550 Sepanggar province\n", - "15889 Other,Other insulin_regimen\n", - "15901 Premixed 30/70\n", - "15999 Binh column\n", - "16011 Ho for\n", - "16035 Vinh column\n", - "16071 Can column\n", - "16128 Basal-bolus BD\n", - "16290 Dong column\n", - "16302 Vung column\n", - "16326 TP column\n", - "17793 Insulin, &\n", - "17804 Insulin, HbA1c\n", - "18011 Vientiane for\n", - "18033 Luangphabang province\n", - "18088 savannaket province\n", - "18391 Loey province\n", - "19605 NPH,NA insulin_regimen\n", - "20690 Basal-Bolus,Basal-Bolus insulin_regimen\n", - "22678 Tbong column\n", - "22690 Sihanoukville province\n", - "22726 Kampongsoam province\n", - "22738 Kratie province\n", - "22750 Kampongspeu province\n", - "22786 Kampongthom province\n", - "22846 Porsat province\n", - "22858 Kampongchnang province\n", - "22906 Kampongcham province\n", - "23002 Battambong province\n", - "24091 Lumphang province\n", - "24103 Phrayow province\n", - "24127 Chiangmai province\n", - "24139 Lumphun province\n", - "24163 Phayoa province\n", - "24175 Lumpang province\n", - "24199 Phrayaw province\n", - "24831 Sponsor for\n", - "25660 Others,Others insulin_regimen\n", - "25662 Other,Others insulin_regimen\n", - "25913 Basal for\n", - "26124 Other,Basal-bolus insulin_regimen\n", - "26481 Songkhla province\n", - "26529 Patthalung province\n", - "26721 Yala province\n", - "26769 Narathiwat province\n", - "26793 Pattalung province\n", - "26841 Satun province\n", - "26961 Nakornsrithammarat province\n", - "27273 Basal-bolus (AN/HI)\n", - "27285 Basal-bolus (AN)\n", - "27417 Basal-bolus conventional\n", - "27446 Premixed (AN)" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "subdf = df[(~df.warning.isna()) & (df.warning.str.contains(\"not in the list of allowed values\", regex=False))]\n", - "\n", - "subdf.warning.str.split(expand=True)[[3, 6]].drop_duplicates()" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['2022_Sultanah Bahiyah A4D Tracker_DC_patient_raw.log',\n", - " '2021_Vietnam National Children_s Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Khon Kaen Hospital A4D Tracker_patient_raw.log',\n", - " \"2017_Vietnam National Children's Hospital A4D Tracker_patient_raw.log\",\n", - " '2022_Chulalongkorn Hospital A4D Tracker_patient_raw.log',\n", - " '2019_Sultanah Bahiyah Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2019_Mahosot Hospital A4D Tracker_patient_raw.log',\n", - " '2021_Penang General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2020_Lao Friends Hospital for Children A4D Tracker_DC_patient_raw.log',\n", - " '2022_Siriraj Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Putrajaya Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_Jayavarman VII Hospital A4D Tracker_patient_raw.log',\n", - " '2018_Vietnam National Children_s Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Surat Thani A4D Tracker_patient_raw.log',\n", - " '2022_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2021_Mahosot Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2018_Penang General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2021_Putrajaya Hospital A4D Tracker_DC_patient_raw.log',\n", - " \"2022_Mandalay Children's Hospital A4D Tracker_patient_raw.log\",\n", - " '2020_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2021_NPH A4D Tracker_patient_raw.log',\n", - " '2022_Sunprasitthiprasong Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Preah Kossamak Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Phattalung Hospital A4D Tracker_patient_raw.log',\n", - " \"2021_Likas Women & Children's Hospital A4D Tracker_DC_patient_raw.log\",\n", - " '2021_Lao Friends Hospital for Children A4D Tracker_DC_patient_raw.log',\n", - " \"2020_Vietnam National Children's Hospital A4D Tracker_patient_raw.log\",\n", - " '2022_Uni Med Centre A4D Tracker_patient_raw.log',\n", - " \"2022_Children's Hospital 2 A4D Tracker_patient_raw.log\",\n", - " '2019_Vietnam National Children_s Hospital A4D Tracker_patient_raw.log',\n", - " '2020_Sultanah Bahiyah Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_Pahol Polpayuhasena A4D Tracker_patient_raw.log',\n", - " '2017_Mahosot Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Udon Thani Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Mukdahan Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Lao Friends Hospital for Children A4D Tracker_DC_patient_raw.log',\n", - " \"2022_Likas Women & Children's Hospital A4D Tracker_DC_patient_raw.log\",\n", - " '2022_Mahosot Hosptial A4D Tracker_DC_patient_raw.log',\n", - " '2021_Sultanah Bahiyah Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_Kantha Bopha Hospital A4D Tracker_patient_raw.log',\n", - " '2022_Chiang Mai Maharaj Nakorn A4D Tracker_patient_raw.log',\n", - " '2018_Mahosot Hospital A4D Tracker_patient_raw.log',\n", - " '2022_NPH A4D Tracker_patient_raw.log',\n", - " '2018_Lao Friends Hospital for Children A4D Tracker_DC_patient_raw.log',\n", - " '2022_Hat Yai Hospital A4D Tracker_patient_raw.log',\n", - " '2021_Sarawak General Hospital A4D Tracker_DC_patient_raw.log',\n", - " '2022_CDA A4D Tracker_patient_raw.log']" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "subdf.file.unique().tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.15" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/scripts/python/.gitignore b/scripts/python/.gitignore deleted file mode 100644 index 15d50b6..0000000 --- a/scripts/python/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -build -dist -README.html diff --git a/scripts/python/README.md b/scripts/python/README.md deleted file mode 100644 index d0ab71d..0000000 --- a/scripts/python/README.md +++ /dev/null @@ -1,32 +0,0 @@ -## Preprocessing Tracker Files - -This script can be used to replace patient names with patient ids. - -The script can be called like this: - -```bash -python main.py -``` - -The script will do the following: - -1. Find all `.xlsx` files in the `` and its subdirectories. -2. For each `.xlsx` file, it will: - 1. Open the workbook. - 2. Look for the sheet `Patient List`. - 3. Get a list of all patient names and their ids. - 4. Go through all sheets in the workbook and replace the patient names with the patient ids. - 5. Save the file in `output/patient_data_without_names` directory next to the `src` directory. - 6. Create a log file under `output/logs/main_replace_patient_names.log`. - -The script works in a way that it does (hopefully) not change the structure or format of the workbook. It only replaces the patient names with the patient ids by manipulating the value of single cells without changing their attributes. - -## How to build - -Run - -```bash -pyinstaller main.py -F -n "a4d_replacer_tool" -``` - -to generate the executable. diff --git a/scripts/python/a4d_replacer_tool.spec b/scripts/python/a4d_replacer_tool.spec deleted file mode 100644 index 1bc6b89..0000000 --- a/scripts/python/a4d_replacer_tool.spec +++ /dev/null @@ -1,37 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- - - -a = Analysis( - ['main.py'], - pathex=[], - binaries=[], - datas=[], - hiddenimports=[], - hookspath=[], - hooksconfig={}, - runtime_hooks=[], - excludes=[], - noarchive=False, -) -pyz = PYZ(a.pure) - -exe = EXE( - pyz, - a.scripts, - a.binaries, - a.datas, - [], - name='a4d_replacer_tool', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=True, - upx_exclude=[], - runtime_tmpdir=None, - console=True, - disable_windowed_traceback=False, - argv_emulation=False, - target_arch=None, - codesign_identity=None, - entitlements_file=None, -) diff --git a/scripts/python/main.py b/scripts/python/main.py deleted file mode 100644 index 8b485a7..0000000 --- a/scripts/python/main.py +++ /dev/null @@ -1,135 +0,0 @@ -""" Small cli helper tool to replace patient names with patient ids in excel files. - -This script is used to replace patient names with patient ids in excel files. -The script will look for excel files in the source directory and replace the names -with the ids. The script will create a new directory called 'output' next to the source directory -and save the changed files there. -The source directory is specified by the user via prompt. -The output directory is specified by the user via option --output, and defaults to "output". -The script is logging to a file called 'main_replace_patient_names.log' -in a subdirectory called 'logs' inside the output directory. - -Example call: - $ python main.py --src /path/to/excel_files --output output -""" - -import logging -import shutil -from pathlib import Path -from zipfile import BadZipFile - -import click -import openpyxl -import pandas as pd - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("a4d-replacer-tool") - -PATIENT_DATA_RANGE = ("B1", "C200") - - -@click.command("a4d-replacer-tool") -@click.option( - "--src", - "-s", - prompt="Source directory", - default=lambda: Path("."), - help="Source directory", -) -@click.option("--output", "-o", default="output", help="Output directory") -@click.version_option("0.1.0", prog_name="replace patient names") -def replace_name_with_id(src: Path, output: str): - """Replaces patient name with patient id in all excel files found in src. - - Args: - src (Path): Source directory holding excel files as xlsx. - output (str): Output directory holding excel files with replaced names. - - Raises: - ValueError: If no excel files are found in src. - """ - excel_files = list(Path(src).rglob("*.xlsx")) - - if not excel_files: - raise ValueError(f"No excel files found in directory {src}.") - - output_dir = Path(src).parent / output / "patient_data_without_names" - log_dir = Path(src).parent / output / "logs" - - if not output_dir.exists(): - output_dir.mkdir(parents=True) - logger.info("Created output directory %s under %s.", output_dir, src) - - if not log_dir.exists(): - log_dir.mkdir(parents=True) - logger.info("Created log directory %s under %s.", log_dir, src) - - handler = logging.FileHandler(log_dir / "main_replace_patient_names.log") - handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")) - logger.addHandler(handler) - - logger.info("Start processing %s excel files.", len(excel_files)) - for i, excel_file in enumerate(excel_files): - logger.info( - "Start processing %s (%s/%s).", excel_file.name, i + 1, len(excel_files) - ) - - try: - wb = openpyxl.load_workbook(str(excel_file), data_only=True) - except BadZipFile: - logger.warning("Failed to open %s. Skipping.", excel_file.name) - continue - - sheets = wb.sheetnames - - if "Patient List" not in sheets: - logger.warning("Sheet 'Patient List' not found. Skipping.") - # shutil.copy(excel_file, output_dir / excel_file.name) - continue - - patient_data = pd.DataFrame( - [ - (b.value, c.value) - for b, c in wb["Patient List"][ - PATIENT_DATA_RANGE[0] : PATIENT_DATA_RANGE[1] - ] - if b.value and c.value - ], - columns=["id", "name"], - ) - patient_data = patient_data.iloc[1:] - patients_replaced = {name: False for name in patient_data.name} - - if all(patient_data.name == patient_data.id): - logger.info( - "Patient names are already replaced with ids. Copying original file to output dir." - ) - shutil.copy(excel_file, output_dir / excel_file.name) - continue - - for patient in patient_data.itertuples(index=False): - for sheet in sheets: - ws = wb[sheet] - - for col in ws.iter_cols(): - for cell in col: - if cell.value == patient.name: - if sheet != "Patient List": - patients_replaced[patient.name] = True - cell.value = patient.id - - if not all(patients_replaced.values()): - logger.warning( - "Not all patient names were replaced. Missing patients: %s", - ", ".join( - name for name, replaced in patients_replaced.items() if not replaced - ), - ) - - wb.save(output_dir / excel_file.name) - logger.info("Saved changed file to %s.", output_dir) - logger.info(f"Finished processing %s.", excel_file.name) - - -if __name__ == "__main__": - replace_name_with_id() diff --git a/scripts/python/poetry.lock b/scripts/python/poetry.lock deleted file mode 100644 index ce423b8..0000000 --- a/scripts/python/poetry.lock +++ /dev/null @@ -1,440 +0,0 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. - -[[package]] -name = "altgraph" -version = "0.17.4" -description = "Python graph (network) package" -optional = false -python-versions = "*" -files = [ - {file = "altgraph-0.17.4-py2.py3-none-any.whl", hash = "sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff"}, - {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"}, -] - -[[package]] -name = "click" -version = "8.1.7" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.7" -files = [ - {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, - {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "et-xmlfile" -version = "1.1.0" -description = "An implementation of lxml.xmlfile for the standard library" -optional = false -python-versions = ">=3.6" -files = [ - {file = "et_xmlfile-1.1.0-py3-none-any.whl", hash = "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"}, - {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"}, -] - -[[package]] -name = "macholib" -version = "1.16.3" -description = "Mach-O header analysis and editing" -optional = false -python-versions = "*" -files = [ - {file = "macholib-1.16.3-py2.py3-none-any.whl", hash = "sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c"}, - {file = "macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30"}, -] - -[package.dependencies] -altgraph = ">=0.17" - -[[package]] -name = "numpy" -version = "2.1.2" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.10" -files = [ - {file = "numpy-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:30d53720b726ec36a7f88dc873f0eec8447fbc93d93a8f079dfac2629598d6ee"}, - {file = "numpy-2.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d3ca0a72dd8846eb6f7dfe8f19088060fcb76931ed592d29128e0219652884"}, - {file = "numpy-2.1.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:fc44e3c68ff00fd991b59092a54350e6e4911152682b4782f68070985aa9e648"}, - {file = "numpy-2.1.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:7c1c60328bd964b53f8b835df69ae8198659e2b9302ff9ebb7de4e5a5994db3d"}, - {file = "numpy-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cdb606a7478f9ad91c6283e238544451e3a95f30fb5467fbf715964341a8a86"}, - {file = "numpy-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d666cb72687559689e9906197e3bec7b736764df6a2e58ee265e360663e9baf7"}, - {file = "numpy-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6eef7a2dbd0abfb0d9eaf78b73017dbfd0b54051102ff4e6a7b2980d5ac1a03"}, - {file = "numpy-2.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:12edb90831ff481f7ef5f6bc6431a9d74dc0e5ff401559a71e5e4611d4f2d466"}, - {file = "numpy-2.1.2-cp310-cp310-win32.whl", hash = "sha256:a65acfdb9c6ebb8368490dbafe83c03c7e277b37e6857f0caeadbbc56e12f4fb"}, - {file = "numpy-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:860ec6e63e2c5c2ee5e9121808145c7bf86c96cca9ad396c0bd3e0f2798ccbe2"}, - {file = "numpy-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b42a1a511c81cc78cbc4539675713bbcf9d9c3913386243ceff0e9429ca892fe"}, - {file = "numpy-2.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:faa88bc527d0f097abdc2c663cddf37c05a1c2f113716601555249805cf573f1"}, - {file = "numpy-2.1.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:c82af4b2ddd2ee72d1fc0c6695048d457e00b3582ccde72d8a1c991b808bb20f"}, - {file = "numpy-2.1.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:13602b3174432a35b16c4cfb5de9a12d229727c3dd47a6ce35111f2ebdf66ff4"}, - {file = "numpy-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ebec5fd716c5a5b3d8dfcc439be82a8407b7b24b230d0ad28a81b61c2f4659a"}, - {file = "numpy-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2b49c3c0804e8ecb05d59af8386ec2f74877f7ca8fd9c1e00be2672e4d399b1"}, - {file = "numpy-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2cbba4b30bf31ddbe97f1c7205ef976909a93a66bb1583e983adbd155ba72ac2"}, - {file = "numpy-2.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8e00ea6fc82e8a804433d3e9cedaa1051a1422cb6e443011590c14d2dea59146"}, - {file = "numpy-2.1.2-cp311-cp311-win32.whl", hash = "sha256:5006b13a06e0b38d561fab5ccc37581f23c9511879be7693bd33c7cd15ca227c"}, - {file = "numpy-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:f1eb068ead09f4994dec71c24b2844f1e4e4e013b9629f812f292f04bd1510d9"}, - {file = "numpy-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7bf0a4f9f15b32b5ba53147369e94296f5fffb783db5aacc1be15b4bf72f43b"}, - {file = "numpy-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b1d0fcae4f0949f215d4632be684a539859b295e2d0cb14f78ec231915d644db"}, - {file = "numpy-2.1.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f751ed0a2f250541e19dfca9f1eafa31a392c71c832b6bb9e113b10d050cb0f1"}, - {file = "numpy-2.1.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:bd33f82e95ba7ad632bc57837ee99dba3d7e006536200c4e9124089e1bf42426"}, - {file = "numpy-2.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b8cde4f11f0a975d1fd59373b32e2f5a562ade7cde4f85b7137f3de8fbb29a0"}, - {file = "numpy-2.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d95f286b8244b3649b477ac066c6906fbb2905f8ac19b170e2175d3d799f4df"}, - {file = "numpy-2.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ab4754d432e3ac42d33a269c8567413bdb541689b02d93788af4131018cbf366"}, - {file = "numpy-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e585c8ae871fd38ac50598f4763d73ec5497b0de9a0ab4ef5b69f01c6a046142"}, - {file = "numpy-2.1.2-cp312-cp312-win32.whl", hash = "sha256:9c6c754df29ce6a89ed23afb25550d1c2d5fdb9901d9c67a16e0b16eaf7e2550"}, - {file = "numpy-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:456e3b11cb79ac9946c822a56346ec80275eaf2950314b249b512896c0d2505e"}, - {file = "numpy-2.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a84498e0d0a1174f2b3ed769b67b656aa5460c92c9554039e11f20a05650f00d"}, - {file = "numpy-2.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4d6ec0d4222e8ffdab1744da2560f07856421b367928026fb540e1945f2eeeaf"}, - {file = "numpy-2.1.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:259ec80d54999cc34cd1eb8ded513cb053c3bf4829152a2e00de2371bd406f5e"}, - {file = "numpy-2.1.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:675c741d4739af2dc20cd6c6a5c4b7355c728167845e3c6b0e824e4e5d36a6c3"}, - {file = "numpy-2.1.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05b2d4e667895cc55e3ff2b56077e4c8a5604361fc21a042845ea3ad67465aa8"}, - {file = "numpy-2.1.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43cca367bf94a14aca50b89e9bc2061683116cfe864e56740e083392f533ce7a"}, - {file = "numpy-2.1.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:76322dcdb16fccf2ac56f99048af32259dcc488d9b7e25b51e5eca5147a3fb98"}, - {file = "numpy-2.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:32e16a03138cabe0cb28e1007ee82264296ac0983714094380b408097a418cfe"}, - {file = "numpy-2.1.2-cp313-cp313-win32.whl", hash = "sha256:242b39d00e4944431a3cd2db2f5377e15b5785920421993770cddb89992c3f3a"}, - {file = "numpy-2.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:f2ded8d9b6f68cc26f8425eda5d3877b47343e68ca23d0d0846f4d312ecaa445"}, - {file = "numpy-2.1.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2ffef621c14ebb0188a8633348504a35c13680d6da93ab5cb86f4e54b7e922b5"}, - {file = "numpy-2.1.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ad369ed238b1959dfbade9018a740fb9392c5ac4f9b5173f420bd4f37ba1f7a0"}, - {file = "numpy-2.1.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:d82075752f40c0ddf57e6e02673a17f6cb0f8eb3f587f63ca1eaab5594da5b17"}, - {file = "numpy-2.1.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:1600068c262af1ca9580a527d43dc9d959b0b1d8e56f8a05d830eea39b7c8af6"}, - {file = "numpy-2.1.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a26ae94658d3ba3781d5e103ac07a876b3e9b29db53f68ed7df432fd033358a8"}, - {file = "numpy-2.1.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13311c2db4c5f7609b462bc0f43d3c465424d25c626d95040f073e30f7570e35"}, - {file = "numpy-2.1.2-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:2abbf905a0b568706391ec6fa15161fad0fb5d8b68d73c461b3c1bab6064dd62"}, - {file = "numpy-2.1.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ef444c57d664d35cac4e18c298c47d7b504c66b17c2ea91312e979fcfbdfb08a"}, - {file = "numpy-2.1.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:bdd407c40483463898b84490770199d5714dcc9dd9b792f6c6caccc523c00952"}, - {file = "numpy-2.1.2-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:da65fb46d4cbb75cb417cddf6ba5e7582eb7bb0b47db4b99c9fe5787ce5d91f5"}, - {file = "numpy-2.1.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c193d0b0238638e6fc5f10f1b074a6993cb13b0b431f64079a509d63d3aa8b7"}, - {file = "numpy-2.1.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a7d80b2e904faa63068ead63107189164ca443b42dd1930299e0d1cb041cec2e"}, - {file = "numpy-2.1.2.tar.gz", hash = "sha256:13532a088217fa624c99b843eeb54640de23b3414b14aa66d023805eb731066c"}, -] - -[[package]] -name = "openpyxl" -version = "3.1.5" -description = "A Python library to read/write Excel 2010 xlsx/xlsm files" -optional = false -python-versions = ">=3.8" -files = [ - {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, - {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, -] - -[package.dependencies] -et-xmlfile = "*" - -[[package]] -name = "packaging" -version = "24.1" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, - {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, -] - -[[package]] -name = "pandas" -version = "2.2.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -files = [ - {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"}, - {file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"}, - {file = "pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed"}, - {file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"}, - {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42"}, - {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"}, - {file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"}, - {file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"}, - {file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"}, - {file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"}, - {file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"}, - {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"}, - {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"}, - {file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"}, - {file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"}, - {file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"}, - {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"}, - {file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"}, - {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"}, - {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"}, - {file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"}, - {file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"}, - {file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"}, - {file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"}, - {file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"}, - {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"}, - {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"}, - {file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"}, - {file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"}, - {file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"}, - {file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"}, - {file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"}, - {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"}, - {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"}, - {file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"}, - {file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"}, - {file = "pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c"}, - {file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"}, - {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea"}, - {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"}, - {file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"}, - {file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"}, -] - -[package.dependencies] -numpy = [ - {version = ">=1.22.4", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, -] -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - -[[package]] -name = "pefile" -version = "2023.2.7" -description = "Python PE parsing module" -optional = false -python-versions = ">=3.6.0" -files = [ - {file = "pefile-2023.2.7-py3-none-any.whl", hash = "sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6"}, - {file = "pefile-2023.2.7.tar.gz", hash = "sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc"}, -] - -[[package]] -name = "pyinstaller" -version = "6.11.0" -description = "PyInstaller bundles a Python application and all its dependencies into a single package." -optional = false -python-versions = "<3.14,>=3.8" -files = [ - {file = "pyinstaller-6.11.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:6fd68a3c1207635c49326c54881b89d5c3bd9ba061bbc9daa58c0902db1be39e"}, - {file = "pyinstaller-6.11.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:eddd53f231e51adc65088eac4f40057ca803a990239828d4a9229407fb866239"}, - {file = "pyinstaller-6.11.0-py3-none-manylinux2014_i686.whl", hash = "sha256:e6d229009e815542833fe00332b589aa6984a06f794dc16f2ce1acab1c567590"}, - {file = "pyinstaller-6.11.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:7d2cd2ebdcd6860f8a4abe2977264a7b6d260a7147047008971c7cfc66a656a4"}, - {file = "pyinstaller-6.11.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:d9ec6d4398b4eebc1d4c00437716264ba8406bc2746f594e253070a82378a584"}, - {file = "pyinstaller-6.11.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:04f71828aa9531ab18c9656985c1f09b83d10332c73a1f4a113a48b491906955"}, - {file = "pyinstaller-6.11.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:a843d470768d68b05684ccf4860c45b2eb13727f41667c0b2cd8f57ae231bd18"}, - {file = "pyinstaller-6.11.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:963dedc1f37144a4385f58f7f65f1c69c004a67faae522a2085b5ddb230c908b"}, - {file = "pyinstaller-6.11.0-py3-none-win32.whl", hash = "sha256:c71024c8a19c7b221b9152b2baff4c3ba849cada68dcdd34382ba09f0107451f"}, - {file = "pyinstaller-6.11.0-py3-none-win_amd64.whl", hash = "sha256:0e229610c22b96d741d905706f9496af472c1a9216a118988f393c98ecc3f51f"}, - {file = "pyinstaller-6.11.0-py3-none-win_arm64.whl", hash = "sha256:a5f716bb507517912fda39d109dead91fc0dd2e7b2859562522b63c61aa21676"}, - {file = "pyinstaller-6.11.0.tar.gz", hash = "sha256:cb4d433a3db30d9d17cf5f2cf7bb4df80a788d493c1d67dd822dc5791d9864af"}, -] - -[package.dependencies] -altgraph = "*" -macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} -packaging = ">=22.0" -pefile = {version = ">=2022.5.30,<2024.8.26 || >2024.8.26", markers = "sys_platform == \"win32\""} -pyinstaller-hooks-contrib = ">=2024.8" -pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} -setuptools = ">=42.0.0" - -[package.extras] -completion = ["argcomplete"] -hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] - -[[package]] -name = "pyinstaller-hooks-contrib" -version = "2024.9" -description = "Community maintained hooks for PyInstaller" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pyinstaller_hooks_contrib-2024.9-py3-none-any.whl", hash = "sha256:1ddf9ba21d586afa84e505bb5c65fca10b22500bf3fdb89ee2965b99da53b891"}, - {file = "pyinstaller_hooks_contrib-2024.9.tar.gz", hash = "sha256:4793869f370d1dc4806c101efd2890e3c3e703467d8d27bb5a3db005ebfb008d"}, -] - -[package.dependencies] -packaging = ">=22.0" -setuptools = ">=42.0.0" - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2024.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -files = [ - {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, - {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, -] - -[[package]] -name = "pywin32-ctypes" -version = "0.2.3" -description = "A (partial) reimplementation of pywin32 using ctypes/cffi" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"}, - {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, -] - -[[package]] -name = "pyyaml" -version = "6.0.2" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, - {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, - {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, - {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, - {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, - {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, - {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, - {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, - {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, - {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, - {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, - {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, - {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, - {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, - {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, - {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, - {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, - {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, - {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, - {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, - {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, - {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, - {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, - {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, - {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, - {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, - {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, - {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, - {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, - {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, - {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, - {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, - {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, - {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, - {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, - {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, - {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, - {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, - {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, - {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, - {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, - {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, - {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, - {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, - {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, - {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, - {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, - {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, - {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, - {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, - {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, - {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, - {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, -] - -[[package]] -name = "setuptools" -version = "75.2.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "setuptools-75.2.0-py3-none-any.whl", hash = "sha256:a7fcb66f68b4d9e8e66b42f9876150a3371558f98fa32222ffaa5bced76406f8"}, - {file = "setuptools-75.2.0.tar.gz", hash = "sha256:753bb6ebf1f465a1912e19ed1d41f403a79173a9acf66a42e7e6aec45c3c16ec"}, -] - -[package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.11.*)", "pytest-mypy"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] - -[[package]] -name = "tzdata" -version = "2024.2" -description = "Provider of IANA time zone data" -optional = false -python-versions = ">=2" -files = [ - {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, - {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, -] - -[metadata] -lock-version = "2.0" -python-versions = ">=3.10,<3.13" -content-hash = "8a569b0916bae54ae7240cdaac2a5dec92a07dc930bcea00334ad87f0305cd5a" diff --git a/scripts/python/pyproject.toml b/scripts/python/pyproject.toml deleted file mode 100644 index a21275c..0000000 --- a/scripts/python/pyproject.toml +++ /dev/null @@ -1,21 +0,0 @@ -[tool.poetry] -name = "a4d-replacer-tool" -version = "0.1.0" -description = "" -authors = ["Michael Aydinbas "] -readme = "README.md" -package-mode = false - -[tool.poetry.dependencies] -python = ">=3.10,<3.13" -pandas = "^2.2.1" -openpyxl = "^3.1.5" -click = "^8.1.7" -pyyaml = "^6.0.2" - -[tool.poetry.group.dev.dependencies] -pyinstaller = "^6.11.0" - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" diff --git a/scripts/python/sort_yaml.py b/scripts/python/sort_yaml.py deleted file mode 100644 index ca4a1ac..0000000 --- a/scripts/python/sort_yaml.py +++ /dev/null @@ -1,29 +0,0 @@ -from argparse import ArgumentParser, FileType -from pathlib import Path - -import yaml - - -def parse_args(): - parser = ArgumentParser(description='Sort YAML file') - parser.add_argument('file', type=str, help='YAML file to sort') - return parser.parse_args() - - -def sort_yaml(): - args = parse_args() - yaml_file = Path(args.file).resolve() - if not yaml_file.is_file(): - print(f'File not found: {yaml_file}') - return - - with open(yaml_file, 'r') as f: - data = yaml.safe_load(f) - - with open(yaml_file, 'w') as f: - yaml.dump(data, f, sort_keys=True) - print("YAML file sorted") - -if __name__ == '__main__': - sort_yaml() - diff --git a/scripts/verify_production_run.py b/scripts/verify_production_run.py new file mode 100644 index 0000000..c3ba4a2 --- /dev/null +++ b/scripts/verify_production_run.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Sanity-check a production Cloud Run Job execution against a `just backup-bq` snapshot. + +Compares row counts, distinct clinic counts and schema for each WRITE_TRUNCATE'd +pipeline table against its dated snapshot, flagging gross regressions. Not a +cell-by-cell R-vs-Python comparison -- see scripts/compare_outputs.py for that. + +Usage: + uv run python scripts/verify_production_run.py --backup-suffix 20260809 +""" + +import typer +from rich.console import Console +from rich.table import Table + +from a4d.gcp.bigquery import get_bigquery_client +from a4d.gcp.verify import VERIFIED_TABLES, diff_table_stats, fetch_table_stats + +console = Console() +app = typer.Typer() + + +@app.command() +def verify( + backup_suffix: str = typer.Option( + ..., "--backup-suffix", help="Date suffix `just backup-bq` used, e.g. 20260809" + ), +) -> None: + client = get_bigquery_client() + + before = { + name: fetch_table_stats(client, f"{name}_{backup_suffix}") for name in VERIFIED_TABLES + } + after = {name: fetch_table_stats(client, name) for name in VERIFIED_TABLES} + anomalies = diff_table_stats(before, after) + + report = Table(title="Production run verification") + report.add_column("Table") + report.add_column("Rows before", justify="right") + report.add_column("Rows after", justify="right") + report.add_column("Clinics before", justify="right") + report.add_column("Clinics after", justify="right") + for name in VERIFIED_TABLES: + report.add_row( + name, + str(before[name].row_count), + str(after[name].row_count), + str(before[name].distinct_clinics), + str(after[name].distinct_clinics), + ) + console.print(report) + + if anomalies: + console.print("[bold red]Anomalies found:[/bold red]") + for anomaly in anomalies: + console.print(f" - {anomaly.table}: {anomaly.reason}") + raise typer.Exit(code=1) + + console.print("[bold green]No anomalies detected.[/bold green]") + + +if __name__ == "__main__": + app() diff --git a/src/a4d/__init__.py b/src/a4d/__init__.py new file mode 100644 index 0000000..733bf4a --- /dev/null +++ b/src/a4d/__init__.py @@ -0,0 +1,15 @@ +"""A4D Medical Tracker Data Processing Pipeline.""" + +from a4d.config import settings +from a4d.errors import DataError, ErrorCollector +from a4d.logging import file_logger, setup_logging + +__version__ = "0.1.0" + +__all__ = [ + "settings", + "setup_logging", + "file_logger", + "ErrorCollector", + "DataError", +] diff --git a/src/a4d/__main__.py b/src/a4d/__main__.py new file mode 100644 index 0000000..e82ca3c --- /dev/null +++ b/src/a4d/__main__.py @@ -0,0 +1,6 @@ +"""Make package executable with 'python -m a4d'.""" + +from a4d.cli import main + +if __name__ == "__main__": + main() diff --git a/src/a4d/clean/__init__.py b/src/a4d/clean/__init__.py new file mode 100644 index 0000000..e821633 --- /dev/null +++ b/src/a4d/clean/__init__.py @@ -0,0 +1,15 @@ +"""Data cleaning and transformation modules.""" + +from a4d.clean.converters import ( + correct_decimal_sign, + cut_numeric_value, + safe_convert_column, + safe_convert_multiple_columns, +) + +__all__ = [ + "safe_convert_column", + "safe_convert_multiple_columns", + "correct_decimal_sign", + "cut_numeric_value", +] diff --git a/src/a4d/clean/buddhist_era.py b/src/a4d/clean/buddhist_era.py new file mode 100644 index 0000000..2672526 --- /dev/null +++ b/src/a4d/clean/buddhist_era.py @@ -0,0 +1,37 @@ +"""Buddhist-era date handling, shared by the patient and product cleaning arms. + +Thai clinics write dates in the Buddhist calendar (BE = CE + 543), and their +workbooks are often kept in a Thai-locale Excel, so the shift arrives as a +BE-shifted serial rather than as anything a clinician typed wrongly. The +pipeline converts such a date to Gregorian in the cleaned stage only (ticket +61); the raw stage keeps what the workbook actually says. +""" + +import polars as pl + +# Buddhist Era is Common Era + 543. +BUDDHIST_ERA_OFFSET: int = 543 + +# Buddhist Era / Common Era disambiguation point. A parsed year at or beyond +# this is implausible as a Gregorian date in a 2017+ tracker corpus, so it is +# either a Buddhist-era year or a corrupt Excel serial. Keeping the threshold +# well above any real tracker year is what makes the conversion unable to fire +# on a Gregorian date. +BUDDHIST_ERA_THRESHOLD: int = 2400 + + +def gregorian_from_buddhist(col: str) -> pl.Expr: + """Shift ``col`` back by 543 years, yielding null where that date does not exist. + + Goes via a string rather than ``pl.date``/``dt.replace`` because both of + those raise on invalid components: 543 is not a multiple of 4, so a 29 + February in a Buddhist leap year can land on a non-leap Gregorian one + (2568-02-29 -> 2025-02-29). A null there leaves the cell unconverted, which + routes it to the ordinary implausible-date handling instead of aborting the + tracker. + """ + return pl.format( + "{}-{}", + pl.col(col).dt.year() - BUDDHIST_ERA_OFFSET, + pl.col(col).dt.strftime("%m-%d"), + ).str.to_date(format="%Y-%m-%d", strict=False) diff --git a/src/a4d/clean/converters.py b/src/a4d/clean/converters.py new file mode 100644 index 0000000..a2445f6 --- /dev/null +++ b/src/a4d/clean/converters.py @@ -0,0 +1,575 @@ +"""Type conversion utilities with error tracking. + +This module provides vectorized type conversion functions that track failures +in an ErrorCollector, without ever raising: a bad cell is a data-quality +finding to report, not a reason to abandon a tracker. + +The pattern is: +1. Try vectorized conversion (fast, handles 95%+ of data) +2. Detect failures (nulls after conversion but not before) +3. Log only failed rows to ErrorCollector +4. Replace failures with error value +""" + +from datetime import date + +import polars as pl +from loguru import logger + +from a4d.clean.date_parser import ( + MISSING_VALUE_MARKERS, + TextDateRecovery, + parse_date_detailed, + rescue_date_typos, +) +from a4d.config import settings +from a4d.errors import ErrorCode, ErrorCollector +from a4d.extract.common import EXCEL_ERROR_STRINGS + + +def normalize_excel_formula_errors( + df: pl.DataFrame, + error_collector: ErrorCollector, + file_name_col: str = "file_name", + patient_id_col: str = "patient_id", +) -> pl.DataFrame: + """Null out Excel formula-error strings, logging each one (ticket 27). + + A tracker's own spreadsheet formula writes a literal error string + (``#NUM!``, ``#DIV/0!``, ...) into a cell whenever an input it depends on + is missing -- e.g. age-at-diagnosis when no diagnosis date was recorded, + or BMI when height is blank. Extraction deliberately preserves that text + so the raw layer stays a faithful capture of the source file; this is + where it becomes ``null``. + + ``null``, not ``settings.error_val_numeric`` (999999): the sentinel means + "a value was recorded but is invalid" (a garbled reading that failed to + parse). Here no value could be computed at all, because a required input + was never entered -- semantically absent, not invalid. Running before + type conversion keeps these out of ``safe_convert_column``'s + parse-failure branch, which would otherwise assign that wrong sentinel. + + Each nulled cell is logged under ``source_formula_error`` so the + distinction between "source formula could not compute this" and "field + was simply blank" survives into the error log, even though both end up + ``null`` in the data. + """ + data_cols = [col for col in df.columns if df.schema[col] == pl.String] + if not data_cols: + return df + + error_mask = pl.any_horizontal([pl.col(col).is_in(EXCEL_ERROR_STRINGS) for col in data_cols]) + if not df.select(error_mask.any()).item(): + return df + + for col in data_cols: + offenders = df.filter(pl.col(col).is_in(EXCEL_ERROR_STRINGS)) + if offenders.is_empty(): + continue + for row in offenders.iter_rows(named=True): + error_collector.add_error( + file_name=row.get(file_name_col) or "unknown", + patient_id=row.get(patient_id_col) or "unknown", + column=col, + original_value=row[col], + error_message=( + f"Source tracker formula error '{row[col]}' in {col}: " + "a required input was not recorded, so no value could be computed" + ), + error_code="source_formula_error", + function_name="normalize_excel_formula_errors", + ) + + return df.with_columns( + [ + pl.when(pl.col(col).is_in(EXCEL_ERROR_STRINGS)) + .then(None) + .otherwise(pl.col(col)) + .alias(col) + for col in data_cols + ] + ) + + +def safe_convert_column( + df: pl.DataFrame, + column: str, + target_type: type[pl.DataType] | pl.DataType, + error_collector: ErrorCollector, + error_value: float | str | None = None, + file_name_col: str = "file_name", + patient_id_col: str = "patient_id", +) -> pl.DataFrame: + """Convert column to target type with vectorized error tracking. + + This function attempts vectorized type conversion and tracks any failures + in the ErrorCollector. Vectorized: the slow per-row path runs only over + the cells that actually failed. + + Args: + df: Input DataFrame + column: Column name to convert + target_type: Target Polars data type (pl.Int32, pl.Float64, etc.) + error_collector: ErrorCollector instance to track failures + error_value: Value to use for failed conversions (default from settings) + file_name_col: Column containing file name for error tracking + patient_id_col: Column containing patient ID for error tracking + + Returns: + DataFrame with converted column (failures replaced with error_value) + + Example: + >>> collector = ErrorCollector() + >>> df = safe_convert_column( + ... df=df, + ... column="age", + ... target_type=pl.Int32, + ... error_collector=collector, + ... ) + >>> # Failures are logged in collector, replaced with ERROR_VAL_NUMERIC + """ + # Determine error value based on target type if not provided + if error_value is None: + if target_type in (pl.Int32, pl.Int64, pl.Float32, pl.Float64): + error_value = settings.error_val_numeric + elif target_type in (pl.Utf8, pl.Categorical, pl.String): + error_value = settings.error_val_character + elif target_type == pl.Date: + error_value = settings.error_val_date + elif target_type == pl.Boolean: + error_value = False # Default for boolean conversion failures + else: + raise ValueError(f"Cannot determine error value for type {target_type}") + + # Skip if column doesn't exist + if column not in df.columns: + return df + + # Normalize empty/whitespace/missing-value strings to null BEFORE conversion + # This ensures missing data stays null rather than becoming error values + # A cell reading 'Nil' or '-' recorded an absence, not an unusable value, + # so it must not pick up the 999999 'recorded but invalid' sentinel. + if df[column].dtype in (pl.Utf8, pl.String): + # Shared with the date path (MISSING_VALUE_MARKERS, clean/date_parser.py) + # so the two cannot drift apart again -- they had, and a date cell + # holding "-" reached the error sentinel where the numeric one nulled. + df = df.with_columns( + pl.when( + pl.col(column) + .str.strip_chars() + .str.to_lowercase() + .is_in(list(MISSING_VALUE_MARKERS)) + | (pl.col(column).str.strip_chars().str.len_chars() == 0) + ) + .then(None) + .otherwise(pl.col(column)) + .alias(column) + ) + + # Store original values for error reporting + df = df.with_columns(pl.col(column).alias(f"_orig_{column}")) + + # Try vectorized conversion (strict=False allows nulls for failures) + df = df.with_columns(pl.col(column).cast(target_type, strict=False).alias(f"_conv_{column}")) + + # Detect failures: became null but wasn't null before + failed_mask = pl.col(f"_conv_{column}").is_null() & pl.col(f"_orig_{column}").is_not_null() + + # Extract failed rows for error logging + failed_rows = df.filter(failed_mask) + + # Log each failure + if len(failed_rows) > 0: + for row in failed_rows.iter_rows(named=True): + error_collector.add_error( + file_name=row.get(file_name_col) or "unknown", + patient_id=row.get(patient_id_col) or "unknown", + column=column, + original_value=row[f"_orig_{column}"], + error_message=f"Could not convert '{row[f'_orig_{column}']}' to {target_type}", + error_code="type_conversion", + function_name="safe_convert_column", + ) + + # Replace failures with error value (cast to target type) + df = df.with_columns( + pl.when(failed_mask) + .then(pl.lit(error_value).cast(target_type)) + .otherwise(pl.col(f"_conv_{column}")) + .alias(column) + ) + + # Clean up temporary columns + df = df.drop([f"_orig_{column}", f"_conv_{column}"]) + + return df + + +def _apply_typo_rescue( + df: pl.DataFrame, + column: str, + error_collector: ErrorCollector, + file_name_col: str, + patient_id_col: str, +) -> pl.DataFrame: + """Rewrite known month-name typos in-place before parsing. + + Builds a rescue_map from unique strings, logs each affected row to + error_collector + loguru with code "typo_rescued", then applies the + substitutions column-wide. No-op if no typos match. + """ + rescue_map: dict[str, str] = {} + for s in df[column].drop_nulls().unique().to_list(): + rescued, was_rescued = rescue_date_typos(s) + if was_rescued: + rescue_map[s] = rescued + + if not rescue_map: + return df + + select_cols = [c for c in (file_name_col, patient_id_col) if c in df.columns] + for original, rescued_val in rescue_map.items(): + if select_cols: + affected = df.filter(pl.col(column) == original).select(select_cols) + for row in affected.iter_rows(named=True): + file_name = row.get(file_name_col) or "unknown" + patient_id = row.get(patient_id_col) or "unknown" + logger.bind(error_code="typo_rescued").warning( + f"date typo rescued in {column}: {original!r} -> {rescued_val!r} " + f"(file={file_name!r}, {patient_id_col}={patient_id!r})" + ) + error_collector.add_error( + file_name=str(file_name), + patient_id=str(patient_id), + column=column, + original_value=original, + error_message=f"date typo rescued: '{original}' -> '{rescued_val}'", + error_code="typo_rescued", + function_name="parse_date_column", + ) + + repl_expr = pl.col(column) + for original, rescued_val in rescue_map.items(): + repl_expr = ( + pl.when(pl.col(column) == original).then(pl.lit(rescued_val)).otherwise(repl_expr) + ) + return df.with_columns(repl_expr.alias(column)) + + +def _log_text_recoveries( + df: pl.DataFrame, + column: str, + detailed: dict[tuple[str, int | None], tuple[date | None, TextDateRecovery | None]], + error_collector: ErrorCollector, + file_name_col: str, + patient_id_col: str, +) -> None: + """Record what the free-text recogniser decided, one code per decision. + + Three codes rather than one (ticket 39): "we read what was written", "we + chose among several dates the cell named", and "we supplied a year the cell + did not state" carry different amounts of confidence, and only the middle + one is a source-tracker defect worth reporting back to a clinic. + """ + reported = { + pair: recovery + for pair, (_, recovery) in detailed.items() + if recovery is not None and recovery.value is not None + } + if not reported: + return + + select_cols = [c for c in (file_name_col, patient_id_col) if c in df.columns] + for (text, tracker_year), recovery in reported.items(): + messages: list[tuple[str, ErrorCode]] = [ + ( + f"date read out of free text: {text!r} -> {recovery.value}", + "date_recovered_from_text", + ) + ] + if recovery.tokens_found > 1: + messages.append( + ( + f"cell names {recovery.tokens_found} dates, first used: " + f"{text!r} -> {recovery.value}", + "date_multiple_in_cell", + ) + ) + if recovery.year_inferred: + messages.append( + ( + f"no year in cell, tracker year {tracker_year} used: " + f"{text!r} -> {recovery.value}", + "date_year_inferred", + ) + ) + + affected = df.filter(pl.col(column).cast(pl.Utf8) == text) + if select_cols: + affected = affected.select(select_cols) + for row in affected.iter_rows(named=True): + file_name = row.get(file_name_col) or "unknown" + patient_id = row.get(patient_id_col) or "unknown" + for message, code in messages: + logger.bind(error_code=code).warning( + f"{message} in {column} (file={file_name!r}, {patient_id_col}={patient_id!r})" + ) + error_collector.add_error( + file_name=str(file_name), + patient_id=str(patient_id), + column=column, + original_value=text, + error_message=message, + error_code=code, + function_name="parse_date_column", + ) + + +def parse_date_column( + df: pl.DataFrame, + column: str, + error_collector: ErrorCollector, + file_name_col: str = "file_name", + patient_id_col: str = "patient_id", +) -> pl.DataFrame: + """Parse date column using flexible date parser. + + Uses parse_date_flexible() to handle various date formats including: + - Standard formats (ISO, DD/MM/YYYY, etc.) + - Abbreviated month-year (Mar-18, Jan-20) + - Excel serial numbers + - 4-letter month names + + Args: + df: Input DataFrame + column: Column name to parse + error_collector: ErrorCollector instance to track failures + file_name_col: Column containing file name for error tracking + patient_id_col: Column containing patient ID for error tracking + + Returns: + DataFrame with parsed date column + + Example: + >>> df = parse_date_column( + ... df=df, + ... column="hba1c_updated_date", + ... error_collector=collector, + ... ) + """ + if column not in df.columns: + return df + + # Substitute known month-name typos (e.g. "MACH" -> "MAR") before parsing, + # logging each affected row so the source tracker remains visible to + # data-quality triage. Skipped silently when no typos match. + df = _apply_typo_rescue(df, column, error_collector, file_name_col, patient_id_col) + + # Store original values for error reporting + df = df.with_columns(pl.col(column).alias(f"_orig_{column}")) + + # Parse each distinct string once, then map back. Tracker data has heavy + # duplication in date columns (e.g. "1/1/2024" repeating per row), so + # dedup-then-map is much faster than a per-row Python call. + # All-null columns short-circuit: map_elements can't infer Date dtype on + # an empty-after-drop_nulls Series. + # A note with no year in it is resolved against the tracker's own year + # (ticket 39), so the same string can mean different dates in two files -- + # 2020 VNCH and 2021 VNCH both carry "26 Jun (ceton urine high)" for the + # same patient. The dedup key therefore has to be the pair, not the string. + col_str = df[column].cast(pl.Utf8) + if "tracker_year" in df.columns: + year_str = df["tracker_year"].cast(pl.Int32, strict=False) + else: + year_str = pl.Series("tracker_year", [None] * df.height, dtype=pl.Int32) + pairs = [(s, y) for s, y in zip(col_str, year_str, strict=True) if s is not None] + unique_strs = list(dict.fromkeys(pairs)) + if unique_strs: + detailed = { + pair: parse_date_detailed(pair[0], settings.error_val_date, pair[1]) + for pair in unique_strs + } + lookup = {pair: value for pair, (value, _) in detailed.items()} + _log_text_recoveries(df, column, detailed, error_collector, file_name_col, patient_id_col) + parsed_series = pl.Series( + f"_parsed_{column}", + [ + lookup.get((s, y)) if s is not None else None + for s, y in zip(col_str, year_str, strict=True) + ], + dtype=pl.Date, + ) + else: + parsed_series = pl.Series(f"_parsed_{column}", [None] * df.height, dtype=pl.Date) + df = df.with_columns(parsed_series) + + # Detect failures: parsed to error date + error_date = pl.lit(settings.error_val_date).str.to_date() + failed_mask = ( + pl.col(f"_parsed_{column}").is_not_null() + & (pl.col(f"_parsed_{column}") == error_date) + & pl.col(f"_orig_{column}").is_not_null() + ) + + # Extract failed rows for error logging + failed_rows = df.filter(failed_mask) + + # Log each failure + if len(failed_rows) > 0: + for row in failed_rows.iter_rows(named=True): + error_collector.add_error( + file_name=row.get(file_name_col) or "unknown", + patient_id=row.get(patient_id_col) or "unknown", + column=column, + original_value=row[f"_orig_{column}"], + error_message=f"Could not parse date '{row[f'_orig_{column}']}'", + error_code="type_conversion", + function_name="parse_date_column", + ) + + # Use parsed values + df = df.with_columns(pl.col(f"_parsed_{column}").alias(column)) + + # Clean up temporary columns + df = df.drop([f"_orig_{column}", f"_parsed_{column}"]) + + return df + + +def correct_decimal_sign(df: pl.DataFrame, column: str) -> pl.DataFrame: + """Replace comma decimal separator with dot. + + Some trackers use European decimal format (1,5 instead of 1.5). + + Args: + df: Input DataFrame + column: Column name to correct + + Returns: + DataFrame with corrected decimal signs + + Example: + >>> df = correct_decimal_sign(df, "weight") + """ + if column not in df.columns: + return df + + df = df.with_columns(pl.col(column).cast(pl.Utf8).str.replace(",", ".").alias(column)) + + return df + + +def cut_numeric_value( + df: pl.DataFrame, + column: str, + min_val: float, + max_val: float, + error_collector: ErrorCollector, + file_name_col: str = "file_name", + patient_id_col: str = "patient_id", +) -> pl.DataFrame: + """Replace out-of-range numeric values with error value. + + Args: + df: Input DataFrame + column: Column name to check + min_val: Minimum allowed value + max_val: Maximum allowed value + error_collector: ErrorCollector instance to track violations + file_name_col: Column containing file name for error tracking + patient_id_col: Column containing patient ID for error tracking + + Returns: + DataFrame with out-of-range values replaced + + Example: + >>> df = cut_numeric_value( + ... df=df, + ... column="age", + ... min_val=0, + ... max_val=25, + ... error_collector=collector, + ... ) + """ + if column not in df.columns: + return df + + # Find values outside allowed range (excluding nulls and existing error values) + invalid_mask = ( + pl.col(column).is_not_null() + & (pl.col(column) != settings.error_val_numeric) + & ((pl.col(column) < min_val) | (pl.col(column) > max_val)) + ) + + # Extract invalid rows for error logging + invalid_rows = df.filter(invalid_mask) + + # Log each invalid value + if len(invalid_rows) > 0: + for row in invalid_rows.iter_rows(named=True): + error_collector.add_error( + file_name=row.get(file_name_col) or "unknown", + patient_id=row.get(patient_id_col) or "unknown", + column=column, + original_value=row[column], + error_message=f"Value {row[column]} outside allowed range [{min_val}, {max_val}]", + error_code="invalid_value", + function_name="cut_numeric_value", + ) + + # Replace invalid values with error value + df = df.with_columns( + pl.when(invalid_mask) + .then(pl.lit(settings.error_val_numeric)) + .otherwise(pl.col(column)) + .alias(column) + ) + + return df + + +def safe_convert_multiple_columns( + df: pl.DataFrame, + columns: list[str], + target_type: type[pl.DataType] | pl.DataType, + error_collector: ErrorCollector, + error_value: float | str | None = None, + file_name_col: str = "file_name", + patient_id_col: str = "patient_id", +) -> pl.DataFrame: + """Convert multiple columns to the same target type. + + Convenience function for batch conversion of columns. + + Args: + df: Input DataFrame + columns: List of column names to convert + target_type: Target Polars data type + error_collector: ErrorCollector instance + error_value: Value to use for failed conversions + file_name_col: Column containing file name for error tracking + patient_id_col: Column containing patient ID for error tracking + + Returns: + DataFrame with all specified columns converted + + Example: + >>> df = safe_convert_multiple_columns( + ... df=df, + ... columns=["age", "height", "weight"], + ... target_type=pl.Float64, + ... error_collector=collector, + ... ) + """ + for column in columns: + df = safe_convert_column( + df=df, + column=column, + target_type=target_type, + error_collector=error_collector, + error_value=error_value, + file_name_col=file_name_col, + patient_id_col=patient_id_col, + ) + + return df diff --git a/src/a4d/clean/date_parser.py b/src/a4d/clean/date_parser.py new file mode 100644 index 0000000..a7c4411 --- /dev/null +++ b/src/a4d/clean/date_parser.py @@ -0,0 +1,558 @@ +"""Flexible date parsing for A4D tracker data. + +Handles the date formats found across nine years of tracker templates, +including: +- Standard formats: "28/8/2017", "01-03-2018" +- Abbreviated month-year: "Mar-18", "Jan-20" +- Full month-year: "March-2018", "January-20" +- Excel serial numbers: "45341.0" (days since 1899-12-30) +- Year only: "2018", "18" +""" + +import re +from dataclasses import dataclass +from datetime import date, datetime, timedelta + +from dateutil import parser as date_parser +from loguru import logger + +# Excel epoch: dates stored as days since this date +EXCEL_EPOCH = date(1899, 12, 30) + +# Known data-entry typos in month names. Patterns are applied case-insensitively +# with word boundaries so unrelated text containing these substrings (e.g. +# "CON0CT") is not rewritten. Replacements use uppercase since downstream +# parsers are case-insensitive. +TYPO_REPLACEMENTS: list[tuple[str, str]] = [ + (r"(?i)\bMACH\b", "MAR"), + (r"(?i)\b0CT\b", "OCT"), + (r"(?i)\b0CTOBER\b", "OCTOBER"), + (r"(?i)\bN0V\b", "NOV"), + (r"(?i)\bN0VEMBER\b", "NOVEMBER"), + # A transposed and a dropped letter, each observed on real trackers + # (ticket 56): "9-Dce-20" at Mahosot DC, "6-ug-2025" at NPH. + (r"(?i)\bDCE\b", "DEC"), + (r"(?i)\bUG\b", "AUG"), + # Bahasa Malaysia month abbreviations, not typos: the Malaysian clinics + # (Likas, Putrajaya, Sultanah Malihah, Sarawak) write the month in their + # own language in a column every other clinic writes in English. Mac, Mei + # and Okt are observed on the current tracker set; Ogos and Dis are the + # rest of the set that differs from English, added so the next one to + # appear is read rather than sentinelled. The other seven Malay + # abbreviations are spelled as in English and need no entry. + (r"(?i)\bMAC\b", "MAR"), + (r"(?i)\bMEI\b", "MAY"), + (r"(?i)\bOGOS\b", "AUG"), + (r"(?i)\bOKT\b", "OCT"), + (r"(?i)\bDIS\b", "DEC"), +] + + +# Thai month abbreviations, with or without the abbreviating full stops, as the +# Thai clinics write them (ticket 56: Nakornping's 2025 and 2026 trackers). The +# guards are lookarounds on the Thai block rather than \b, because a full stop +# between two Thai letters kills the word boundary a trailing \b would need. +# Longest-first so "มี.ค." (March) is not read as "ม.ค." (January). +_THAI_MONTH_REPLACEMENTS: list[tuple[str, str]] = [ + (r"(? bool: + """True when the cell records an absence rather than an unusable date.""" + normalized = value.strip().lower() + if normalized in MISSING_VALUE_MARKERS or normalized in DATE_ABSENCE_MARKERS: + return True + return any(fragment in normalized for fragment in DATE_PLACEHOLDER_FRAGMENTS) + + +def rescue_date_typos(s: str) -> tuple[str, bool]: + """Substitute known month-name typos. Returns (possibly-rewritten, was_rescued).""" + rescued = False + for pattern, replacement in [*TYPO_REPLACEMENTS, *_THAI_MONTH_REPLACEMENTS]: + new_s, n = re.subn(pattern, replacement, s) + if n > 0: + s, rescued = new_s, True + return s, rescued + + +# A year below this is a source cell with a digit missing, not a date: the +# trackers record births, diagnoses and visits, none of which predate 1900. +# dateutil reads "1/16/224" as the year 224 without complaint, so the guard has +# to sit outside it, and it mirrors the cleaned stage's own beyond-tracker-year +# guard at the other end of the calendar (ticket 55). +_MIN_PLAUSIBLE_YEAR = 1900 + + +# Characters that carry no meaning of their own but survive a copy-paste out of +# a browser or a chat message and make an otherwise clean date unparseable. +_INVISIBLE_CHARS = str.maketrans(dict.fromkeys("\u200b\u200c\u200d\u2060\ufeff")) + + +# A separator run damaged by a stray keystroke: either two or more separator +# characters where one belongs ("26-05- 2007", "23/05//2025", "02-Apr=-2026"), +# or a single character that is never a date separator to begin with ("_", "="). +# dateutil requires the separator to be well-formed, so without this repair +# each of these loses its date entirely (ticket 56). +# +# Deliberately narrow. A single "/" or "." is left alone so that no +# already-parsing value changes reading, and a whitespace-only run is left alone +# so the trailing-free-text path ("16-Nov-2019 due to DKA") still sees its own +# word boundaries. +_DAMAGED_SEPARATOR = re.compile(r"(?=[-/_=.\s]*[-/_=.])[-/_=.\s]{2,}|[_=]") + + +def parse_date_flexible( + date_str: str | None, + error_val: str = "9999-09-09", + tracker_year: int | None = None, +) -> date | None: + """Parse date strings flexibly using Python's dateutil.parser. + + Handles common edge cases from A4D tracker data: + - NA/None/empty values → None + - Excel serial numbers (e.g., "45341.0") → converted from days since 1899-12-30 + - Long month names (e.g., "March") → truncated to 3 letters before parsing + - A date followed by a free-text clause ("16-Nov-2019 due to DKA") → the + date, via the longest parseable prefix + - All standard date formats via dateutil.parser (very flexible) + + Examples: + "Mar-18" → 2018-03-01 + "28/8/2017" → 2017-08-28 + "45341.0" → 2024-01-13 (Excel serial) + "January-20" → 2020-01-01 + "Jun 2006" → 2006-06-01 + + Args: + date_str: Date string to parse + error_val: Value to parse and return on failure (default "9999-09-09") + + Returns: + Parsed date, None for NA/empty, or error date if parsing fails + """ + return parse_date_detailed(date_str, error_val, tracker_year)[0] + + +def parse_date_detailed( + date_str: str | None, + error_val: str = "9999-09-09", + tracker_year: int | None = None, +) -> tuple[date | None, TextDateRecovery | None]: + """``parse_date_flexible`` plus what the free-text recogniser did, if it ran. + + The second element is None whenever the cell parsed without reaching the + recogniser, which is the overwhelming majority. It exists so + ``parse_date_column`` can log the decisions taken on a note -- what was + read, what was discarded, what was filled in from the tracker year -- + without parsing every string twice to find out (ticket 39). + """ + # Handle None and every way a tracker records "no date here" + if date_str is None or _is_missing_date_text(str(date_str)): + return None, None + + date_str = str(date_str).translate(_INVISIBLE_CHARS).strip() + + recovery: TextDateRecovery | None = None + + # A stay written as a bare range, with no prose around it to stop the + # ordinary readings, is misread rather than refused: dateutil takes the + # range's first number for a year, so "6-12 Nov 2020" becomes 2006-11-12 and + # "3-9 Sep 2020" becomes 2003-09-09. The range patterns are explicit about + # which number is the admission day, so they are consulted first when the + # cell *opens* with one (ticket 39). + leading = _DATE_IN_TEXT.match(date_str) + if leading is not None and _is_range(leading): + recovery = recover_date_from_text(date_str, tracker_year) + if recovery.value is not None: + return recovery.value, recovery + recovery = None + + result = _parse_date_str(date_str) + if result is None: + repaired = _DAMAGED_SEPARATOR.sub("-", date_str) + # A fourth number means the repair joined something that was never one + # date: "26-05- 2007" is a damaged separator, where "1/2/2021-11/2/2021" + # is two dates and reading it as one invents a date neither says. + # Repair only what can still be a day/month/year. + if repaired != date_str and len(re.findall(r"\d+", repaired)) <= 3: + result = _parse_date_str(repaired) + if result is None: + # The date is somewhere inside a note rather than in a date-shaped cell. + # Tried ahead of the prefix walk because it is the *explicit* reading: + # the walk shortens the string until something parses and ends at + # dateutil, so it will read the fragment "3-9 Sep" of "3-9 Sep 2020" as + # the year 3 and publish 2003-09-09, where the range pattern reads the + # admission day the note states (ticket 39). + recovery = recover_date_from_text(date_str, tracker_year) + result = recovery.value + if result is None: + recovery = None + result = _parse_longest_parseable_prefix(date_str) + if result is not None: + # Reaching the walk means the ordinary readings failed, so this is + # still a date recovered from free text and still reportable -- + # it is the shape the recogniser has no pattern for, e.g. a note + # opening with a bare year ("2019 DKA"). + recovery = TextDateRecovery(result, 1, False) + if result is not None and result.year >= _MIN_PLAUSIBLE_YEAR: + return result, recovery + + logger.bind(error_code="invalid_value").warning( + f"Could not parse date '{date_str}'. Returning error value {error_val}" + ) + try: + return datetime.strptime(error_val, "%Y-%m-%d").date(), recovery + except ValueError: + return None, recovery + + +def _parse_longest_parseable_prefix(date_str: str) -> date | None: + """Drop trailing words until what remains parses as a date. + + Hospitalisation and diagnosis cells routinely carry a clause after the + date ("16-Nov-2019 due to DKA", "Jan-2020 due to poor glycaemic control"). + The caller used to strip these by taking everything before the first + space, which also destroyed genuine space-separated dates like + "Jun 2006" (ticket 37) -- so the truncation lives here instead, tried only + after the whole string has failed. + + A purely alphabetic prefix is skipped: dateutil would complete a bare + month name from *today*, making the result depend on the run date. + + A small bare number is skipped for the mirror-image reason (ticket 53): + truncating "26 Jun (ceton urine high)" down to "26" leaves a day-of-month, + which ``_parse_date_str`` would read as an Excel serial and turn into a + 1900 date. A four-digit year is still a legitimate prefix ("2019 DKA"), so + the cut is at the bare-year floor rather than at "is it numeric". + """ + tokens = date_str.split() + for end in range(len(tokens) - 1, 0, -1): + prefix = " ".join(tokens[:end]).rstrip(",;.-/ ") + if prefix.replace("-", "").replace(",", "").isalpha(): + continue + if prefix.isdigit() and int(prefix) < BARE_YEAR_MIN: + continue + result = _parse_date_str(prefix) + if result is not None: + logger.debug(f"Parsed '{date_str}' via prefix '{prefix}' → {result}") + return result + return None + + +_MONTH_TOKEN = r"(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?" + + +# The date shapes a clinician actually writes inside a note, in the order they +# are tried at each scan position (ticket 39). Every alternative is anchored on +# a day/month/year shape, so a cell carrying numbers and no date -- "3 month +# come back meet Doctor", "on stamlor 5mg" -- yields nothing rather than a date +# built from today, which is what dateutil's fuzzy mode returns for all of +# them. +# +# The two range alternatives come first because a stay is written with both +# endpoints ("6-12 Nov 2020", "20-29/12/2020") and the day wanted is the +# admission, i.e. the first: without them the general alternatives would match +# from the second number and publish the discharge date. +_DATE_IN_TEXT = re.compile( + rf""" + (?P\b(?P\d{{1,2}})\s*[-–]\s*\d{{1,2}}\s*[/.]\s* + (?P\d{{1,2}})\s*[/.]\s*(?P\d{{2,4}})\b) + | (?P\b(?P\d{{1,2}})(?:st|nd|rd|th)?\s*[-–]\s* + \d{{1,2}}(?:st|nd|rd|th)?\s*[-–]?\s* + (?P{_MONTH_TOKEN}) + (?:\s*[-,./ ]?\s*'?(?P\d{{2,4}})\b)?) + | (?P\b(?P\d{{1,2}})\s*[/.-]\s*(?P\d{{1,2}})\s*[/.-]\s* + (?P\d{{2,4}})\b(?!\s*/\s*\d)) + | (?P\b(?P\d{{1,2}})(?:st|nd|rd|th)?\s*[-./ ]?\s* + (?P{_MONTH_TOKEN})\s*[-,./ ]?\s*'?(?P\d{{2,4}})\b) + | (?P\b(?P{_MONTH_TOKEN})\s*[-,./ ]?\s*'?(?P\d{{2,4}})\b) + | (?P\b(?P\d{{1,2}})(?:st|nd|rd|th)?\s*[-./ ]?\s* + (?P{_MONTH_TOKEN})\b(?![-,./=_ ]*'?\d)) + """, + re.IGNORECASE | re.VERBOSE, +) + +_MONTH_NUMBERS = { + "jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6, + "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12, +} # fmt: skip + + +def _is_range(match: re.Match[str]) -> bool: + """True when this match is a stay written with both its endpoints.""" + return match.group("range_num") is not None or match.group("range_mon") is not None + + +@dataclass(frozen=True) +class TextDateRecovery: + """What the free-text recogniser read out of one cell. + + ``tokens_found`` and ``year_inferred`` exist so the caller can log *which* + decision was taken, not just the result: a cell naming three admissions and + a cell naming one are both published as a single date, and only the first + is a source-tracker defect worth reporting back to the clinic. + """ + + value: date | None + tokens_found: int + year_inferred: bool + + +def _expand_two_digit_year(digits: str) -> int: + """00-68 -> 2000-2068, 69-99 -> 1969-1999, matching the month-year branch.""" + if len(digits) == 4: + return int(digits) + year = int(digits) + return 2000 + year if year <= 68 else 1900 + year + + +def recover_date_from_text(text: str, tracker_year: int | None = None) -> TextDateRecovery: + """Read the first date a clinical note contains. + + The ``hospitalisation_date`` header is free text -- "Hospitalisation due to + diabetes emergency or glucose control (Include Date)" -- so clinicians write + a case note with the date somewhere inside it. Recovery previously depended + on where the date sat, because the prefix walk can only reach one at the + front of the string: "16-Nov-2019 due to DKA" was published and + "DKA 16-Nov-2019" was sentinelled. + + Where a note records several dates, the first is published and the rest are + discarded -- a single date column cannot represent three admissions, and the + repair for that is the source workbook, not a cleverer parser. A day the + note omits becomes the 1st; a year it omits comes from the tracker, which is + the one component taken from outside the cell. + """ + matches = list(_DATE_IN_TEXT.finditer(text)) + if not matches: + return TextDateRecovery(None, 0, False) + + first = matches[0] + # A range names two days, so it is a discard even though it is one stay. + tokens = len(matches) + (1 if _is_range(first) else 0) + groups = first.groupdict() + + for prefix in ("rn", "rm", "fn", "fm", "my", "dm"): + if groups.get(f"{prefix}_month") is None: + continue + raw_month = groups[f"{prefix}_month"] + month = ( + int(raw_month) if raw_month.isdigit() else _MONTH_NUMBERS.get(raw_month[:3].lower(), 0) + ) + day = int(groups[f"{prefix}_day"]) if groups.get(f"{prefix}_day") else 1 + raw_year = groups.get(f"{prefix}_year") + year_inferred = raw_year is None + if year_inferred: + if tracker_year is None: + return TextDateRecovery(None, tokens, False) + year = tracker_year + else: + year = _expand_two_digit_year(raw_year) + try: + value = date(year, month, day) + except ValueError: + return TextDateRecovery(None, tokens, False) + if not _MIN_PLAUSIBLE_YEAR <= year <= BARE_YEAR_MAX: + return TextDateRecovery(None, tokens, False) + return TextDateRecovery(value, tokens, year_inferred) + + return TextDateRecovery(None, tokens, False) + + +def _parse_date_str(date_str: str) -> date | None: + """Parse one already-trimmed, non-null string. None means unparseable.""" + # Handle Excel serial numbers + # Excel stores dates as number of days since 1899-12-30 + try: + numeric_val = float(date_str) + if numeric_val.is_integer() and BARE_YEAR_MIN <= numeric_val <= BARE_YEAR_MAX: + result = date(int(numeric_val), 1, 1) + logger.debug(f"Parsed bare year {date_str} → {result}") + return result + if 1 < numeric_val < MAX_EXCEL_DATE_SERIAL: + days = int(numeric_val) + result = EXCEL_EPOCH + timedelta(days=days) + logger.debug(f"Parsed Excel serial {date_str} → {result}") + return result + except ValueError: + pass # Not a number, continue with text parsing + + # Truncate long month names to their 3-letter abbreviation + # ("March" -> "Mar", "January" -> "Jan", "Sept" -> "Sep"), which the + # month-year branch below and dateutil both handle. + # + # Anchored on the month names themselves rather than on any run of 4+ + # letters (ticket 37): the previous form dropped only the *fourth* letter, + # so "March" became "Marh" and "January" became "Janary" -- unparseable, + # and sentinelled as an error date. Every full month name in the trackers + # was affected. + date_str = MONTH_NAME_PATTERN.sub(lambda m: m.group(1).title(), date_str) + + # Special handling for month-year formats (e.g., "Mar-18", "Jan-20", "May18") + # These should be interpreted as "Mar 2018", "Jan 2020", not "Mar day-18 of current year" + # Separator (hyphen/space) is optional to handle both "May-18" and "May18" + # A 4-digit year ("Jun 2006", the 2017-era trackers' diagnosis-date format) + # is handled here too rather than left to dateutil: dateutil fills the + # missing day from datetime.now(), which makes the parse depend on the day + # the pipeline runs -- and can push a genuinely historical date past the + # tracker year, where _validate_dates then sentinels it (ticket 37). + # The apostrophe spelling ("Jun'09", "Apr'21") is admitted alongside the + # hyphen and the space (ticket 53): without it the value fell through to + # dateutil, which read the two digits as a day-of-month in the current year. + month_year_pattern = r"^([A-Za-z]{3})[-\s']?(\d{2}|\d{4})$" + match = re.match(month_year_pattern, date_str) + if match: + month_abbr, year_digits = match.groups() + if len(year_digits) == 4: + year_4digit = int(year_digits) + else: + # Convert 2-digit year to 4-digit: 00-68 → 2000-2068, 69-99 → 1969-1999 + year_int = int(year_digits) + if year_int <= 68: + year_4digit = 2000 + year_int + else: + year_4digit = 1900 + year_int + # Parse as "Mon YYYY" format, defaults to first day of month + date_str_full = f"{month_abbr} {year_4digit}" + try: + result = datetime.strptime(date_str_full, "%b %Y").date() + logger.debug(f"Parsed month-year '{date_str}' → {result}") + return result + except ValueError: + pass # Fall through to general parser + + # Try explicit DD/MM/YYYY and DD-MM-YYYY formats first (Southeast Asian standard) + # This is more reliable than dateutil.parser's dayfirst=True parameter + for fmt in [ + "%d/%m/%Y", # 06/05/2013 → 2013-05-06 (6th May) + "%d-%m-%Y", # 06-05-2013 → 2013-05-06 + "%d/%m/%y", # 06/05/13 → 2013-05-06 + "%d-%m-%y", # 06-05-13 → 2013-05-06 + "%Y-%m-%d", # 2013-05-06 (ISO format from Excel) + "%d/%m/%Y %H:%M:%S", # With time component + "%Y-%m-%d %H:%M:%S", # ISO with time + ]: + try: + result = datetime.strptime(date_str, fmt).date() + logger.debug(f"Parsed '{date_str}' using format {fmt} → {result}") + return result + except ValueError: + continue + + # Fall back to dateutil.parser for other formats (month names, etc.) + # dayfirst=True is still useful for remaining ambiguous cases + return _parse_with_dateutil(date_str) + + +# Two defaults that share no field value, so any component dateutil takes from +# the default differs between the two parses and can be told apart from one the +# string actually supplied. Both are far outside the trackers' range, so a +# genuine value can never coincide with either. +_PROBE_DEFAULT_A = datetime(1111, 1, 1) +_PROBE_DEFAULT_B = datetime(2222, 2, 2) + + +def _parse_with_dateutil(date_str: str) -> date | None: + """dateutil's reading, with every component it invented from today rejected. + + ``dateutil.parser.parse`` fills any field the string omits from + ``datetime.now()``, so "10/2019" becomes the 19th of October on the 19th + and the 3rd on the 3rd -- the cleaned output changes with no input change. + Ticket 50 hit this through a rich-text-damaged "July2014" and fixed that one + route into it; ticket 53 found the same fill still reachable through the + numeric ("10/2019") and comma-separated ("Mar, 2017") month-year spellings, + which the alphabetic month-year branch above does not match, and through a + bare month name inside a longer string. + + Parsing twice against two disjoint defaults tells a supplied component from + an invented one without having to enumerate the spellings that reach here: + - an invented **day** resolves to the 1st, the same convention the + alphabetic month-year branch already uses; + - an invented **month or year** makes the cell unparseable, because + neither can be recovered from the string and the current date is not an + answer. + """ + try: + probe_a = date_parser.parse(date_str, dayfirst=True, default=_PROBE_DEFAULT_A) + probe_b = date_parser.parse(date_str, dayfirst=True, default=_PROBE_DEFAULT_B) + except ValueError, date_parser.ParserError, OverflowError: + return None + + if probe_a.year != probe_b.year or probe_a.month != probe_b.month: + logger.debug(f"Rejected '{date_str}': dateutil would supply the month/year from today") + return None + + day = probe_a.day if probe_a.day == probe_b.day else 1 + result = date(probe_a.year, probe_a.month, day) + logger.debug(f"Parsed '{date_str}' with dateutil → {result}") + return result diff --git a/src/a4d/clean/glucose.py b/src/a4d/clean/glucose.py new file mode 100644 index 0000000..6bd501b --- /dev/null +++ b/src/a4d/clean/glucose.py @@ -0,0 +1,159 @@ +"""Resolve glucose readings recorded in the wrong unit. + +The trackers carry fasting blood glucose in unit-specific columns, but the unit +a clinic actually records in does not always match the header it records under. +A4D's medical advisor confirmed (2026-08-17) the analytical limits of the +machines involved and authorised correcting a reading "if it is clearly a wrong +unit mixed up" -- which makes the question where "clearly" stops. + +Measured across all 254 trackers, the confusion is overwhelmingly *per column* +rather than per row, and it clusters by clinic: 29 file-column groups have +nine tenths or more of their readings in mmol territory under an mg/dL header +(``2025_Kantha Bopha II``'s updated FBG is 99.8% of 951 readings), which cannot +be a run of severe hypoglycaemia. A single low reading in an otherwise ordinary +column is a different matter -- a genuine severe hypo and a mis-entered unit +look identical -- so those are flagged and left alone. Converting them would +multiply the clinically most important readings in the file by 18 and hide them. +""" + +import polars as pl +from loguru import logger + +from a4d.errors import ErrorCollector + +# mmol/L * 18 = mg/dL. The pipeline's existing cross-derivation uses the same +# rounded factor, so unit resolution stays consistent with it. +MMOL_TO_MG_FACTOR = 18.0 + +# A reading below this in an mg/dL column lands exactly where a mmol/L number +# would. It is not by itself impossible -- the advisor's mg/dL analytical floor +# is ~2-5 -- which is why it only ever triggers suspicion, never a row-level +# conversion. +MMOL_MG_BOUNDARY = 30.0 + +# A column is taken to be recorded in mmol/L only when nearly all of it reads +# that way. The measured distribution is strongly bimodal: 29 groups sit above +# 0.9 and 300 sit at 0.0, with the 0.5-0.9 band being per-patient mixing inside +# one column at two clinics rather than a mislabelled column. +UNIT_SWAP_COLUMN_SHARE = 0.9 +UNIT_SWAP_MIN_READINGS = 10 + +# The analytical limits A4D's medical advisor gave, at the permissive end of +# each range he stated (mg/dL ~2-5 to ~720-800; mmol/L ~0.1-0.3 to ~40-45), so +# only a reading no machine could have produced is rejected. He separately +# confirmed that above 100 mmol/L is outright impossible. +MG_ANALYTICAL_MIN = 2.0 +MG_ANALYTICAL_MAX = 800.0 +MMOL_ANALYTICAL_MIN = 0.1 +MMOL_ANALYTICAL_MAX = 45.0 + +GLUCOSE_COLUMN_PAIRS: list[tuple[str, str]] = [ + ("fbg_baseline_mg", "fbg_baseline_mmol"), + ("fbg_updated_mg", "fbg_updated_mmol"), +] + +# Zero is below the analytical floor of both units (~2-5 mg/dL, ~0.1-0.3 +# mmol/L), so it cannot be a measurement. All 244 baseline mmol/L readings under +# 3 mmol/L in the tracker set are exactly 0 -- a placeholder for "not measured", +# which the advisor did not contest. +GLUCOSE_COLUMNS: list[str] = [col for pair in GLUCOSE_COLUMN_PAIRS for col in pair] + + +def column_is_recorded_in_mmol(values: pl.Series) -> bool: + """Is this mg/dL-labelled column actually holding mmol/L readings? + + True only when the column has enough readings to judge and nearly all of + them fall below the mmol/mg boundary. Nulls are not readings. + """ + readings = values.drop_nulls() + if len(readings) < UNIT_SWAP_MIN_READINGS: + return False + below = int((readings < MMOL_MG_BOUNDARY).sum()) + return below / len(readings) >= UNIT_SWAP_COLUMN_SHARE + + +def _blank_zero_readings(df: pl.DataFrame) -> pl.DataFrame: + present = [col for col in GLUCOSE_COLUMNS if col in df.columns] + if not present: + return df + return df.with_columns( + [pl.when(pl.col(col) == 0).then(None).otherwise(pl.col(col)).alias(col) for col in present] + ) + + +def _swap_column_to_mmol( + df: pl.DataFrame, mg_col: str, mmol_col: str, error_collector: ErrorCollector +) -> pl.DataFrame: + """Move a wholly mmol-recorded column into its mmol sibling and rescale.""" + recorded = pl.col(mg_col) + df = df.with_columns( + # An mmol reading already present was recorded under a correctly + # labelled header, so it wins over the one being recovered. + pl.when(pl.col(mmol_col).is_null()) + .then(recorded) + .otherwise(pl.col(mmol_col)) + .alias(mmol_col), + (recorded * MMOL_TO_MG_FACTOR).alias(mg_col), + ) + + # The defect is the column's label, so the correction the source workbook + # needs is file-level; a row-level record here would be the same finding + # repeated hundreds of times. + file_name = df["file_name"][0] if "file_name" in df.columns and len(df) else "unknown" + readings = int(df[mmol_col].drop_nulls().len()) + error_collector.add_error( + file_name=file_name or "unknown", + patient_id="all", + column=mg_col, + original_value=None, + error_message=( + f"Column is labelled mg/dL but {UNIT_SWAP_COLUMN_SHARE:.0%}+ of its " + f"{readings} readings are in mmol/L range; values moved to {mmol_col} " + f"and rescaled by {MMOL_TO_MG_FACTOR:g}. Correct the header in the source workbook." + ), + error_code="glucose_unit_swapped", + function_name="resolve_glucose_units", + ) + logger.bind(error_code="glucose_unit_swapped").warning( + f"{file_name}: {mg_col} is recorded in mmol/L; moved to {mmol_col}" + ) + return df + + +def _flag_suspect_readings(df: pl.DataFrame, mg_col: str, error_collector: ErrorCollector) -> None: + suspect = df.filter(pl.col(mg_col).is_not_null() & (pl.col(mg_col) < MMOL_MG_BOUNDARY)) + for row in suspect.iter_rows(named=True): + error_collector.add_error( + file_name=row.get("file_name") or "unknown", + patient_id=row.get("patient_id") or "unknown", + column=mg_col, + original_value=row[mg_col], + error_message=( + f"Reading {row[mg_col]} in an mg/dL column is below " + f"{MMOL_MG_BOUNDARY:g} mg/dL, where a mmol/L value lands. Kept as " + "recorded -- a severe hypoglycaemic reading looks the same. " + "Confirm the unit in the source workbook." + ), + error_code="glucose_unit_suspect", + function_name="resolve_glucose_units", + ) + + +def resolve_glucose_units(df: pl.DataFrame, error_collector: ErrorCollector) -> pl.DataFrame: + """Correct whole columns recorded in the wrong unit; flag stray readings. + + Runs before range validation so the analytical limits are applied to + corrected values, and before the mg/mmol cross-derivation so that step sees + a column whose unit matches its name. + """ + df = _blank_zero_readings(df) + + for mg_col, mmol_col in GLUCOSE_COLUMN_PAIRS: + if mg_col not in df.columns or mmol_col not in df.columns: + continue + if column_is_recorded_in_mmol(df[mg_col]): + df = _swap_column_to_mmol(df, mg_col, mmol_col, error_collector) + else: + _flag_suspect_readings(df, mg_col, error_collector) + + return df diff --git a/src/a4d/clean/patient.py b/src/a4d/clean/patient.py new file mode 100644 index 0000000..21dd26f --- /dev/null +++ b/src/a4d/clean/patient.py @@ -0,0 +1,1125 @@ +"""Patient data cleaning pipeline. + +This module orchestrates the complete cleaning pipeline for patient data. The +ordering below is load-bearing, not cosmetic: the meta schema is applied before +type conversion so that unit conversions and derivations can address columns a +given tracker never had, and every derivation runs after the conversion that +produces its inputs and before the validation that bounds its outputs. + +1. Load raw patient data +2. Apply legacy format fixes +3. Apply transformations +4. Type conversions +5. Validation +6. Apply meta schema (ensure all columns exist, consistent output) +""" + +from pathlib import Path + +import polars as pl +from loguru import logger + +from a4d.clean.buddhist_era import BUDDHIST_ERA_THRESHOLD, gregorian_from_buddhist +from a4d.clean.converters import ( + correct_decimal_sign, + cut_numeric_value, + normalize_excel_formula_errors, + parse_date_column, + safe_convert_column, +) +from a4d.clean.glucose import ( + MG_ANALYTICAL_MAX, + MG_ANALYTICAL_MIN, + MMOL_ANALYTICAL_MAX, + MMOL_ANALYTICAL_MIN, + resolve_glucose_units, +) +from a4d.clean.schema import ( + apply_schema, + get_date_columns, + get_patient_data_schema, +) +from a4d.clean.transformers import extract_regimen, strip_string_whitespace +from a4d.clean.validators import validate_all_columns +from a4d.config import settings +from a4d.errors import ErrorCollector +from a4d.extract.common import normalize_patient_id_expr + + +def clean_patient_data( + df_raw: pl.DataFrame, + error_collector: ErrorCollector, +) -> pl.DataFrame: + """Clean raw patient data following the complete pipeline. + + This function orchestrates all cleaning steps and ensures the output + conforms to the meta schema, regardless of which columns exist in input. + + Args: + df_raw: Raw patient data from extraction + error_collector: ErrorCollector instance for tracking errors + + Returns: + Cleaned DataFrame with complete meta schema applied + + Example: + >>> from a4d.extract.patient import extract_patient_data + >>> from a4d.errors import ErrorCollector + >>> + >>> collector = ErrorCollector() + >>> df_raw = extract_patient_data(tracker_file) + >>> df_clean = clean_patient_data(df_raw, collector) + >>> # df_clean has ALL schema columns, with consistent types + """ + logger.info( + f"Starting patient data cleaning: {len(df_raw)} rows, {len(df_raw.columns)} columns" + ) + + # Step 0: Null out (and log) the source trackers' own formula-error + # strings, which extraction preserves verbatim (ticket 27). Must run + # before type conversion so these don't land in safe_convert_column's + # parse-failure branch and pick up the 999999 sentinel. + df_raw = normalize_excel_formula_errors(df_raw, error_collector) + + # Step 0.5: Strip whitespace from the ends of every string cell (ticket + # 36). End-whitespace never carries meaning in this data, and it runs + # before validation rather than after because a stray trailing space makes + # an otherwise-valid value fail allowed-value validation and land on the + # "Undefined" sentinel: 2019 Kantha Bopha's KH_KB023 has a sex cell reading + # 'F ', and 72 rows of patient `sex` were being lost that way. Product's + # cleaning already did this (step 2.16), which left the two arms + # disagreeing on `file_name` and `sheet_name` for any tracker whose + # filename or sheet tab carries a stray space -- identifiers that join the + # arms' tables together. + df_raw = strip_string_whitespace(df_raw) + + # Step 1: Legacy format fixes + df = _apply_legacy_fixes(df_raw) + + # Step 2: Pre-processing transformations + df = _apply_preprocessing(df) + + # Step 3: Data transformations (regimen extraction, lowercasing, etc.) + df = _apply_transformations(df) + + # Step 4: Apply the meta schema EARLY, so every column exists before the + # conversions below. Trackers differ in which columns they carry, and the + # unit conversions and derivations that follow address columns by name -- + # without this they would silently skip whatever a given workbook omitted. + df = apply_schema(df) + + # Step 5: Type conversions + df = _apply_type_conversions(df, error_collector) + + # Step 5.4: Convert Buddhist-era dates to Gregorian (ticket 61). + # Before _fix_age_from_dob so no age is derived from a BE dob, and before + # _validate_dates, which would otherwise sentinel every one of them. + df = _convert_buddhist_era_dates(df, error_collector) + + # Step 5.5: Derive age from DOB, overriding the sheet's own Age cell -- + # a typed age goes stale between the month it was written and the month + # the sheet covers, while a date of birth does not (LA_MH060's D.O.B. of + # 2009-01-16 makes Jan23 an age of 14; the clinic's cell says 13). + # Must happen after type conversions so DOB is a proper date, and before + # range validation so the validated age is the derived one. + df = _fix_age_from_dob(df, error_collector) + + # Step 5.5b: Fill t1d_diagnosis_age from dob and t1d_diagnosis_date, but + # only where the tracker recorded no usable age of its own (ticket 52) + df = _fix_t1d_diagnosis_age(df) + + # Step 5.6: Validate dates (replace future dates with error value) + # Must happen after type conversions so dates are proper date types + df = _validate_dates(df, error_collector) + + # Step 5.7: Resolve glucose readings recorded under the wrong unit's header. + # Before range validation so the analytical limits judge corrected values, + # and before step 8 so the mg/mmol cross-derivation sees matching units. + df = resolve_glucose_units(df, error_collector) + + # Step 6: Range validation and cleanup + df = _apply_range_validation(df, error_collector) + + # Step 7: Allowed values validation + df = validate_all_columns(df, error_collector) + + # Step 8: Unit conversions (requires schema to be applied first!) + df = _apply_unit_conversions(df) + + # Step 9: Create tracker_date from year/month + df = _add_tracker_date(df) + + # Step 10: Sort by tracker_date and patient_id + df = df.sort(["tracker_date", "patient_id"]) + + logger.info(f"Cleaning complete: {len(df)} rows, {len(df.columns)} columns") + logger.info(f"Errors collected: {len(error_collector)}") + + return df + + +def _extract_date_from_measurement(df: pl.DataFrame, col_name: str) -> pl.DataFrame: + """Extract date from measurement values in legacy trackers. + + For pre-2019 trackers, values and dates are combined in format: + - "14.5 (Jan-20)" → value="14.5 ", date="Jan-20" + - ">14 (Mar-18)" → value=">14 ", date="Mar-18" + - "148 mg/dl (Mar-18)" → value="148 mg/dl ", date="Mar-18" + + Args: + df: Input DataFrame + col_name: Column name containing combined value+date + + Returns: + DataFrame with extracted date in {col_name}_date column + """ + if col_name not in df.columns: + return df + + date_col_name = col_name.replace("_mg", "").replace("_mmol", "") + "_date" + + # Check if date column already exists (2019+ trackers) + if date_col_name in df.columns: + return df + + # The closing parenthesis is OPTIONAL, and that is the whole point: 25 of + # the 30 cells in this population are written `180(May-2017` with no closing + # bracket at all, near-universal in the 2017/2018 trackers. Requiring it + # lost the date on every one of them. The prefix is greedy so that a doubled + # bracket (`196((Dec-2017)`) yields the date after the LAST `(`. + df = df.with_columns( + [ + # The greedy prefix leaves a trailing "(" behind on a doubled + # bracket, which would then fail the numeric cast and lose the + # reading entirely -- so strip it and keep the 196. + pl.when(pl.col(col_name).str.contains(r"\(", literal=False)) + .then( + pl.col(col_name).str.extract(r"^(.*)\(", 1).str.strip_chars().str.strip_chars("(") + ) + .otherwise(pl.col(col_name)) + .alias(col_name), + pl.col(col_name).str.extract(r".*\(([^)]*)\)?$", 1).alias(date_col_name), + ] + ) + + logger.debug(f"Extracted date from {col_name} into {date_col_name}") + + return df + + +def _apply_legacy_fixes(df: pl.DataFrame) -> pl.DataFrame: + """Apply fixes for legacy tracker formats (pre-2024). + + Legacy trackers may have: + - Combined date+value columns (e.g., hba1c_updated contains both) + - Combined blood pressure values (sys/dias in one column) + - Different column structures + + Args: + df: Input DataFrame + + Returns: + DataFrame with legacy fixes applied + """ + # Extract dates from measurement columns for pre-2019 trackers. Skipped + # where the tracker already carries a dedicated *_date column (2019+). + df = _extract_date_from_measurement(df, "hba1c_updated") + df = _extract_date_from_measurement(df, "fbg_updated_mg") + df = _extract_date_from_measurement(df, "fbg_updated_mmol") + + # Split blood pressure for pre-2024 trackers, which record sys/dias in + # one cell. + if "blood_pressure_mmhg" in df.columns: + from a4d.clean.transformers import split_bp_in_sys_and_dias + + df = split_bp_in_sys_and_dias(df) + + return df + + +def _fix_fbg_column(col: pl.Expr) -> pl.Expr: + """Fix FBG column text values to numeric equivalents. + + Converts qualitative text to numeric values and removes DKA markers. + + Every pattern here is anchored to the whole cell, deliberately. Matching + them as substrings turns `Lost follow up` into a glucose of 140 (because + "fol-low" contains "low") and every `SMBG 50-HI` / `129-HI` into 200, + discarding the number the clinic actually wrote. A cell that is not + wholly one of these words is left for numeric parsing or sentinelled. + + Conversions (based on CDC guidelines): + - "high", "bad", "hi", "hight" (typo) → "200" + - "medium", "med" → "170" + - "low", "good", "okay" → "140" + - Remove "(DKA)" text, "mg/dl", "mmol/l" suffixes + - Trim whitespace + + Args: + col: Polars expression for FBG column + + Returns: + Polars expression with fixed values + """ + return ( + col.str.to_lowercase() + # Remove unit suffixes (from legacy trackers like 2018) + .str.replace_all(r"\s*mg/dl\s*", "", literal=False) + .str.replace_all(r"\s*mmol/l\s*", "", literal=False) + # Use case-when to match full words, not substrings + .str.replace_all(r"^(high|hight|bad|hi)$", "200") # Anchored to full string + .str.replace_all(r"^(med|medium)$", "170") + .str.replace_all(r"^(low|good|okay)$", "140") + .str.replace_all(r"\(DKA\)", "", literal=True) + .str.strip_chars() + ) + + +def _apply_preprocessing(df: pl.DataFrame) -> pl.DataFrame: + """Apply preprocessing transformations before type conversion. + + This includes: + - Normalizing patient_id (remove transfer clinic suffix) + - Removing > and < signs from HbA1c values (but tracking them) + - Fixing FBG text values (high/medium/low → numeric, removing (DKA)) + - Replacing "-" with "N" in Y/N columns + - Deriving insulin_type and insulin_subtype from individual columns (2024+) + + Args: + df: Input DataFrame + + Returns: + DataFrame with preprocessing applied + """ + # Normalize patient_id: Keep only COUNTRY_ID part, remove transfer clinic suffix + # Pattern: "MY_SM003_SB" → "MY_SM003" (keep first two underscore-separated parts) + # Also normalizes hyphens first: "LA-MH093_LF" → "LA_MH093_LF" → "LA_MH093" + # This ensures consistent patient linking across years when patients transfer clinics + # Shared with the extraction-side Patient List join key (ticket 58), so the + # two cannot drift into disagreeing about what one patient's identity is. + if "patient_id" in df.columns: + df = df.with_columns(normalize_patient_id_expr(pl.col("patient_id")).alias("patient_id")) + + # Track HbA1c exceeds markers (> or <) + if "hba1c_baseline" in df.columns: + df = df.with_columns( + pl.col("hba1c_baseline") + .str.contains(r"[><]") + .fill_null(False) + .alias("hba1c_baseline_exceeds") + ) + df = df.with_columns( + pl.col("hba1c_baseline").str.replace_all(r"[><]", "").alias("hba1c_baseline") + ) + + if "hba1c_updated" in df.columns: + df = df.with_columns( + pl.col("hba1c_updated") + .str.contains(r"[><]") + .fill_null(False) + .alias("hba1c_updated_exceeds") + ) + df = df.with_columns( + pl.col("hba1c_updated").str.replace_all(r"[><]", "").alias("hba1c_updated") + ) + + # Convert qualitative values to numeric: high→200, medium→170, low→140 + # Source: https://www.cdc.gov/diabetes/basics/getting-tested.html + if "fbg_updated_mg" in df.columns: + df = df.with_columns(_fix_fbg_column(pl.col("fbg_updated_mg")).alias("fbg_updated_mg")) + + if "fbg_updated_mmol" in df.columns: + df = df.with_columns(_fix_fbg_column(pl.col("fbg_updated_mmol")).alias("fbg_updated_mmol")) + + # Replace "-" with "N" in Y/N columns (2024+ trackers use "-" for No) + yn_columns = [ + "analog_insulin_long_acting", + "analog_insulin_rapid_acting", + "human_insulin_intermediate_acting", + "human_insulin_pre_mixed", + "human_insulin_short_acting", + ] + + for col in yn_columns: + if col in df.columns: + df = df.with_columns(pl.col(col).str.replace("-", "N").alias(col)) + + # Derive insulin_type and insulin_subtype from individual columns (2024+). + # Values are emitted lowercase here; allowed-value validation canonicalises + # the case afterwards. + if "human_insulin_pre_mixed" in df.columns: + df = _derive_insulin_fields(df) + + return df + + +_INSULIN_NOT_TICKED = ("", "-", "0", "n", "no") + + +def _insulin_ticked(column: str) -> pl.Expr: + """Whether a 2024+ insulin tick box says this subtype applies. + + The template asks for `Y`, but 2024 Sarawak ticks by writing the drug's + name instead -- `Novorapid` in the rapid-acting column, `Glargine`, + `Toujeo` or `Ryzodeg` in the long-acting one. Testing for `Y` alone + discarded those 56 rows' subtypes entirely (ticket 55). The negative + markers are enumerated instead, because they are the closed set: across all + 248 trackers these five columns hold only `Y`, `-`, `0`, the four drug + names, and null. + """ + value = pl.col(column).cast(pl.String).str.strip_chars().str.to_lowercase() + return pl.col(column).is_not_null() & ~value.is_in(_INSULIN_NOT_TICKED) + + +def _derive_insulin_fields(df: pl.DataFrame) -> pl.DataFrame: + """Derive insulin_type and insulin_subtype from individual columns. + + Emits lowercase values; allowed-value validation canonicalises the case + afterwards. + + For 2024+ trackers: + - insulin_type: "human insulin" if any human column is Y, else "analog insulin" + - insulin_subtype: Comma-separated list like "pre-mixed,rapid-acting,long-acting" + Validation uses allow_csv_subset (see reference_data/validation_rules.yaml) to + accept each token against allowed_values and rejoin in canonical case. + + A row that ticks no insulin column at all yields None rather than a + subtype, so a patient with nothing recorded is not reported as having an + unrecognised regimen. + + Args: + df: Input DataFrame with individual insulin columns + + Returns: + DataFrame with insulin_type and insulin_subtype derived + """ + # Derive insulin_type only when at least one insulin column carries a + # value: a row where every column is null recorded nothing, and must not + # be reported as having a type. + df = df.with_columns( + pl.when( + # Only derive if at least one insulin column is not null + pl.col("human_insulin_pre_mixed").is_not_null() + | pl.col("human_insulin_short_acting").is_not_null() + | pl.col("human_insulin_intermediate_acting").is_not_null() + | pl.col("analog_insulin_rapid_acting").is_not_null() + | pl.col("analog_insulin_long_acting").is_not_null() + ) + .then( + # Now check which type + pl.when( + _insulin_ticked("human_insulin_pre_mixed") + | _insulin_ticked("human_insulin_short_acting") + | _insulin_ticked("human_insulin_intermediate_acting") + ) + .then(pl.lit("human insulin")) + .otherwise(pl.lit("analog insulin")) + ) + .otherwise(None) # every insulin column null: nothing was recorded + .alias("insulin_type") + ) + + # Build insulin_subtype as a comma-separated list, lowercase; validation + # canonicalises the case and checks each token separately (allow_csv_subset + # in reference_data/validation_rules.yaml). + df = df.with_columns( + pl.concat_list( + [ + pl.when(_insulin_ticked("human_insulin_pre_mixed")) + .then(pl.lit("pre-mixed")) + .otherwise(pl.lit(None)), + pl.when(_insulin_ticked("human_insulin_short_acting")) + .then(pl.lit("short-acting")) + .otherwise(pl.lit(None)), + pl.when(_insulin_ticked("human_insulin_intermediate_acting")) + .then(pl.lit("intermediate-acting")) + .otherwise(pl.lit(None)), + pl.when(_insulin_ticked("analog_insulin_rapid_acting")) + .then(pl.lit("rapid-acting")) + .otherwise(pl.lit(None)), + pl.when(_insulin_ticked("analog_insulin_long_acting")) + .then(pl.lit("long-acting")) + .otherwise(pl.lit(None)), + ] + ) + .list.drop_nulls() + .list.join(",") + # An unticked row deliberately keeps the empty string, which + # allowed-value validation publishes as "Undefined". That overstates + # what the clinic recorded -- it ticked nothing, rather than ticking + # something unrecognised -- and it affects 17,418 rows. Nulling them + # instead was measured and reverted (ticket 55): it resolved nothing + # downstream, and whether the consumer distinguishes the two cases has + # never been established. Left as-is, deliberately, not by oversight. + .alias("insulin_subtype") + ) + + return df + + +def _apply_transformations(df: pl.DataFrame) -> pl.DataFrame: + """Apply data transformations. + + Transformations are explicit Python code (not config-driven): + - Lowercase status for case-insensitive validation + - Standardize insulin regimen descriptions + - Map sex synonyms to M/F + - Correct European decimal format + + Args: + df: Input DataFrame + + Returns: + DataFrame with transformations applied + """ + # Standardize insulin regimen + if "insulin_regimen" in df.columns: + df = extract_regimen(df) + + # Map sex synonyms to M/F + if "sex" in df.columns: + from a4d.clean.transformers import fix_sex + + df = fix_sex(df) + + # Fix testing frequency ranges + if "testing_frequency" in df.columns: + from a4d.clean.transformers import fix_testing_frequency + + df = fix_testing_frequency(df) + + # Correct European decimal format (comma → dot) + numeric_cols = [ + "hba1c_baseline", + "hba1c_updated", + "fbg_updated_mg", + "fbg_updated_mmol", + "weight", + "height", + "bmi", + ] + + for col in numeric_cols: + if col in df.columns: + df = correct_decimal_sign(df, col) + + return df + + +def _apply_type_conversions(df: pl.DataFrame, error_collector: ErrorCollector) -> pl.DataFrame: + """Convert columns to target types using safe_convert_column. + + Only converts columns that exist in both the DataFrame and the schema. + + Special handling: + - Date columns: Use flexible date parser (handles Mar-18, Excel serials, etc.) + - Integer columns: Convert via Float64 first to handle decimals + + Args: + df: Input DataFrame + error_collector: ErrorCollector for tracking conversion failures + + Returns: + DataFrame with types converted + """ + schema = get_patient_data_schema() + + # Convert each column that exists + for col, target_type in schema.items(): + if col not in df.columns: + continue + + # Skip if already the correct type (happens when schema adds NULL columns) + if df[col].dtype == target_type: + continue + + # Special handling for Date columns: use flexible date parser + if target_type == pl.Date: + # Strip a trailing time component if present + # (e.g., "2009-04-17 00:00:00" → "2009-04-17"). Matches the time + # itself rather than splitting on the first space: a date can + # legitimately contain one ("Jun 2006", ticket 37), and taking the + # first token dropped the year and left a month name the parser + # then completed from the current date. + df = df.with_columns( + pl.col(col) + .cast(pl.Utf8) + .str.replace(r"\s+\d{1,2}:\d{2}(:\d{2})?(\.\d+)?$", "") + .alias(col) + ) + # Use custom date parser for flexibility (handles Mar-18, Excel serials, etc.) + df = parse_date_column(df, col, error_collector) + # Special handling for Int32: convert via Float64 first (handles "14.0" → 14.0 → 14) + elif target_type == pl.Int32: + df = safe_convert_column(df, col, pl.Float64, error_collector) + df = df.with_columns(pl.col(col).round(0).cast(pl.Int32, strict=False).alias(col)) + else: + df = safe_convert_column( + df=df, + column=col, + target_type=target_type, + error_collector=error_collector, + ) + + return df + + +def _calculate_bmi(df: pl.DataFrame) -> pl.DataFrame: + """Calculate BMI from weight and height. + + REPLACES any existing BMI value with weight / height^2, because a typed + BMI goes stale as the patient's weight changes while the two measurements + it derives from are re-recorded each month. + + Must be called after height and weight have been range-validated and before + the BMI bound is applied. Deriving it first produced a BMI that passed its + own bound while resting on an impossible height: 60 / 2.43^2 = 10.16, from + a height cell that should have been rejected outright. + + Args: + df: Input DataFrame + + Returns: + DataFrame with calculated BMI column + """ + from a4d.clean.transformers import fix_bmi + + return fix_bmi(df) + + +def _apply_range_validation(df: pl.DataFrame, error_collector: ErrorCollector) -> pl.DataFrame: + """Apply range validation and value cleanup. + + This includes: + - Height: 0-2.3m (convert cm to m if needed) + - Weight: 0-200kg + - BMI: 4-60 + - Age: 0-25 years + - HbA1c: 4-18% + - FBG: 0-136.5 mmol/l + + Args: + df: Input DataFrame + error_collector: ErrorCollector for tracking violations + + Returns: + DataFrame with range validation applied + """ + # Height: convert cm to m only above 50. A value between 2.3 and 50 is + # neither unit, and dividing it by 100 turns an unusable cell into a + # plausible-looking metre reading. The threshold was 2.3 and 120 source + # cells reading 2.43, 6.9 and 13.0 were being published as 0.069 metres + # (ticket 55); they are now rejected instead. + if "height" in df.columns: + df = df.with_columns( + pl.when(pl.col("height") > 50) + .then(pl.col("height") / 100.0) + .otherwise(pl.col("height")) + .alias("height") + ) + df = cut_numeric_value(df, "height", 0, 2.3, error_collector) + + # Weight: 0-200 kg + if "weight" in df.columns: + df = cut_numeric_value(df, "weight", 0, 200, error_collector) + + # BMI is derived here rather than earlier so it sees the validated height: + # an out-of-bounds height voids the BMI instead of producing one + # from an impossible measurement. + df = _calculate_bmi(df) + + # BMI: 4-60 + if "bmi" in df.columns: + df = cut_numeric_value(df, "bmi", 10, 80, error_collector) + + # Age: 0-25 years + if "age" in df.columns: + df = cut_numeric_value(df, "age", 0, 100, error_collector) + + # HbA1c baseline: 4-18% + if "hba1c_baseline" in df.columns: + df = cut_numeric_value(df, "hba1c_baseline", 0, 25, error_collector) + + # HbA1c updated: 4-18% + if "hba1c_updated" in df.columns: + df = cut_numeric_value(df, "hba1c_updated", 0, 25, error_collector) + + # FBG: the analytical limits of the machines in use, given by A4D's medical + # advisor 2026-08-17 (ticket 42). The permissive end of each range he gave is + # used, so only a reading no machine could have produced is rejected. This + # replaces an inherited 0-150 mmol/L bound that was more than three times + # his ceiling, and covers the three columns that carried no bound at all. + # 832 readings are rejected on these limits. + for column in ("fbg_baseline_mg", "fbg_updated_mg"): + if column in df.columns: + df = cut_numeric_value( + df, column, MG_ANALYTICAL_MIN, MG_ANALYTICAL_MAX, error_collector + ) + for column in ("fbg_baseline_mmol", "fbg_updated_mmol"): + if column in df.columns: + df = cut_numeric_value( + df, column, MMOL_ANALYTICAL_MIN, MMOL_ANALYTICAL_MAX, error_collector + ) + + return df + + +def _apply_unit_conversions(df: pl.DataFrame) -> pl.DataFrame: + """Apply unit conversions. + + - FBG mmol/l ↔ mg/dl conversion (18x factor) + - Only convert if one is missing but the other exists + + Args: + df: Input DataFrame + + Returns: + DataFrame with unit conversions applied + """ + # Convert fbg_updated_mg to mmol if mmol is all NULL + if "fbg_updated_mmol" in df.columns and "fbg_updated_mg" in df.columns: + if df["fbg_updated_mmol"].is_null().all(): + df = df.with_columns( + pl.when(pl.col("fbg_updated_mg") != settings.error_val_numeric) + .then(pl.col("fbg_updated_mg") / 18.0) + .otherwise(None) + .alias("fbg_updated_mmol") + ) + + # Convert fbg_updated_mmol to mg if mg is all NULL + if "fbg_updated_mg" in df.columns and "fbg_updated_mmol" in df.columns: + if df["fbg_updated_mg"].is_null().all(): + df = df.with_columns( + pl.when(pl.col("fbg_updated_mmol") != settings.error_val_numeric) + .then(pl.col("fbg_updated_mmol") * 18.0) + .otherwise(None) + .alias("fbg_updated_mg") + ) + + return df + + +def _fix_age_from_dob(df: pl.DataFrame, error_collector: ErrorCollector) -> pl.DataFrame: + """Fix age by calculating from DOB and tracker date. + + Always uses the age calculated from DOB rather than the sheet's own Age + cell. A typed age is correct only for the month someone typed it; the + date of birth stays correct. LA_MH060's recovered D.O.B. of 2009-01-16 + makes Jan23 an age of 14, where the workbook's own cell still reads 13. + + Logic: + 1. Calculate age: tracker_year - birth_year + 2. Adjust if birthday hasn't occurred yet: if tracker_month < birth_month: age -= 1 + 3. If calculated age differs from Excel age, log warning and use calculated + 4. If calculated age is negative, use error value and log warning + + Args: + df: DataFrame with age, dob, tracker_year, tracker_month, patient_id columns + error_collector: ErrorCollector for tracking data quality issues + + Returns: + DataFrame with corrected age values + + Example: + >>> df = pl.DataFrame({ + ... "patient_id": ["P001"], + ... "age": [21.0], # Wrong value from Excel + ... "dob": [date(2006, 8, 8)], + ... "tracker_year": [2025], + ... "tracker_month": [2] + ... }) + >>> collector = ErrorCollector() + >>> fixed = _fix_age_from_dob(df, collector) + >>> fixed["age"][0] # Should be 18, not 21 + 18.0 + """ + # Only fix if we have the necessary columns + required_cols = ["age", "dob", "tracker_year", "tracker_month", "patient_id"] + if not all(col in df.columns for col in required_cols): + logger.debug("Skipping age fix: missing required columns") + return df + + logger.info("Deriving age from DOB") + + error_date = pl.lit(settings.error_val_date).str.to_date() + + # Only calculate if dob is valid (not null, not error date) + valid_dob = pl.col("dob").is_not_null() & (pl.col("dob") != error_date) + + # Calculate age from DOB + # calc_age = tracker_year - year(dob) + # if tracker_month < month(dob): calc_age -= 1 + df = df.with_columns( + pl.when(valid_dob) + .then( + pl.col("tracker_year") + - pl.col("dob").dt.year() + - pl.when(pl.col("tracker_month") < pl.col("dob").dt.month()).then(1).otherwise(0) + ) + .otherwise(None) + .alias("_calc_age") + ) + + # Track which ages were fixed + ages_fixed = 0 + ages_missing = 0 + ages_negative = 0 + + # For each row where calc_age differs from age, log and fix + for row in df.filter( + pl.col("_calc_age").is_not_null() + & ((pl.col("age").is_null()) | (pl.col("age") != pl.col("_calc_age"))) + ).iter_rows(named=True): + patient_id = row["patient_id"] + file_name = row.get("file_name") or "unknown" + excel_age = row["age"] + calc_age = row["_calc_age"] + + if excel_age is None or (excel_age == settings.error_val_numeric): + logger.bind(error_code="missing_value").warning( + f"Patient {patient_id}: age is missing. " + f"Using calculated age {calc_age} instead of original age." + ) + error_collector.add_error( + file_name=file_name, + patient_id=patient_id, + column="age", + original_value=excel_age if excel_age is not None else "NULL", + error_message=f"Age missing, calculated from DOB as {calc_age}", + error_code="missing_value", + function_name="_fix_age_from_dob", + ) + ages_missing += 1 + elif calc_age < 0: + logger.bind(error_code="invalid_value").warning( + f"Patient {patient_id}: calculated age is negative ({calc_age}). " + f"Please check this manually. Using error value instead." + ) + error_collector.add_error( + file_name=file_name, + patient_id=patient_id, + column="age", + original_value=str(excel_age), + error_message=f"Calculated age is negative ({calc_age}), check DOB", + error_code="invalid_value", + function_name="_fix_age_from_dob", + ) + ages_negative += 1 + else: + logger.bind(error_code="invalid_value").warning( + f"Patient {patient_id}: age {excel_age} is different " + f"from calculated age {calc_age}. " + f"Using calculated age instead of original age." + ) + error_collector.add_error( + file_name=file_name, + patient_id=patient_id, + column="age", + original_value=str(excel_age), + error_message=( + f"Age mismatch: Excel={excel_age}, Calculated={calc_age}. Using calculated age." + ), + error_code="invalid_value", + function_name="_fix_age_from_dob", + ) + ages_fixed += 1 + + # Apply fixes: + # 1. Use calculated age when available and non-negative + # 2. Use error value for negative ages + df = df.with_columns( + pl.when(pl.col("_calc_age").is_not_null()) + .then( + pl.when(pl.col("_calc_age") < 0) + .then(pl.lit(settings.error_val_numeric)) + .otherwise(pl.col("_calc_age")) + ) + .otherwise(pl.col("age")) + .alias("age") + ) + + # Drop temporary column + df = df.drop("_calc_age") + + if ages_fixed > 0 or ages_missing > 0 or ages_negative > 0: + logger.info( + f"Age fixes applied: {ages_fixed} corrected, " + f"{ages_missing} filled from DOB, " + f"{ages_negative} negative (set to error)" + ) + + return df + + +def _fix_t1d_diagnosis_age(df: pl.DataFrame) -> pl.DataFrame: + """Fill t1d_diagnosis_age from dob and t1d_diagnosis_date, but only when + the tracker's own recorded value is missing or an Excel error sentinel. + + A directly recorded diagnosis age is a real clinic-entered value, not + something to discard in favour of date arithmetic. An earlier version + overwrote it whenever both dates parsed, which threw the recorded value + away entirely when a date failed to parse (falling to null) and silently + shifted it by +/-1 when it parsed but disagreed with the tracker's figure. + Deriving only into the gap keeps both: MM_MD010's cell reads `11yr`, which + no arithmetic is needed to read, while a patient with a blank cell and two + good dates still gets an age. + + "Missing" includes the 999999 sentinel, since a cell that failed numeric + conversion recorded no usable age either. + + Args: + df: DataFrame with dob, t1d_diagnosis_date, t1d_diagnosis_age columns + + Returns: + DataFrame with t1d_diagnosis_age filled from dates where missing + """ + required_cols = ["dob", "t1d_diagnosis_date", "t1d_diagnosis_age"] + if not all(col in df.columns for col in required_cols): + return df + + error_date = pl.lit(settings.error_val_date).str.to_date() + + # Only calculate if both dates are valid (not null, not error date) + valid_dob = pl.col("dob").is_not_null() & (pl.col("dob") != error_date) + valid_diagnosis = pl.col("t1d_diagnosis_date").is_not_null() & ( + pl.col("t1d_diagnosis_date") != error_date + ) + # Keep the raw recorded age unless it's missing or an Excel error sentinel + has_recorded_age = pl.col("t1d_diagnosis_age").is_not_null() & ( + pl.col("t1d_diagnosis_age") != settings.error_val_numeric + ) + + # Calculate age at diagnosis: year(diagnosis_date) - year(dob) + # Adjust if birthday hasn't occurred yet in diagnosis year + calculated_age = ( + pl.col("t1d_diagnosis_date").dt.year() + - pl.col("dob").dt.year() + - pl.when(pl.col("t1d_diagnosis_date").dt.month() < pl.col("dob").dt.month()) + .then(1) + .otherwise(0) + ) + + df = df.with_columns( + pl.when(has_recorded_age) + .then(pl.col("t1d_diagnosis_age")) + .when(valid_dob & valid_diagnosis) + # A negative result means the tracker dates contradict each other -- + # diagnosis recorded before birth (ticket 52). The derivation has + # nothing to say about such a row, and emitting the arithmetic put + # impossible ages into production output. + .then(pl.when(calculated_age >= 0).then(calculated_age).otherwise(None)) + .otherwise(None) + .cast(pl.Int32) + .alias("t1d_diagnosis_age") + ) + + return df + + +def _convert_buddhist_era_dates(df: pl.DataFrame, error_collector: ErrorCollector) -> pl.DataFrame: + """Shift Buddhist-era dates to Gregorian before anything else reads them. + + Thai clinics keep their trackers in a Thai-locale Excel, so a date arrives + with a Buddhist-era year (BE = CE + 543) -- the calendar the clinic uses, + not an error it made. Without this step ``_validate_dates`` sees a year + centuries ahead and clobbers the cell with the 9999-09-09 sentinel, so the + reading is destroyed rather than merely published oddly: 381 cells across + 95 distinct values in the 254-tracker corpus (ticket 61). 2022 Hat Yai's + 2560-01-01 converts to the 2017-01-01 that the same patient's D.O.B. and + recruitment date independently give, which is the check that established + the shift is a recovery rather than a rewrite. + + A cell converts when its year is at or beyond ``BUDDHIST_ERA_THRESHOLD`` + and the shifted year is no later than the tracker's own year. There is no + lower bound, unlike the product arm's band: a diagnosis or screening date + legitimately predates its tracker by decades, and the upper bound alone + already rejects what does not decode (``3035`` -> 2492, ``5025`` -> 4482), + leaving those to be sentinelled and reported as source defects. + + Runs before ``_fix_age_from_dob`` so an age is never derived from a BE + ``dob``. Cleaned stage only -- the raw stage keeps what the workbook says. + """ + converting = [c for c in get_date_columns() if c in df.columns and c != "tracker_date"] + converted_cells = 0 + + for col in converting: + shifted = gregorian_from_buddhist(col) + mask = ( + pl.col(col).is_not_null() + & (pl.col(col).dt.year() >= BUDDHIST_ERA_THRESHOLD) + & shifted.is_not_null() + & (shifted <= pl.date(pl.col("tracker_year"), 12, 31)) + ) + + candidates = df.with_columns(shifted.alias("_shifted")).filter(mask) + for patient_id, file_name, original, shifted_value in candidates.select( + "patient_id", "file_name", col, "_shifted" + ).iter_rows(): + error_collector.add_error( + file_name=file_name or "UNKNOWN", + patient_id=patient_id or "UNKNOWN", + column=col, + original_value=str(original), + error_message=( + f"Date {original} is a Buddhist-era year; converted to {shifted_value}" + ), + error_code="buddhist_era_converted", + function_name="_convert_buddhist_era_dates", + ) + converted_cells += 1 + + df = df.with_columns(pl.when(mask).then(shifted).otherwise(pl.col(col)).alias(col)) + + if converted_cells > 0: + logger.info(f"Buddhist-era conversion: {converted_cells} dates shifted to Gregorian") + + return df + + +def _validate_dates(df: pl.DataFrame, error_collector: ErrorCollector) -> pl.DataFrame: + """Validate date columns and replace future dates with error value. + + Dates beyond the tracker year are considered invalid and replaced with + the error date value (9999-09-09). + + A date after the tracker's own year cannot be a record of something that + happened during it. The population is a real source defect rather than a + parsing artifact: 2022 Vietnam National Children's Hospital records every + diagnosis date as a 2023 one, for patients recruited in 2017 (ticket 51, + 722 cells). Sentinelling marks the value unusable rather than inventing a + plausible one; the workbook is what needs correcting, which is why each + rejection is also logged per patient. + + Args: + df: Input DataFrame with date columns + error_collector: ErrorCollector for tracking validation errors + + Returns: + DataFrame with invalid dates replaced + """ + date_columns = get_date_columns() + dates_fixed = 0 + + # Get the error date as a date type + error_date = pl.lit(settings.error_val_date).str.to_date() + + for col in date_columns: + if col not in df.columns: + continue + + # Skip tracker_date as it's derived and shouldn't be validated + if col == "tracker_date": + continue + + # Create a date representing end of tracker year (December 31) + # Find invalid dates and log them + temp_df = df.with_columns(pl.date(pl.col("tracker_year"), 12, 31).alias("_max_valid_date")) + + invalid_dates = temp_df.filter( + pl.col(col).is_not_null() & (pl.col(col) > pl.col("_max_valid_date")) + ) + + # Log each error (tuple-unpack avoids per-row dict construction) + for patient_id, file_name, original_date, tracker_year in invalid_dates.select( + "patient_id", "file_name", col, "tracker_year" + ).iter_rows(): + patient_id = patient_id if patient_id is not None else "UNKNOWN" + file_name = file_name if file_name is not None else "UNKNOWN" + + logger.bind(error_code="invalid_value").warning( + f"Patient {patient_id}: {col} = {original_date} " + f"is beyond tracker year {tracker_year}. " + f"Replacing with error date." + ) + error_collector.add_error( + file_name=file_name, + patient_id=patient_id, + column=col, + original_value=str(original_date), + error_message=f"Date {original_date} is beyond tracker year {tracker_year}", + error_code="invalid_value", + function_name="_validate_dates", + ) + dates_fixed += 1 + + # Replace invalid dates with error date (using inline expression) + df = temp_df.with_columns( + pl.when(pl.col(col).is_not_null() & (pl.col(col) > pl.col("_max_valid_date"))) + .then(error_date) + .otherwise(pl.col(col)) + .alias(col) + ).drop("_max_valid_date") + + if dates_fixed > 0: + logger.info(f"Date validation: {dates_fixed} future dates replaced with error value") + + return df + + +def _add_tracker_date(df: pl.DataFrame) -> pl.DataFrame: + """Create tracker_date from tracker_year and tracker_month. + + Args: + df: Input DataFrame + + Returns: + DataFrame with tracker_date column + """ + if "tracker_year" in df.columns and "tracker_month" in df.columns: + # Parse year-month to date (first day of month) + # Cast to string first since they're now Int32 + df = df.with_columns( + pl.concat_str( + [ + pl.col("tracker_year").cast(pl.String), + pl.lit("-"), + pl.col("tracker_month").cast(pl.String), + pl.lit("-01"), + ] + ) + .str.to_date("%Y-%m-%d") + .alias("tracker_date") + ) + + return df + + +def clean_patient_file( + raw_parquet_path: Path, + output_parquet_path: Path, + error_collector: ErrorCollector | None = None, +) -> None: + """Clean a single patient data parquet file. + + This is the main entry point for cleaning a tracker file. + + Args: + raw_parquet_path: Path to raw patient parquet (from extraction) + output_parquet_path: Path to write cleaned parquet + error_collector: Optional ErrorCollector (creates new one if not provided) + + Example: + >>> from pathlib import Path + >>> raw_path = Path("output/patient_data_raw/2024_Hospital_patient_raw.parquet") + >>> clean_path = Path("output/patient_data_clean/2024_Hospital_patient_clean.parquet") + >>> clean_patient_file(raw_path, clean_path) + """ + if error_collector is None: + error_collector = ErrorCollector() + + logger.info(f"Cleaning patient file: {raw_parquet_path}") + + # Read raw parquet + df_raw = pl.read_parquet(raw_parquet_path) + + # Clean data + df_clean = clean_patient_data(df_raw, error_collector) + + # Create output directory if needed + output_parquet_path.parent.mkdir(parents=True, exist_ok=True) + + # Write cleaned parquet + df_clean.write_parquet(output_parquet_path) + + logger.info(f"Cleaned patient file written: {output_parquet_path}") + logger.info(f"Total errors: {len(error_collector)}") diff --git a/src/a4d/clean/product.py b/src/a4d/clean/product.py new file mode 100644 index 0000000..8c7d3f9 --- /dev/null +++ b/src/a4d/clean/product.py @@ -0,0 +1,1228 @@ +"""Product data cleaning pipeline. + +Mirrors ``clean/patient.py`` architecture: a single orchestrator +(``clean_product_data``) dispatches to step-scoped private helpers. + +The ``2.x`` step numbers below are this pipeline's own ordering vocabulary and +are referenced by the tests. They are not decorative: several steps are only +correct in position -- dates are parsed (2.6) before rows are sorted (2.7), +because the sort is chronological; the balance is accumulated (2.15) only after +that sort; and units are recoded to zero both before and after schema seeding +(2.9/2.12) because a column the schema adds arrives null. +""" + +from pathlib import Path + +import polars as pl + +from a4d.clean.buddhist_era import ( + BUDDHIST_ERA_OFFSET, + BUDDHIST_ERA_THRESHOLD, + gregorian_from_buddhist, +) +from a4d.clean.converters import ( + normalize_excel_formula_errors, + parse_date_column, + safe_convert_column, +) +from a4d.clean.schema_product import apply_schema, get_product_data_schema, get_string_columns +from a4d.config import settings +from a4d.errors import ErrorCollector +from a4d.reference.products import load_known_products, load_product_categories + +ACTIVITY_COLS: tuple[str, ...] = ( + "product_entry_date", + "product_units_received", + "product_received_from", + "product_units_released", + "product_released_to", + "product_units_returned", + "product_returned_by", +) + +UNIT_COLS: tuple[str, ...] = ( + "product_units_received", + "product_units_released", + "product_units_returned", +) + +EMPTY_ROW_COLS: tuple[str, ...] = ( + "product_units_received", + "product_units_released", + "product_units_returned", + "product_released_to", + "product_entry_date", + "product_balance", +) + +# Lower-bound year guard for _validate_entry_dates. A parsed Gregorian +# year more than YEAR_FLOOR_DELTA years before the tracker's calendar +# year is implausible (start-balance backfill is months-to-a-few-years, +# not decades) and almost certainly a tiny-int Excel-serial mis-coercion +# (e.g. a raw cell holding `29` parsing to 1900-01-29). +YEAR_FLOOR_DELTA: int = 5 + +# End-of-block summary-row residue scrub for product_entry_date. Two flavours +# observed in the corpus, both junk written into the date column between +# product sub-blocks: (1) literal marker strings, (2) tiny ints typed as a +# day-of-month into the wrong row (e.g. "30" in 2025 NPH row 81). Nulled +# before parse_date_flexible runs so the cleaned output is NULL rather than +# 9999-09-09 (parse-failure sentinel) or 1900-01-29 (Excel-leap-year-bug +# artefact). Product-only — patient pipeline must stay untouched. +PRODUCT_DATE_NA_MARKERS: frozenset[str] = frozenset({"amount left"}) + +# Excel serial for 2000-01-01. A4D trackers are 2017+, so a strict floor would +# be 42736 (2017-01-01); 36526 is deliberately looser to leave a ~17-year buffer +# that _validate_entry_dates handles via YEAR_FLOOR_DELTA. The scrub here only +# needs to catch unambiguous residue (raw day-of-month ints landing as +# 1900-0X-XX via Excel's leap-year bug); legitimate-looking but out-of-range +# serials are left for the year-floor check so they show up in the audit log. +MIN_PLAUSIBLE_EXCEL_SERIAL: int = 36526 + + +def clean_product_data( + df_raw: pl.DataFrame, + error_collector: ErrorCollector, +) -> pl.DataFrame: + """Clean raw product data through the full 2.x step sequence. + + Executes steps 2.1-2.22 in order and returns a DataFrame conforming to + the product meta schema defined in ``clean/schema_product.py``. + + Args: + df_raw: Raw product DataFrame from extraction. + error_collector: Accumulator for row-level data quality errors. + + Returns: + Cleaned product DataFrame. + """ + if df_raw.width == 0: + # No product section found in any sheet (e.g. pre-product-tracking + # tracker years) -- extraction hands back a columnless DataFrame. + # The step sequence below assumes at least a "product" column, so + # short-circuit to an empty-but-schema-conformant result rather than + # letting `apply_schema` synthesize a single all-null row from a + # literal with nothing to broadcast against. + return pl.DataFrame(schema=get_product_data_schema()) + + # Null out (and log) the source trackers' own formula-error strings, + # which extraction preserves verbatim (ticket 27). Runs before the step + # sequence so no step sees a "#DIV/0!" where it expects a value. + df_raw = normalize_excel_formula_errors(df_raw, error_collector, patient_id_col="product") + + df = _normalize_empty_strings_to_null(df_raw) # 2.0 (see helper docstring) + df = _split_multi_product_cells(df) # 2.1 + df = _switch_misplaced_columns(df) # 2.3 + df = _remove_uninformative_rows(df) # 2.4 + df = _add_row_index(df) # 2.5 + df = _format_dates(df, error_collector) # 2.6 + _check_entry_dates_match_sheet(df, error_collector) # 2.6a (log only) + df = _validate_entry_dates(df, error_collector) # 2.6b + df = _fill_product_names_and_sort(df) # 2.7 + df = _extract_balance_from_received(df) # 2.8 + df = _recode_na_units_to_zero(df) # 2.9 + df = _clean_received_from(df) # 2.10 + df = _clean_units_received(df, error_collector) # 2.11 + df = _recode_na_units_to_zero(df) # 2.12 + df = _remove_empty_data_rows(df) # 2.13 + df = _compute_balance_status(df) # 2.14 + df = _compute_running_balance(df, error_collector) # 2.15 + + # 2.16 — type cast numeric/date columns via ErrorCollector; strip strings, + # because end-whitespace never carries meaning here and an untrimmed value + # fails allowed-value validation and lands on a sentinel. + # No string->Int intermediate needed (cf. patient pipeline's Int32-via-Float64 + # path): product unit columns are Float64 by schema, and the only Int columns + # (product_table_year/month) arrive from extraction as numeric, not strings. + schema = get_product_data_schema() + cast_targets = (pl.Int32, pl.Int64, pl.Float32, pl.Float64, pl.Date) + for col, target in schema.items(): + if col not in df.columns or target not in cast_targets: + continue + if df.schema[col] == target: + continue + df = safe_convert_column(df, col, target, error_collector) + + string_cols = [c for c in get_string_columns() if c in df.columns] + if string_cols: + df = df.with_columns([pl.col(c).str.strip_chars() for c in string_cols]) + + # 2.17 — drop helper index column added in 2.5. + if "index" in df.columns: + df = df.drop("index") + + df = _validate_negative_balances(df, error_collector) # 2.18 + df = _report_unknown_products(df, error_collector) # 2.19 + df = _add_product_categories(df) # 2.20 + df = _extract_unit_capacity(df) # 2.21 + # 2.22 cross-month combine happens at the table stage (S4-T1), not here. + + # Final schema conformance: guarantees 20 columns in schema order. + df = apply_schema(df) + # UNIT_COLS treat absence as 0 -- a stock movement that records no + # quantity moved nothing -- so re-run the recode after schema seeding, + # which fills any newly-added column with null. + return _recode_na_units_to_zero(df) + + +def clean_product_file( + raw_parquet_path: Path, + output_parquet_path: Path, + error_collector: ErrorCollector | None = None, +) -> None: + """Clean a single product parquet file (I/O wrapper). + + Reads the raw parquet, runs ``clean_product_data``, and writes the result + to the output path. Mirrors ``clean_patient_file``. + + Args: + raw_parquet_path: Raw product parquet produced by extraction. + output_parquet_path: Destination for the cleaned parquet. + error_collector: Optional ErrorCollector; a new one is created if None. + """ + ec = error_collector if error_collector is not None else ErrorCollector() + df_raw = pl.read_parquet(raw_parquet_path) + df_clean = clean_product_data(df_raw, ec) + df_clean.write_parquet(output_parquet_path) + + +def _normalize_empty_strings_to_null(df: pl.DataFrame) -> pl.DataFrame: + """Step 2.0 — coerce "" / whitespace-only string cells to null. + + openpyxl returns an empty string for a blank Excel cell rather than None, + so without this normalisation a row whose only content is that empty string + survives `_remove_uninformative_rows` and `_remove_empty_data_rows` and + reaches the output as a phantom stock movement (observed on 2024 CDA Dec24, + which has one such row in product_units_received). + """ + exprs = [ + pl.when(pl.col(c).str.strip_chars() == "").then(None).otherwise(pl.col(c)).alias(c) + for c, dtype in df.schema.items() + if dtype == pl.String + ] + return df.with_columns(exprs) if exprs else df + + +def _split_multi_product_cells(df: pl.DataFrame) -> pl.DataFrame: + """Step 2.1 — split cells containing multiple products. + + Splits the ``product`` column on "; " and " and ". Parenthesized unit + hints (e.g. "Product A (1 box)") are extracted into ``product_units_notes`` + and the numeric quantity is routed to ``product_units_received`` or + ``product_units_released`` based on which side has a counterparty. + """ + if "product" not in df.columns: + return df + + df = df.with_columns( + pl.col("product").cast(pl.Utf8).str.replace_all(" and ", "; ").alias("product") + ) + df = df.with_columns(pl.col("product").str.split("; ")).explode("product") + + if "product_units_notes" not in df.columns: + df = df.with_columns(pl.lit(None, dtype=pl.Utf8).alias("product_units_notes")) + else: + df = df.with_columns(pl.col("product_units_notes").cast(pl.Utf8)) + + for col in ("product_units_received", "product_units_released"): + if col in df.columns: + df = df.with_columns(pl.col(col).cast(pl.Utf8)) + else: + df = df.with_columns(pl.lit(None, dtype=pl.Utf8).alias(col)) + + product = pl.col("product").cast(pl.Utf8) + has_paren_and_kw = ( + product.str.contains(r"\(") + & product.str.contains(r"\)") + & (product.str.contains("box") | product.str.contains("unit")) + ) + no_slash = ~product.str.contains("/") + + paren_text = product.str.extract(r"\(([^()]+)\)", 1) + # "\d+" rather than "[1-9]+": the latter drops the leading digit of any + # count containing a zero, turning "(10 box)" into 1. No tracker row in the + # current corpus triggers box/unit extraction, so this is a correctness + # guard rather than an observed fix. + number_str = paren_text.str.extract(r"(\d+)", 1) + + df = df.with_columns( + pl.when(has_paren_and_kw) + .then(paren_text) + .otherwise(pl.col("product_units_notes")) + .alias("product_units_notes") + ) + + if "product_received_from" in df.columns: + df = df.with_columns( + pl.when(has_paren_and_kw & no_slash & pl.col("product_received_from").is_not_null()) + .then(number_str) + .otherwise(pl.col("product_units_received")) + .alias("product_units_received") + ) + + if "product_released_to" in df.columns: + df = df.with_columns( + pl.when(has_paren_and_kw & no_slash & pl.col("product_released_to").is_not_null()) + .then(number_str) + .otherwise(pl.col("product_units_released")) + .alias("product_units_released") + ) + + return df + + +def _switch_misplaced_columns(df: pl.DataFrame) -> pl.DataFrame: + """Step 2.3 — swap ``product_units_received`` with ``product_received_from`` + for each sheet that contains "Remaining Stock" in units_received. + + Observed in the 2018 PNG Nov/Dec trackers. The swap is scoped with + .over(product_sheet_name) because this pipeline operates on the whole-file + DataFrame: the defect is per-sheet, and an unscoped swap would corrupt the + correctly-laid-out sheets in the same workbook. + """ + if ( + "product_units_received" not in df.columns + or "product_received_from" not in df.columns + or "product_sheet_name" not in df.columns + ): + return df + + has_remaining_in_sheet = ( + pl.col("product_units_received") + .cast(pl.Utf8) + .str.contains("Remaining Stock") + .any() + .over("product_sheet_name") + ) + + if not df.select(has_remaining_in_sheet.any()).item(): + return df + + return df.with_columns( + pl.when(has_remaining_in_sheet) + .then(pl.col("product_received_from")) + .otherwise(pl.col("product_units_received")) + .alias("product_units_received"), + pl.when(has_remaining_in_sheet) + .then(pl.col("product_units_received")) + .otherwise(pl.col("product_received_from")) + .alias("product_received_from"), + ) + + +def _remove_uninformative_rows(df: pl.DataFrame) -> pl.DataFrame: + """Step 2.4 — drop rows where every activity column is null.""" + existing = [c for c in ACTIVITY_COLS if c in df.columns] + if not existing: + return df + all_null = pl.all_horizontal([pl.col(c).is_null() for c in existing]) + return df.filter(~all_null) + + +def _add_row_index(df: pl.DataFrame) -> pl.DataFrame: + """Step 2.5 — add a 1-based ``index`` column recording input row order. + + Kept because several later steps sort the frame, and this is the only + surviving record of the order the workbook actually listed rows in. + """ + return df.with_row_index("index", offset=1) + + +def _null_entry_date_residues(df: pl.DataFrame) -> pl.DataFrame: + """Pre-clean for step 2.6 — null end-of-block summary-row residue cells. + + Two patterns observed between product sub-blocks in tracker source files, + both nulled before any date parsing runs: + + 1. Marker strings (case-insensitive, whitespace-trimmed) listed in + ``PRODUCT_DATE_NA_MARKERS``. Currently only ``"Amount Left"`` — + observed in 2018 Mahosot Nov18/Dec18, six per sheet between sub-blocks. + Add more only after a corpus sweep finds them. + 2. Tiny Excel serials below ``MIN_PLAUSIBLE_EXCEL_SERIAL`` (2000-01-01). + Catches stray day-of-month integers typed into the wrong row + (e.g. raw ``30`` in 2025 NPH row 81 → 1900-01-29 via the pre-Mar-1900 + leap-year-bug epoch). + + Without this scrub the residue surfaces in cleaned output as either + ``9999-09-09`` (parse-failure sentinel for "Amount Left") or + ``1900-01-29`` (Excel-leap-year-bug artefact for tiny serials). Both + are junk: neither is a date the clinic recorded, and both would otherwise + be published as though it had. + """ + if "product_entry_date" not in df.columns: + return df + + raw = pl.col("product_entry_date").cast(pl.Utf8).str.strip_chars() + is_marker = raw.str.to_lowercase().is_in(list(PRODUCT_DATE_NA_MARKERS)) + as_num = raw.cast(pl.Float64, strict=False) + is_tiny_serial = as_num.is_not_null() & (as_num > 0) & (as_num < MIN_PLAUSIBLE_EXCEL_SERIAL) + + return df.with_columns( + pl.when(is_marker | is_tiny_serial) + .then(pl.lit(None, dtype=pl.Utf8)) + .otherwise(pl.col("product_entry_date").cast(pl.Utf8)) + .alias("product_entry_date") + ) + + +def _format_dates(df: pl.DataFrame, error_collector: ErrorCollector) -> pl.DataFrame: + """Step 2.6 — parse ``product_entry_date`` with the flexible date parser. + + Three preprocessing steps run before delegating to ``parse_date_column``: + + 1. Null end-of-block summary-row residue (marker strings and + tiny-int Excel serials) via ``_null_entry_date_residues``, so they + become NULL in the cleaned output instead of the parse-failure + sentinel or 1900-01-29 Excel-leap-year artefacts. + 2. Strip a trailing time component (``" HH:MM[:SS]"``) from Excel + datetimes cast to string. Uses a precise regex rather than splitting + on the first space, so date strings that legitimately contain + spaces (e.g. ``"24 Feb 2020"``) survive intact. + 3. Normalize separator typos observed in the corpus (2026-04-29): + ``"--"`` → ``"-"``, period-then-letter → space, underscore → space. + Each rule is unambiguous and does not collide with valid date + formats. Product-scoped only; ``parse_date_flexible`` is shared + with the patient pipeline and is not modified. + """ + if "product_entry_date" not in df.columns: + return df + if df.schema["product_entry_date"] == pl.Date: + return df + df = _null_entry_date_residues(df) + df = df.with_columns( + pl.col("product_entry_date") + .cast(pl.Utf8) + .str.replace(r"\s+\d{1,2}:\d{2}(:\d{2})?$", "") + .str.replace_all(r"-{2,}", "-") + .str.replace_all(r"(\d)\.([A-Za-z])", r"$1 $2") + .str.replace_all(r"_", " ") + .alias("product_entry_date") + ) + return parse_date_column(df, "product_entry_date", error_collector, patient_id_col="product") + + +def _check_entry_dates_match_sheet(df: pl.DataFrame, error_collector: ErrorCollector) -> None: + """Warn on entry dates that disagree with the sheet they were found on. + + One log entry per row where the parsed ``product_entry_date`` doesn't match + ``(product_table_year, product_table_month)``. A movement filed on the + March sheet but dated in July is either a typo or a misfiled row, and + either way the workbook is what needs correcting -- so this reports rather + than repairs. Sentinels, nulls, and Buddhist-era dates are skipped, since + each is already accounted for elsewhere. + + Every parsed date is checked, not just numeric Excel serials, because + ``parse_date_flexible`` accepts both serials and text and a mis-dated text + cell is no less wrong than a mis-dated serial. + + Side-effecting only: pushes log entries; never mutates ``df``. + """ + needed = {"product_entry_date", "product_table_year", "product_table_month"} + if not needed.issubset(df.columns): + return + + error_date = pl.lit(settings.error_val_date).str.to_date() + table_year = pl.col("product_table_year").cast(pl.Int32, strict=False) + table_month = pl.col("product_table_month").cast(pl.Int32, strict=False) + + is_real_date = ( + pl.col("product_entry_date").is_not_null() + & (pl.col("product_entry_date") != error_date) + & (pl.col("product_entry_date").dt.year() < BUDDHIST_ERA_THRESHOLD) + ) + mismatch = (pl.col("product_entry_date").dt.month() != table_month) | ( + pl.col("product_entry_date").dt.year() != table_year + ) + + select_cols = [ + "file_name" if "file_name" in df.columns else pl.lit("unknown").alias("file_name"), + "product" if "product" in df.columns else pl.lit("unknown").alias("product"), + "product_entry_date", + table_year.alias("_table_year"), + table_month.alias("_table_month"), + "product_sheet_name" + if "product_sheet_name" in df.columns + else pl.lit("unknown").alias("product_sheet_name"), + ] + offenders = df.filter(is_real_date & mismatch).select(select_cols) + + for file_name, product, entry_date, ty, tm, sheet_name in offenders.iter_rows(): + error_collector.add_error( + file_name=file_name or "unknown", + patient_id=product or "unknown", + column="product_entry_date", + original_value=str(entry_date), + error_message=( + f"product_entry_date {entry_date} does not match sheet " + f"'{sheet_name or 'unknown'}' (expected {ty}-{tm:02d})" + ), + error_code="invalid_value", + function_name="check_entry_dates", + ) + + +def _validate_entry_dates(df: pl.DataFrame, error_collector: ErrorCollector) -> pl.DataFrame: + """Step 2.6b — flag fat-fingered Gregorian entry dates outside the tracker window. + + A row is flagged when its parsed Gregorian year falls outside + ``[product_table_year - YEAR_FLOOR_DELTA, product_table_year]`` and is below + ``BUDDHIST_ERA_THRESHOLD``. Two divergence patterns are caught, handled + asymmetrically: + + * **Above-max** (future-year typos, e.g. ``2099-03-15`` in a 2024 tracker): + logged AND replaced with ``error_val_date`` (9999-09-09). Future dates are + genuinely ambiguous (premature next-month entry vs typo); the sentinel + preserves that uncertainty signal in the output. + * **Below-min** (year-floor, e.g. ``1967-02-05`` in a 2024 tracker, or raw + cell ``29`` → 1900-01-29 from Excel-serial mis-coercion): logged ONLY; + the parsed date is preserved. Year-floor cells are unambiguously bad data + (1900-2014 in 2020+ trackers), but sentinelling them would drop the row + out of chronological order and so distort the running balance that step + 2.15 accumulates. Preserving the date keeps the ledger trajectory intact + while the audit log retains the data-quality flag. + + Above/below cases are logged with distinct messages so triage in + ``table_error_messages.parquet`` can distinguish them. + + * **Implausible era** (year at or beyond ``BUDDHIST_ERA_THRESHOLD`` that is + not this tracker's own Buddhist-era year): logged under + ``implausible_era_date`` AND replaced with ``error_val_date``. Ticket 32 + found the earlier blanket ">= 2400 is exempt" rule published + ``5567-08-19`` and ``3026-04-30`` — Excel serials 1,339,576 and 411,384 — + straight into the product table, plus a Hat Yai ``2525`` in a tracker + whose BE year is 2568. + * **Buddhist era** (year inside this tracker's own BE band): shifted back + 543 years and logged under ``buddhist_era_converted`` (ticket 61). A BE + date is the calendar a Thai clinic uses, not an error it made, so the + pipeline publishes one calendar downstream rather than a stock movement + dated 543 years in the future -- 22 product rows. A BE leap day that has + no Gregorian counterpart + (2568-02-29 -> 2025-02-29) yields null from the shift and falls through + to the implausible-era branch rather than being invented. + + The parse-failure sentinel (9999-09-09) is exempt from all branches so it is + not re-clobbered or double-logged. + """ + if "product_entry_date" not in df.columns or "product_table_year" not in df.columns: + return df + + error_date = pl.lit(settings.error_val_date).str.to_date() + table_year = pl.col("product_table_year").cast(pl.Int32) + max_valid = pl.date(table_year, 12, 31) + min_valid = pl.date(table_year - YEAR_FLOOR_DELTA, 1, 1) + + entry_year = pl.col("product_entry_date").dt.year() + # A year at or beyond the threshold is only credible as this tracker's own + # Buddhist-era year, so the exemption mirrors the Gregorian window rather + # than admitting everything above 2400 (ticket 32). + in_buddhist_band = (entry_year >= table_year + BUDDHIST_ERA_OFFSET - YEAR_FLOOR_DELTA) & ( + entry_year <= table_year + BUDDHIST_ERA_OFFSET + ) + is_sentinel = pl.col("product_entry_date") == error_date + converted = gregorian_from_buddhist("product_entry_date") + convert_mask = ( + pl.col("product_entry_date").is_not_null() + & (entry_year >= BUDDHIST_ERA_THRESHOLD) + & in_buddhist_band + & ~is_sentinel + & converted.is_not_null() + ) + implausible_era_mask = ( + pl.col("product_entry_date").is_not_null() + & (entry_year >= BUDDHIST_ERA_THRESHOLD) + & ~convert_mask + & ~is_sentinel + ) + + convertible = df.filter(convert_mask).select( + "file_name", "product", "product_entry_date", "product_table_year", "product_sheet_name" + ) + for file_name, product, entry_date, table_year_val, sheet_name in convertible.iter_rows(): + error_collector.add_error( + file_name=file_name or "unknown", + patient_id=product or "unknown", + column="product_entry_date", + original_value=str(entry_date), + error_message=( + f"product_entry_date {entry_date} is the Buddhist-era year of " + f"product_table_year {table_year_val}; converted to " + f"{entry_date.year - BUDDHIST_ERA_OFFSET}-{entry_date.month:02d}-" + f"{entry_date.day:02d} (sheet '{sheet_name or 'unknown'}')" + ), + error_code="buddhist_era_converted", + function_name="_validate_entry_dates", + ) + + implausible = df.filter(implausible_era_mask).select( + "file_name", "product", "product_entry_date", "product_table_year", "product_sheet_name" + ) + for file_name, product, entry_date, table_year_val, sheet_name in implausible.iter_rows(): + error_collector.add_error( + file_name=file_name or "unknown", + patient_id=product or "unknown", + column="product_entry_date", + original_value=str(entry_date), + error_message=( + f"product_entry_date {entry_date} is neither a Gregorian date nor " + f"the Buddhist-era year of product_table_year {table_year_val} " + f"(sheet '{sheet_name or 'unknown'}')" + ), + error_code="implausible_era_date", + function_name="_validate_entry_dates", + ) + + not_buddhist = entry_year < BUDDHIST_ERA_THRESHOLD + above_max_mask = ( + pl.col("product_entry_date").is_not_null() + & (pl.col("product_entry_date") > max_valid) + & not_buddhist + ) + below_min_mask = ( + pl.col("product_entry_date").is_not_null() + & (pl.col("product_entry_date") < min_valid) + & not_buddhist + ) + + above = df.filter(above_max_mask).select( + "file_name", "product", "product_entry_date", "product_table_year", "product_sheet_name" + ) + for file_name, product, entry_date, table_year_val, sheet_name in above.iter_rows(): + error_collector.add_error( + file_name=file_name or "unknown", + patient_id=product or "unknown", + column="product_entry_date", + original_value=str(entry_date), + error_message=( + f"product_entry_date {entry_date} beyond " + f"product_table_year {table_year_val} " + f"(sheet '{sheet_name or 'unknown'}')" + ), + error_code="invalid_value", + function_name="_validate_entry_dates", + ) + + below = df.filter(below_min_mask).select( + "file_name", "product", "product_entry_date", "product_table_year", "product_sheet_name" + ) + for file_name, product, entry_date, table_year_val, sheet_name in below.iter_rows(): + error_collector.add_error( + file_name=file_name or "unknown", + patient_id=product or "unknown", + column="product_entry_date", + original_value=str(entry_date), + error_message=( + f"product_entry_date {entry_date} before " + f"product_table_year {table_year_val} - {YEAR_FLOOR_DELTA} " + f"(sheet '{sheet_name or 'unknown'}')" + ), + error_code="invalid_value", + function_name="_validate_entry_dates", + ) + + return df.with_columns( + pl.when(above_max_mask | implausible_era_mask) + .then(error_date) + .when(convert_mask) + .then(converted) + .otherwise(pl.col("product_entry_date")) + .alias("product_entry_date") + ) + + +def _fill_product_names_and_sort(df: pl.DataFrame) -> pl.DataFrame: + """Step 2.7 — forward-fill ``product`` then sort inside each product group. + + Forward-fill and the first/last tier windows are scoped to + (product, product_sheet_name). Without that scoping a null product at the + top of sheet N picks up the last product of sheet N-1, and a single product + appearing in several sheets collapses to one start/end pair instead of one + per sheet. + + Within each (sheet, product) group the rank expression is: + rank = 1 if first row in group + = n + 2 if last row in group + = row_number if middle row with null date (preserves input order) + = dense_rank(date) + 1 if middle row with valid date + Stable sort on (product_table_month, product, _rank) gives sheet-major, + product-minor order, which is how the ledger reads in the workbook. + + The parse-failure sentinel (``settings.error_val_date`` = 9999-09-09) + counts as null for rank purposes: a row whose date could not be read has + no place in a chronological ordering, so it holds its input position. + Without this it sorts to the dense_rank+1 end-of-changes slot and the + running balance accumulates in the wrong order. + """ + if "product" not in df.columns: + return df + if "index" not in df.columns: + raise KeyError( + "_fill_product_names_and_sort requires an 'index' column; run _add_row_index first" + ) + + group = ["product_sheet_name", "product"] if "product_sheet_name" in df.columns else ["product"] + sheet_group = ["product_sheet_name"] if "product_sheet_name" in df.columns else None + + if sheet_group is not None: + df = df.with_columns(pl.col("product").forward_fill().over(sheet_group)) + else: + df = df.with_columns(pl.col("product").forward_fill()) + + row_n = pl.col("index").rank("ordinal").over(group).cast(pl.Int64) + group_n = pl.col("index").count().over(group).cast(pl.Int64) + has_date = "product_entry_date" in df.columns + dense_d = ( + pl.col("product_entry_date").rank("dense").over(group).cast(pl.Int64) + if has_date + else pl.lit(None, dtype=pl.Int64) + ) + date_is_null = ( + ( + pl.col("product_entry_date").is_null() + | (pl.col("product_entry_date") == pl.lit(settings.error_val_date).str.to_date()) + ) + if has_date + else pl.lit(True) + ) + + rank_expr = ( + pl.when(row_n == 1) + .then(pl.lit(1, dtype=pl.Int64)) + .when(row_n == group_n) + .then(group_n + 2) + .when(date_is_null) + .then(row_n) + .otherwise(dense_d + 1) + .alias("_rank") + ) + + sort_cols: list[str] = [] + if "product_table_month" in df.columns: + sort_cols.append("product_table_month") + sort_cols.extend(["product", "_rank"]) + + return ( + df.with_columns(rank_expr) + .sort(sort_cols, nulls_last=True, maintain_order=True) + .drop("_rank") + ) + + +def _extract_balance_from_received(df: pl.DataFrame) -> pl.DataFrame: + """Step 2.8 — move stray balance values out of ``product_units_released``. + + For trackers (e.g. 2019 PKH, 2020 STH) where "Balance" appears in + ``product_units_received``, the value sitting in ``product_units_released`` + is relocated to ``product_received_from`` and released is cleared. + + The trigger is scoped with .over("product_sheet_name") because this + pipeline operates on the whole-file DataFrame and the convention is + per-sheet: an unscoped trigger blanks ``product_received_from`` on the + clean sheets sharing the workbook. + + Rows on a triggered sheet that are *not* Balance rows keep their + ``product_received_from`` via the ``.otherwise`` arm. This matters because + the trigger regex ``(?i)Balance`` substring-matches "START BALANCE" and + "END BALANCE", which appear on every standard sheet -- so a rule without a + default arm silently wipes legitimate supplier names (``DKSH`` on the + Mahosot 2020 stock-receipt rows). Mutation of ``product_units_released`` is + correspondingly scoped to actual Balance rows, so preserving + ``received_from`` on a non-Balance row does not leave a stray + ``released`` value beside it. + """ + required = ( + "product_units_received", + "product_units_released", + "product_received_from", + "product_sheet_name", + ) + if not all(c in df.columns for c in required): + return df + + balance_mask = pl.col("product_units_received").cast(pl.Utf8).str.contains("(?i)Balance") + sheet_triggered = balance_mask.any().over("product_sheet_name") & pl.col( + "product_received_from" + ).is_null().any().over("product_sheet_name") + + if not df.select(sheet_triggered.any()).item(): + return df + + # Materialize the trigger predicates before mutating product_received_from. + # `sheet_triggered` includes `received_from.is_null().any().over(sheet)`; if + # we let it re-evaluate in the second .with_columns() below, sheets where + # every row is a Balance marker (e.g. 2019 PKH Oct19) flip the trigger to + # False after the first call populates received_from on every row, and the + # released-clear pass becomes a no-op. Caching pins the pre-mutation truth. + df = df.with_columns( + sheet_triggered.alias("_sheet_triggered"), + balance_mask.alias("_balance_mask"), + ) + + # Two positive arms relocate the balance value into received_from on + # actual Balance-marker rows; the catch-all preserves received_from on + # all other rows -- see docstring for why the catch-all is required. + # The "Total" arm before the catch-all nulls + # typist subtotal labels (e.g. "Accu-Chek Performa | Total | 35" + # subtotal rows in Penang DC / VNCH / Mandalay 2019 trackers). "Total" is + # never a real supplier -- it is the label the typist put on the + # end-of-product-block subtotal row. + df = df.with_columns( + pl.when( + pl.col("_sheet_triggered") + & pl.col("_balance_mask") + & pl.col("product_units_released").is_not_null() + ) + .then(pl.col("product_units_released").cast(pl.Utf8)) + .when( + pl.col("_sheet_triggered") + & pl.col("_balance_mask") + & pl.col("product_received_from").is_not_null() + ) + .then(pl.col("product_received_from").cast(pl.Utf8)) + .when(pl.col("product_received_from") == "Total") + .then(pl.lit(None, dtype=pl.Utf8)) + .otherwise(pl.col("product_received_from")) + .alias("product_received_from") + ) + df = df.with_columns( + pl.when( + pl.col("_sheet_triggered") + & pl.col("_balance_mask") + & pl.col("product_received_from").is_not_null() + ) + .then(pl.lit(None, dtype=pl.Utf8)) + .otherwise(pl.col("product_units_released")) + .alias("product_units_released") + ) + return df.drop("_sheet_triggered", "_balance_mask") + + +def _recode_na_units_to_zero(df: pl.DataFrame) -> pl.DataFrame: + """Steps 2.9 and 2.12 — fill NA in unit columns with 0. + + Applied twice: once before the string cleans in 2.10/2.11 and once after + to catch nulls introduced by those cleans. Respects column dtype — Utf8 + cols get "0", numeric cols get 0. + """ + for col in UNIT_COLS: + if col not in df.columns: + continue + fill: float | str = 0 if df.schema[col].is_numeric() else "0" + df = df.with_columns(pl.col(col).fill_null(fill)) + return df + + +def _clean_received_from(df: pl.DataFrame) -> pl.DataFrame: + """Step 2.10 — normalise the ``product_received_from`` column. + + If ``product_units_received`` contains "START", the corresponding + ``product_received_from`` value is copied to ``product_balance`` as the + start balance. Purely numeric entries (should be supplier names) are + removed. + """ + if "product_received_from" not in df.columns: + return df + + if "product_units_received" in df.columns: + # Preserve existing product_balance on non-START rows. Earlier code + # used .otherwise(pl.lit(None)) which only happened to be a no-op + # because nothing populates product_balance before this step; if a + # future step seeds it, that work would be silently wiped. + # .over("product_sheet_name") scopes this per sheet, matching how the + # convention is applied in the workbooks -- defensive here, since the + # row-wise when/then/otherwise has no cross-row dependency today. + if "product_balance" not in df.columns: + df = df.with_columns(pl.lit(None, dtype=pl.Float64).alias("product_balance")) + start_mask = pl.col("product_units_received").cast(pl.Utf8).str.contains("(?i)START") + df = df.with_columns( + pl.when(start_mask) + .then(pl.col("product_received_from").cast(pl.Utf8)) + .otherwise(pl.col("product_balance").cast(pl.Utf8)) + .over("product_sheet_name") + .alias("product_balance") + ) + + no_alpha = ~pl.col("product_received_from").cast(pl.Utf8).str.contains(r"[A-Za-z]") + df = df.with_columns( + pl.when(pl.col("product_received_from").is_not_null() & no_alpha) + .then(pl.lit(None)) + .otherwise(pl.col("product_received_from")) + .alias("product_received_from") + ) + return df + + +def _clean_units_received( + df: pl.DataFrame, + error_collector: ErrorCollector, +) -> pl.DataFrame: + """Step 2.11 — zero out balance markers and cast to numeric. + + Rows where ``product_units_received`` contains "START", "END" or + "BALANCE" are set to 0. Remaining values are cast to numeric; failures + yield null (then 0 via the second pass of step 2.12) and emit one + ``type_conversion`` entry per row to ``error_collector``, so a quantity + that could not be read is recoverable from the log rather than silently + becoming a zero movement. + """ + if "product_units_received" not in df.columns: + return df + + marker_mask = ( + pl.col("product_units_received").cast(pl.Utf8).str.contains("(?i)START|END|BALANCE") + ) + casted = pl.col("product_units_received").cast(pl.Float64, strict=False) + + failure_mask = pl.col("product_units_received").is_not_null() & ~marker_mask & casted.is_null() + failures = df.filter(failure_mask) + for row in failures.iter_rows(named=True): + original = row["product_units_received"] + error_collector.add_error( + file_name=row.get("file_name") or "unknown", + patient_id=row.get("product") or "unknown", + column="product_units_received", + error_code="type_conversion", + function_name="_clean_units_received", + original_value=str(original), + error_message=( + f"product_units_received '{original}' could not be converted to numeric" + ), + ) + + return df.with_columns( + pl.when(marker_mask).then(pl.lit(0.0)).otherwise(casted).alias("product_units_received") + ) + + +def _remove_empty_data_rows(df: pl.DataFrame) -> pl.DataFrame: + """Step 2.13 — drop rows with no meaningful transaction data. + + Removes rows where units_received, units_released, units_returned, + released_to, entry_date and balance are all null. + """ + existing = [c for c in EMPTY_ROW_COLS if c in df.columns] + if not existing: + return df + all_null = pl.all_horizontal([pl.col(c).is_null() for c in existing]) + return df.filter(~all_null) + + +def _compute_balance_status(df: pl.DataFrame) -> pl.DataFrame: + """Step 2.14 — label each row ``start`` / ``change`` / ``end`` per (product, sheet). + + First row in each (product, product_sheet_name) group is ``start``, last + is ``end``, all others are ``change``. The grouping is per-sheet, so a + product appearing in 12 sheets has 12 start rows and 12 end rows rather + than a single pair spanning the tracker -- each month's sheet opens and + closes its own ledger, and the closing balance of one is what the next + opens with. + """ + if "index" not in df.columns: + raise KeyError( + "_compute_balance_status requires an 'index' column; run _add_row_index first" + ) + + group = ["product_sheet_name", "product"] if "product_sheet_name" in df.columns else ["product"] + + return df.with_columns( + pl.when(pl.col("index") == pl.col("index").first().over(group)) + .then(pl.lit("start")) + .when(pl.col("index") == pl.col("index").last().over(group)) + .then(pl.lit("end")) + .otherwise(pl.lit("change")) + .alias("product_balance_status") + ) + + +def _compute_running_balance( + df: pl.DataFrame, error_collector: ErrorCollector | None = None +) -> pl.DataFrame: + """Step 2.15 — compute the running stock balance per product. + + Formula: ``balance[i] = balance[i-1] - released[i] + received[i]``. + For year >= 2021 the ``end`` rows' received/released are zeroed first + (they are summary rows, not real transactions). The year is read from + the ``product_table_year`` column (set by extraction). + + Implemented as a vectorized cumsum per product. This is only correct + because step 2.12 has already filled unit-column nulls with 0 and step + 2.7's sort is stable: cumsum treats a null as 0, so an unfilled null would + silently contribute nothing instead of breaking the ledger visibly. + + Preconditions: ``_fill_product_names_and_sort`` (step 2.7) and + ``_compute_balance_status`` (step 2.14) must have run. The first + positional row in each (sheet, product) group must be labeled + ``"start"`` because the cumsum below seeds from + ``product_balance.first().over(group)``. + + Recomputing overwrites whatever balance the tracker recorded on + ``change``/``end`` rows. Because step 2.7 sorts chronologically while the + source's own balance column accumulates in data-entry order, the + *intermediate* balances legitimately differ from the tracker's (31% of + recorded rows on the real 248-tracker set) and are not worth reporting. + The *closing* balance is order-independent, though, so a disagreement + there means the transactions and the tracker's own recorded total do not + add up -- a real source problem, and the only thing this step reports + (ticket 36; 1.1% of groups on the same data). + """ + group = ["product_sheet_name", "product"] if "product_sheet_name" in df.columns else ["product"] + + if "product_balance_status" not in df.columns: + raise RuntimeError( + "_compute_running_balance requires product_balance_status; " + "run _compute_balance_status (step 2.14) first" + ) + if df.height > 0: + first_status = df.select(pl.col("product_balance_status").first().over(group).alias("_fs"))[ + "_fs" + ] + if not (first_status == "start").all(): + bad_groups = ( + df.with_columns(pl.col("product_balance_status").first().over(group).alias("_fs")) + .filter(pl.col("_fs") != "start") + .select(group) + .unique() + .height + ) + raise RuntimeError( + f"_compute_running_balance precondition violated: {bad_groups} " + "(sheet, product) groups have a non-'start' row first. " + "Did _fill_product_names_and_sort (step 2.7) run?" + ) + + casts = [ + pl.col("product_balance").cast(pl.Float64, strict=False), + pl.col("product_units_released").cast(pl.Float64, strict=False), + ] + if "product_units_returned" in df.columns: + casts.append(pl.col("product_units_returned").cast(pl.Float64, strict=False)) + df = df.with_columns(casts) + + if "product_table_year" in df.columns: + year_gate = ( + pl.col("product_table_year").is_not_null() + & (pl.col("product_table_year") >= 2021) + & (pl.col("product_balance_status") == "end") + ) + df = df.with_columns( + pl.when(year_gate) + .then(pl.lit(0.0)) + .otherwise(pl.col("product_units_received")) + .alias("product_units_received"), + pl.when(year_gate) + .then(pl.lit(0.0)) + .otherwise(pl.col("product_units_released")) + .alias("product_units_released"), + ) + + df = df.with_columns( + pl.when((pl.col("product_balance_status") == "start") & pl.col("product_balance").is_null()) + .then(pl.col("product_units_received") - pl.col("product_units_released")) + .otherwise(pl.col("product_balance")) + .alias("product_balance") + ) + + is_start = pl.col("product_balance_status") == "start" + delta = ( + pl.when(is_start) + .then(pl.lit(0.0)) + .otherwise(pl.col("product_units_received") - pl.col("product_units_released")) + ) + source_balance = df.select( + pl.col("product_balance").alias("_source_balance"), + *[pl.col(c) for c in group], + pl.col("product_balance_status"), + *([pl.col("index")] if "index" in df.columns else []), + *([pl.col("file_name")] if "file_name" in df.columns else []), + ) + + df = df.with_columns( + (pl.col("product_balance").first().over(group) + delta.cum_sum().over(group)) + .round(10) + .alias("product_balance") + ) + + if error_collector is not None: + _report_balance_reconciliation(df, source_balance, group, error_collector) + return df + + +BALANCE_RECONCILIATION_ABS_TOL = 1e-6 +BALANCE_RECONCILIATION_REL_TOL = 1e-9 + + +def _report_balance_reconciliation( + df: pl.DataFrame, + source_balance: pl.DataFrame, + group: list[str], + error_collector: ErrorCollector, +) -> None: + """Flag groups whose recomputed closing stock contradicts the tracker's own. + + Compares the last balance the source recorded on a ``change``/``end`` row + (in the file's original input order, so the chronological re-sort cannot + change which value counts as "the tracker's closing figure") against the + computed closing balance. Groups where the source left every non-start + balance blank -- the common case -- have nothing to reconcile and are + skipped, since warning on them would bury the real signal. + """ + recorded = source_balance.filter( + (pl.col("product_balance_status") != "start") & pl.col("_source_balance").is_not_null() + ) + if recorded.height == 0: + return + if "index" in recorded.columns: + recorded = recorded.sort("index") + source_closing = recorded.group_by(group).agg( + pl.col("_source_balance").last().alias("_source_closing") + ) + + computed_closing = df.group_by(group).agg( + pl.col("product_balance").last().alias("_computed_closing"), + *([pl.col("file_name").last().alias("_file_name")] if "file_name" in df.columns else []), + ) + + joined = source_closing.join(computed_closing, on=group, how="inner") + for row in joined.iter_rows(named=True): + source_value = row["_source_closing"] + computed = row["_computed_closing"] + if computed is None: + continue + tolerance = max( + BALANCE_RECONCILIATION_ABS_TOL, + BALANCE_RECONCILIATION_REL_TOL * max(abs(source_value), abs(computed)), + ) + if abs(source_value - computed) <= tolerance: + continue + product = row.get("product") or "unknown" + sheet_name = row.get("product_sheet_name") or "unknown" + error_collector.add_error( + file_name=row.get("_file_name") or "unknown", + patient_id="unknown", + column="product_balance", + original_value=source_value, + error_message=( + f"Closing balance mismatch for product '{product}' in sheet " + f"'{sheet_name}': tracker recorded {source_value}, but the " + f"recorded transactions add up to {computed} " + f"(difference {round(computed - source_value, 10)}). " + "The stock movements and the tracker's own total do not agree." + ), + error_code="balance_reconciliation", + function_name="_compute_running_balance", + ) + + +def _validate_negative_balances( + df: pl.DataFrame, + error_collector: ErrorCollector, +) -> pl.DataFrame: + """Step 2.18 — log rows with a negative ``product_balance``. + + Data is not modified; each negative-balance row is reported via the + error collector for downstream investigation. + """ + if "product_balance" not in df.columns: + return df + + negatives = df.filter( + pl.col("product_balance").is_not_null() & (pl.col("product_balance") < 0) + ).select("file_name", "product_balance", "product", "product_sheet_name") + for file_name, balance, product, sheet_name in negatives.iter_rows(): + error_collector.add_error( + file_name=file_name or "unknown", + patient_id="unknown", + column="product_balance", + original_value=balance, + error_message=( + f"Negative balance {balance} for product " + f"'{product or 'unknown'}' in sheet " + f"'{sheet_name or 'unknown'}'" + ), + error_code="invalid_value", + function_name="_validate_negative_balances", + ) + return df + + +def _report_unknown_products( + df: pl.DataFrame, + error_collector: ErrorCollector, +) -> pl.DataFrame: + """Step 2.19 — flag product names missing from the Stock_Summary reference. + + Case-insensitive comparison against the known product list. DataFrame is + returned unchanged; violations are logged to the error collector. + """ + if "product" not in df.columns: + return df + + known = set(load_known_products()) + + # Key errors on (file_name, product_sheet_name, product) triples, so an + # unknown product is reported once per sheet it appears on rather than + # once per row -- the correction is to the reference list or the sheet's + # spelling, and both are per-sheet facts. + cols = [c for c in ("file_name", "product_sheet_name", "product") if c in df.columns] + unknowns = ( + df.filter(pl.col("product").is_not_null()) + .with_columns(pl.col("product").str.to_lowercase().alias("_lower")) + .filter(~pl.col("_lower").is_in(list(known))) + .select(cols) + .unique() + ) + for row in unknowns.iter_rows(named=True): + error_collector.add_error( + file_name=row.get("file_name") or "unknown", + patient_id="unknown", + column="product", + original_value=row["product"], + error_message=( + f"Unknown product '{row['product']}' in sheet " + f"'{row.get('product_sheet_name') or 'unknown'}'" + ), + error_code="invalid_value", + function_name="_report_unknown_products", + ) + return df + + +def _add_product_categories(df: pl.DataFrame) -> pl.DataFrame: + """Step 2.20 — left-join ``product_category`` from Stock_Summary. + + Products not present in the reference receive a null category. + """ + if "product" not in df.columns: + return df + + # columns: product (lowercased), product_category + categories = load_product_categories() + + df = df.with_columns(pl.col("product").cast(pl.Utf8).str.to_lowercase().alias("_product_join")) + df = df.join( + categories.rename({"product": "_product_join"}), + on="_product_join", + how="left", + ) + return df.drop("_product_join") + + +def _extract_unit_capacity(df: pl.DataFrame) -> pl.DataFrame: + """Step 2.21 — parse units-per-package from the product name. + + Reads parenthesised hints such as ``(2s)`` or ``(10's)`` into + ``product_unit_capacity``. ``singles`` maps to 1; absent hints default + to 1 as well. + """ + if "product" not in df.columns: + return df.with_columns(pl.lit(1, dtype=pl.Int32).alias("product_unit_capacity")) + + paren = pl.col("product").cast(pl.Utf8).str.extract(r"\(([^()]+)\)", 1) + paren_normalized = ( + pl.when(paren.str.contains("(?i)singles")).then(pl.lit("1s")).otherwise(paren) + ) + # Digits immediately followed by an optional apostrophe and then "s" + # (covers "(10s)", "(5's)"). Non-matching paren content → null → 1. + digits = paren_normalized.str.extract(r"(\d+)'?s", 1) + capacity = digits.cast(pl.Int32, strict=False).fill_null(1) + + return df.with_columns(capacity.alias("product_unit_capacity")) diff --git a/src/a4d/clean/schema.py b/src/a4d/clean/schema.py new file mode 100644 index 0000000..ae595ea --- /dev/null +++ b/src/a4d/clean/schema.py @@ -0,0 +1,163 @@ +"""Meta schema definition for patient data. + +Every cleaned patient parquet conforms to this, whichever columns its source +tracker happened to carry -- so downstream consumers see one stable shape +across 254 trackers and nine template years. +""" + +import polars as pl + + +def get_patient_data_schema() -> dict[str, type[pl.DataType] | pl.DataType]: + """Get the complete meta schema for patient data. + + Columns are in alphabetical order. The order is part of the contract: + BigQuery consumers and the parquet snapshots both depend on it. + + Returns: + Dictionary mapping column names to Polars data types + """ + return { + "age": pl.Int32, + "analog_insulin_long_acting": pl.String, + "analog_insulin_rapid_acting": pl.String, + "blood_pressure_dias_mmhg": pl.Int32, + "blood_pressure_sys_mmhg": pl.Int32, + "blood_pressure_updated": pl.Date, + "bmi": pl.Float64, + "bmi_date": pl.Date, + "clinic_id": pl.String, + "clinic_visit": pl.String, + "complication_screening_eye_exam_date": pl.Date, + "complication_screening_eye_exam_value": pl.String, + "complication_screening_foot_exam_date": pl.Date, + "complication_screening_foot_exam_value": pl.String, + "complication_screening_kidney_test_date": pl.Date, + "complication_screening_kidney_test_value": pl.String, + "complication_screening_lipid_profile_cholesterol_value": pl.String, + "complication_screening_lipid_profile_date": pl.Date, + "complication_screening_lipid_profile_hdl_mmol_value": pl.Float64, + "complication_screening_lipid_profile_hdl_mg_value": pl.Float64, + "complication_screening_lipid_profile_ldl_mmol_value": pl.Float64, + "complication_screening_lipid_profile_ldl_mg_value": pl.Float64, + "complication_screening_lipid_profile_triglycerides_value": pl.Float64, + "complication_screening_remarks": pl.String, + "complication_screening_thyroid_test_date": pl.Date, + "complication_screening_thyroid_test_ft4_pmol_value": pl.Float64, + "complication_screening_thyroid_test_ft4_ng_value": pl.Float64, + "complication_screening_thyroid_test_tsh_value": pl.Float64, + "dm_complication_eye": pl.String, + "dm_complication_kidney": pl.String, + "dm_complication_others": pl.String, + "dm_complication_remarks": pl.String, + "dob": pl.Date, + "edu_occ": pl.String, + "edu_occ_updated": pl.Date, + "family_history": pl.String, + "fbg_baseline_mg": pl.Float64, + "fbg_baseline_mmol": pl.Float64, + "fbg_updated_date": pl.Date, + "fbg_updated_mg": pl.Float64, + "fbg_updated_mmol": pl.Float64, + "file_name": pl.String, + "hba1c_baseline": pl.Float64, + "hba1c_baseline_exceeds": pl.Boolean, + "hba1c_updated": pl.Float64, + "hba1c_updated_exceeds": pl.Boolean, + "hba1c_updated_date": pl.Date, + "height": pl.Float64, + "hospitalisation_cause": pl.String, + "hospitalisation_date": pl.Date, + "human_insulin_intermediate_acting": pl.String, + "human_insulin_pre_mixed": pl.String, + "human_insulin_short_acting": pl.String, + "insulin_injections": pl.Float64, + "insulin_regimen": pl.String, + "insulin_total_units": pl.Float64, + "insulin_type": pl.String, + "insulin_subtype": pl.String, + "last_clinic_visit_date": pl.Date, + "last_remote_followup_date": pl.Date, + "lost_date": pl.Date, + "name": pl.String, + "observations": pl.String, + "observations_category": pl.String, + "other_issues": pl.String, + "patient_consent": pl.String, + "patient_id": pl.String, + "province": pl.String, + "recruitment_date": pl.Date, + "remote_followup": pl.String, + "sex": pl.String, + "sheet_name": pl.String, + "status": pl.String, + "status_out": pl.String, + "support_level": pl.String, + "t1d_diagnosis_age": pl.Int32, + "t1d_diagnosis_date": pl.Date, + "t1d_diagnosis_with_dka": pl.String, + "testing_frequency": pl.Int32, + "tracker_date": pl.Date, + "tracker_month": pl.Int32, + "tracker_year": pl.Int32, + "weight": pl.Float64, + } + + +def apply_schema(df: pl.DataFrame) -> pl.DataFrame: + """Apply the meta schema to a DataFrame. + + This function: + 1. Adds missing columns with NULL values typed per the schema. + 2. Reorders columns to match schema order. + + Casting is the caller's responsibility (see ``safe_convert_column``). + + Args: + df: Input DataFrame (may be missing columns) + + Returns: + DataFrame with complete schema applied + """ + schema = get_patient_data_schema() + + # Start with existing columns + df_result = df + + # Add missing columns with NULL values + missing_cols = set(schema.keys()) - set(df.columns) + for col in missing_cols: + df_result = df_result.with_columns(pl.lit(None, dtype=schema[col]).alias(col)) + + # Reorder columns to match schema order + df_result = df_result.select(list(schema.keys())) + + return df_result + + +def get_numeric_columns() -> list[str]: + """Get list of numeric columns from schema.""" + schema = get_patient_data_schema() + return [ + col + for col, dtype in schema.items() + if dtype in (pl.Int32, pl.Int64, pl.Float32, pl.Float64) + ] + + +def get_date_columns() -> list[str]: + """Get list of date columns from schema.""" + schema = get_patient_data_schema() + return [col for col, dtype in schema.items() if dtype == pl.Date] + + +def get_boolean_columns() -> list[str]: + """Get list of boolean columns from schema.""" + schema = get_patient_data_schema() + return [col for col, dtype in schema.items() if dtype == pl.Boolean] + + +def get_string_columns() -> list[str]: + """Get list of string columns from schema.""" + schema = get_patient_data_schema() + return [col for col, dtype in schema.items() if dtype == pl.String] diff --git a/src/a4d/clean/schema_product.py b/src/a4d/clean/schema_product.py new file mode 100644 index 0000000..bd1adca --- /dev/null +++ b/src/a4d/clean/schema_product.py @@ -0,0 +1,77 @@ +"""Meta schema definition for product data. + +Every cleaned product parquet conforms to this regardless of which columns its +source sheet carried, including trackers from years before product tracking +existed, which yield a conformant empty frame rather than failing. +""" + +import polars as pl + + +def get_product_data_schema() -> dict[str, type[pl.DataType] | pl.DataType]: + """Get the complete meta schema for product data. + + Column order is part of the contract: BigQuery consumers depend on it. + + Returns: + Dictionary mapping column names to Polars data types + """ + return { + "product": pl.String, + "product_units_notes": pl.String, + "product_entry_date": pl.Date, + "product_units_released": pl.Float64, + "product_released_to": pl.String, + "product_units_received": pl.Float64, + "product_received_from": pl.String, + "product_balance": pl.Float64, + "product_units_returned": pl.Float64, + "product_returned_by": pl.String, + "product_table_month": pl.Int32, + "product_table_year": pl.Int32, + "product_sheet_name": pl.String, + "file_name": pl.String, + "product_balance_status": pl.String, + "product_category": pl.String, + "orig_product_released_to": pl.String, + "product_unit_capacity": pl.Int32, + "product_remarks": pl.String, + "clinic_id": pl.String, + } + + +def apply_schema(df: pl.DataFrame) -> pl.DataFrame: + """Apply the meta schema to a DataFrame. + + This function: + 1. Adds missing columns with NULL values typed per the schema. + 2. Reorders columns to match schema order. + + Casting is the caller's responsibility (see ``safe_convert_column``). + + Args: + df: Input DataFrame (may be missing columns) + + Returns: + DataFrame with complete schema applied + """ + schema = get_product_data_schema() + + # Start with existing columns + df_result = df + + # Add missing columns with NULL values + missing_cols = set(schema.keys()) - set(df.columns) + for col in missing_cols: + df_result = df_result.with_columns(pl.lit(None, dtype=schema[col]).alias(col)) + + # Reorder columns to match schema order + df_result = df_result.select(list(schema.keys())) + + return df_result + + +def get_string_columns() -> list[str]: + """Get list of string columns from schema.""" + schema = get_product_data_schema() + return [col for col, dtype in schema.items() if dtype == pl.String] diff --git a/src/a4d/clean/transformers.py b/src/a4d/clean/transformers.py new file mode 100644 index 0000000..80a21a6 --- /dev/null +++ b/src/a4d/clean/transformers.py @@ -0,0 +1,418 @@ +"""Data transformation functions for cleaning. + +This module provides transformation functions that are applied before validation. +These functions standardize values, fix legacy formats, and normalize data. + +Transformations are referenced in reference_data/data_cleaning.yaml with +type: basic_function. +""" + +import polars as pl + +from a4d.config import settings + + +def strip_string_whitespace(df: pl.DataFrame) -> pl.DataFrame: + """Strip leading/trailing whitespace from every string column (ticket 36). + + Applied to the raw frame rather than a schema-typed one, so it catches + source columns before they are parsed or validated. Whitespace at the + ends of a cell never carries meaning in these trackers -- it is a + data-entry artefact, in cell values, sheet tab names and filenames + alike -- and leaving it in place makes identifiers that should join + silently fail to. + """ + string_cols = [col for col, dtype in df.schema.items() if dtype == pl.String] + if not string_cols: + return df + return df.with_columns([pl.col(col).str.strip_chars() for col in string_cols]) + + +def extract_regimen(df: pl.DataFrame, column: str = "insulin_regimen") -> pl.DataFrame: + """Extract and standardize insulin regimen values. + + This function applies regex pattern matching to standardize insulin regimen + descriptions into canonical forms. Matches are case-insensitive. + + Transformations: + - Contains "basal" → "Basal-bolus (MDI)" + - Contains "premixed" → "Premixed 30/70 BD" + - Contains "self-mixed" → "Self-mixed BD" + - Contains "conventional" → "Modified conventional TID" + + Args: + df: Input DataFrame + column: Column name to transform (default: "insulin_regimen") + + Returns: + DataFrame with standardized insulin regimen values + + Example: + >>> df = extract_regimen(df) + >>> # "Basal-bolus" → "Basal-bolus (MDI)" + >>> # "PREMIXED 30/70" → "Premixed 30/70 BD" + """ + if column not in df.columns: + return df + + # The case-insensitive flag belongs in the patterns, not on the data. An + # earlier version lowercased the whole column instead (ticket 29), which + # rewrote every value none of the four patterns matched -- "NPH" -> "nph", + # "Other" -> "other" -- corrupting the values it was supposed to leave + # alone. + df = df.with_columns( + pl.col(column) + .str.replace(r"(?i)^.*basal.*$", "Basal-bolus (MDI)") + .str.replace(r"(?i)^.*premixed.*$", "Premixed 30/70 BD") + .str.replace(r"(?i)^.*self-mixed.*$", "Self-mixed BD") + .str.replace(r"(?i)^.*conventional.*$", "Modified conventional TID") + .alias(column) + ) + + return df + + +def fix_sex(df: pl.DataFrame, column: str = "sex") -> pl.DataFrame: + """Map sex synonyms to canonical values (M/F) or error value. + + - Female synonyms: female, girl, woman, fem, feminine, f → "F" + - Male synonyms: male, boy, man, masculine, m → "M" + - Anything else → "Undefined" (error value) + + Args: + df: Input DataFrame + column: Column name to transform (default: "sex") + + Returns: + DataFrame with sex values normalized to M/F or Undefined + + Example: + >>> df = fix_sex(df) + >>> # "Female" → "F" + >>> # "MALE" → "M" + >>> # "invalid" → "Undefined" + """ + if column not in df.columns: + return df + + synonyms_female = ["female", "girl", "woman", "fem", "feminine", "f"] + synonyms_male = ["male", "boy", "man", "masculine", "m"] + + # Build expression using pl.when().then().when().then()... chain + # Start with null/empty handling + expr = pl.when(pl.col(column).is_null() | (pl.col(column) == "")).then(None) + + # Add female synonyms + for synonym in synonyms_female: + expr = expr.when(pl.col(column).str.to_lowercase() == synonym).then(pl.lit("F")) + + # Add male synonyms + for synonym in synonyms_male: + expr = expr.when(pl.col(column).str.to_lowercase() == synonym).then(pl.lit("M")) + + # Default: anything else becomes Undefined + expr = expr.otherwise(pl.lit(settings.error_val_character)) + + df = df.with_columns(expr.alias(column)) + + return df + + +def fix_bmi(df: pl.DataFrame) -> pl.DataFrame: + """Calculate BMI from weight and height. + + - If weight or height is null → BMI becomes null + - If weight or height is error value → BMI becomes error value + - Otherwise: BMI = weight / height^2 + + Height is converted from cm to m if > 50, so the calculation is correct + whichever unit the clinic recorded in. + + This calculation REPLACES any existing BMI value: a typed BMI goes stale as + the patient's weight changes, while the two measurements it derives from + are re-recorded each month. + + Args: + df: Input DataFrame (must have weight and height columns) + + Returns: + DataFrame with calculated BMI column + + Example: + >>> df = fix_bmi(df) + >>> # weight=70, height=1.75 → bmi=22.86 + >>> # weight=30.7, height=135.5 (cm) → height_m=1.355, bmi=16.72 + """ + if "weight" not in df.columns or "height" not in df.columns: + return df + + # Convert height from cm to m if > 50 + height_m = ( + pl.when(pl.col("height") > 50).then(pl.col("height") / 100.0).otherwise(pl.col("height")) + ) + + # Calculate BMI: weight / height^2. Null and the error sentinel are kept + # distinct -- an absent measurement yields null, an unusable one yields the + # sentinel -- so a derived BMI never claims more than its inputs did. + df = df.with_columns( + pl.when(pl.col("weight").is_null() | pl.col("height").is_null()) + .then(None) + .when( + (pl.col("weight") == settings.error_val_numeric) + | (pl.col("height") == settings.error_val_numeric) + ) + .then(pl.lit(settings.error_val_numeric)) + .otherwise(pl.col("weight") / height_m.pow(2)) + .alias("bmi") + ) + + return df + + +def str_to_lower(df: pl.DataFrame, column: str) -> pl.DataFrame: + """Convert column values to lowercase. + + This is used for case-insensitive validation. For example, the "status" + column may have mixed case values like "Active", "ACTIVE", "active" which + should all be normalized to lowercase before validation. + + Args: + df: Input DataFrame + column: Column name to transform + + Returns: + DataFrame with lowercase column values + + Example: + >>> df = str_to_lower(df, "status") + >>> # "ACTIVE" → "active" + >>> # "Inactive" → "inactive" + """ + if column not in df.columns: + return df + + df = df.with_columns(pl.col(column).str.to_lowercase().alias(column)) + + return df + + +def apply_transformation( + df: pl.DataFrame, + column: str, + function_name: str, +) -> pl.DataFrame: + """Apply a named transformation function to a column. + + This is the dispatcher function that maps function names from + data_cleaning.yaml to actual transformation functions. + + Args: + df: Input DataFrame + column: Column name to transform + function_name: Name of transformation function (from YAML) + + Returns: + DataFrame with transformation applied + + Raises: + ValueError: If function_name is not recognized + + Example: + >>> df = apply_transformation(df, "status", "stringr::str_to_lower") + >>> df = apply_transformation(df, "insulin_regimen", "extract_regimen") + """ + # Transformation names as they appear in reference_data/data_cleaning.yaml. + # The `stringr::` prefix is retained as an accepted alias because the + # config files still carry it. + function_mapping = { + "extract_regimen": lambda df, col: extract_regimen(df, col), + "stringr::str_to_lower": lambda df, col: str_to_lower(df, col), + "str_to_lower": lambda df, col: str_to_lower(df, col), + } + + if function_name not in function_mapping: + raise ValueError(f"Unknown transformation function: {function_name}") + + return function_mapping[function_name](df, column) + + +def correct_decimal_sign_multiple( + df: pl.DataFrame, + columns: list[str], +) -> pl.DataFrame: + """Replace comma decimal separator with dot for multiple columns. + + Some trackers use European decimal format (1,5 instead of 1.5). + This function fixes that for multiple numeric columns. + + Args: + df: Input DataFrame + columns: List of column names to correct + + Returns: + DataFrame with corrected decimal signs + + Example: + >>> df = correct_decimal_sign_multiple(df, ["weight", "height", "hba1c"]) + """ + from a4d.clean.converters import correct_decimal_sign + + for column in columns: + df = correct_decimal_sign(df, column) + + return df + + +def replace_range_with_mean(x: str) -> float: + """Calculate mean of a range string. + + Splits string on "-", converts parts to numeric, returns the mean: a + clinic writing "0-2" recorded a range, and the midpoint is the only + defensible single number to publish for it. + + Args: + x: Range string (e.g., "0-2", "2-3") + + Returns: + Mean of the range values + + Example: + >>> replace_range_with_mean("0-2") + 1.0 + >>> replace_range_with_mean("2-3") + 2.5 + """ + parts = x.split("-") + numbers = [float(p) for p in parts] + return sum(numbers) / len(numbers) + + +def fix_testing_frequency(df: pl.DataFrame) -> pl.DataFrame: + """Fix testing_frequency column by replacing ranges with mean values. + + - Replaces ranges like "0-2" with mean "1" + - Preserves null and empty values as null + - Logs warning when ranges are detected + + Args: + df: Input DataFrame + + Returns: + DataFrame with testing_frequency ranges replaced by mean values + + Example: + >>> df = fix_testing_frequency(df) + >>> # "0-2" → "1" + >>> # "2-3" → "2.5" + >>> # "2" → "2" (unchanged) + """ + if "testing_frequency" not in df.columns: + return df + + from loguru import logger + + # Track if we logged warnings + has_ranges = False + + def fix_value(value: str | None) -> str | None: + """Fix a single testing_frequency value.""" + nonlocal has_ranges + + if value is None or value == "": + return None + + if "-" in value: + has_ranges = True + + try: + mean_value = replace_range_with_mean(value) + # Return as string, remove trailing .0 for whole numbers + if mean_value == int(mean_value): + return str(int(mean_value)) + return str(mean_value) + except Exception: + # If replacement fails, return None + return None + + return value + + # Apply transformation + df = df.with_columns( + pl.col("testing_frequency") + .map_elements(fix_value, return_dtype=pl.String) + .alias("testing_frequency") + ) + + # Log warning if any ranges were found + if has_ranges: + logger.bind(error_code="invalid_value").warning( + "Found ranges in testing_frequency column. Replacing with mean values." + ) + + return df + + +def split_bp_in_sys_and_dias(df: pl.DataFrame) -> pl.DataFrame: + """Split blood_pressure_mmhg into systolic and diastolic columns. + + - Splits "120/80" format into two columns + - Invalid formats (without "/") are replaced with error value + - Logs warning for invalid values + + Args: + df: Input DataFrame with blood_pressure_mmhg column + + Returns: + DataFrame with blood_pressure_sys_mmhg and blood_pressure_dias_mmhg columns + + Example: + >>> df = split_bp_in_sys_and_dias(df) + >>> # "96/55" → sys="96", dias="55" + >>> # "96" → sys="999999", dias="999999" (invalid) + """ + if "blood_pressure_mmhg" not in df.columns: + return df + + from loguru import logger + + # First, replace invalid values (those without "/") with error format + error_val_int = int(settings.error_val_numeric) + df = df.with_columns( + pl.when(~pl.col("blood_pressure_mmhg").str.contains("/", literal=True)) + .then(pl.lit(f"{error_val_int}/{error_val_int}")) + .otherwise(pl.col("blood_pressure_mmhg")) + .alias("blood_pressure_mmhg") + ) + + # Check if any invalid values were found + error_pattern = f"{error_val_int}/{error_val_int}" + has_errors = df.filter(pl.col("blood_pressure_mmhg") == error_pattern).height > 0 + + if has_errors: + logger.bind(error_code="invalid_value").warning( + "Found invalid values for column blood_pressure_mmhg " + f"that do not follow the format X/Y. " + f"Values were replaced with {error_val_int}." + ) + + # Split the column, trimming each fragment. Polars' numeric cast fails on + # surrounding whitespace, so without the trim every blood pressure written + # "70 / 40" reached the cleaned output as the 999999 error sentinel -- + # 465 cells across 7 trackers (ticket 53). + df = df.with_columns( + pl.col("blood_pressure_mmhg") + .str.split("/") + .list.get(0) + .str.strip_chars() + .alias("blood_pressure_sys_mmhg"), + pl.col("blood_pressure_mmhg") + .str.split("/") + .list.get(1) + .str.strip_chars() + .alias("blood_pressure_dias_mmhg"), + ) + + # Drop the original combined column + df = df.drop("blood_pressure_mmhg") + + return df diff --git a/src/a4d/clean/validators.py b/src/a4d/clean/validators.py new file mode 100644 index 0000000..f8661ab --- /dev/null +++ b/src/a4d/clean/validators.py @@ -0,0 +1,579 @@ +"""Schema and validation utilities for data cleaning. + +This module provides functions for validating DataFrame columns against +allowed values defined in reference_data/validation_rules.yaml. + +The validation pattern is: +1. Load validation rules from YAML +2. Check column values against allowed values +3. Log invalid values to ErrorCollector +4. Replace invalid values with error value (if configured) + +Note: Data transformations are NOT in the YAML - they are hardcoded in +transformers.py for better type safety and maintainability. +""" + +import re +from typing import Any + +import polars as pl + +from a4d.config import settings +from a4d.errors import ErrorCollector +from a4d.reference.loaders import get_reference_data_path, load_yaml + + +def sanitize_str(text: str) -> str: + """Sanitize string for case-insensitive matching. + + Punctuation, casing and spacing must not decide whether a clinic's typed + value matches an allowed one: + 1. Convert to lowercase + 2. Remove spaces + 3. Remove special characters (keep only alphanumeric) + + Args: + text: String to sanitize + + Returns: + Sanitized string + + Example: + >>> sanitize_str("Active - Remote") + 'activeremote' + >>> sanitize_str("Lost Follow Up") + 'lostfollowup' + """ + if not isinstance(text, str): + return text + return re.sub(r"[^a-z0-9]", "", text.lower()) + + +def load_validation_rules() -> dict[str, Any]: + """Load validation rules from validation_rules.yaml. + + Returns: + Dictionary mapping column names to their validation rules. + Structure: {column_name: {allowed_values: [...], replace_invalid: bool}} + + Example: + >>> rules = load_validation_rules() + >>> rules["status"]["allowed_values"] + ['active', 'inactive', ...] + >>> rules["status"]["replace_invalid"] + True + """ + yaml_path = get_reference_data_path("validation_rules.yaml") + return load_yaml(yaml_path) + + +def load_numeric_ranges() -> dict[str, dict[str, float]]: + """Load the ``numeric_ranges`` block from validation_rules.yaml. + + Consumed by the source-vs-output validator. Mirrors the hardcoded + thresholds in clean/patient.py; see the YAML comment for the drift caveat. + """ + rules = load_validation_rules() + return rules.get("numeric_ranges", {}) + + +def validate_allowed_values( + df: pl.DataFrame, + column: str, + allowed_values: list[str], + error_collector: ErrorCollector, + replace_invalid: bool = True, + file_name_col: str = "file_name", + patient_id_col: str = "patient_id", + allow_csv_subset: bool = False, + aliases: dict[str, list[str]] | None = None, +) -> pl.DataFrame: + """Validate column against allowed values with case-insensitive matching. + + 1. Sanitize both input values and allowed values for matching + 2. If matched, replace with canonical value from allowed_values + 3. If not matched, replace with error value (if replace_invalid=True) + + Where a tracker generation renamed a label, the retired spelling belongs + in ``aliases`` rather than in ``allowed_values`` (ticket 29): reports need + one label per status, and which spelling wins has to be a stated decision + in the config, not a consequence of list order. Two canonical values that + sanitize to the same key are therefore a config error and raise, instead + of silently resolving to whichever the dict happened to keep. + + Args: + df: Input DataFrame + column: Column name to validate + allowed_values: List of canonical allowed values (e.g., ["Active", "Inactive"]) + error_collector: ErrorCollector instance to track violations + replace_invalid: If True, replace invalid values with error value + file_name_col: Column containing file name for error tracking + patient_id_col: Column containing patient ID for error tracking + aliases: Canonical value -> the retired spellings it absorbs, e.g. + ``{"Active Remote": ["Active - Remote"]}``. Matched + case-insensitively like everything else here. + + Returns: + DataFrame with values normalized to canonical form or replaced + + Example: + >>> collector = ErrorCollector() + >>> df = validate_allowed_values( + ... df=df, + ... column="status", + ... allowed_values=["Active", "Inactive"], # Canonical forms + ... error_collector=collector, + ... ) + >>> # "active", "ACTIVE", "Active" all become "Active" + """ + if column not in df.columns: + return df + + # Map {sanitized -> canonical}, so one label is published per status + # however the clinic spelled it. + # E.g., {"active": "Active", "activeremote": "Active Remote"} + canonical_mapping: dict[str, str] = {} + for val in allowed_values: + key = sanitize_str(val) + if key in canonical_mapping: + raise ValueError( + f"{column}: allowed values {canonical_mapping[key]!r} and {val!r} " + f"sanitize to the same key {key!r}. Keep one as canonical and " + f"declare the other under `aliases`." + ) + canonical_mapping[key] = val + + for canonical, spellings in (aliases or {}).items(): + if canonical not in allowed_values: + raise ValueError( + f"{column}: aliases are declared for {canonical!r}, which is not an allowed value." + ) + for alias in spellings: + key = sanitize_str(alias) + if key in canonical_mapping and canonical_mapping[key] != canonical: + raise ValueError( + f"{column}: alias {alias!r} of {canonical!r} collides with " + f"{canonical_mapping[key]!r}." + ) + canonical_mapping[key] = canonical + + # Get unique non-null values from the column + col_values = df.filter(pl.col(column).is_not_null()).select(column).unique() + + # Track which values need replacement and their canonical forms + value_replacements = {} # {original → canonical or error_value} + + for row in col_values.iter_rows(named=True): + original_val = row[column] + + # Skip if already the error value + if original_val == settings.error_val_character: + value_replacements[original_val] = original_val + continue + + # Sanitize and lookup + sanitized = sanitize_str(original_val) + + if sanitized in canonical_mapping: + # Valid - replace with canonical value + value_replacements[original_val] = canonical_mapping[sanitized] + elif allow_csv_subset and "," in original_val: + # e.g. insulin_subtype "pre-mixed,rapid-acting" is valid if every + # token is in allowed_values. Emit canonical-case CSV. + parts = [p.strip() for p in original_val.split(",") if p.strip()] + canonical_parts = [] + all_matched = bool(parts) + for part in parts: + part_sanitized = sanitize_str(part) + if part_sanitized in canonical_mapping: + canonical_parts.append(canonical_mapping[part_sanitized]) + else: + all_matched = False + break + if all_matched: + value_replacements[original_val] = ",".join(canonical_parts) + continue + # Fall through to the invalid branch below. + error_collector.add_error( + file_name="unknown", + patient_id="unknown", + column=column, + original_value=original_val, + error_message=( + f"Value '{original_val}' not in allowed values " + f"(CSV-subset check): {allowed_values}" + ), + error_code="invalid_value", + function_name="validate_allowed_values", + ) + value_replacements[original_val] = ( + settings.error_val_character if replace_invalid else original_val + ) + else: + # Invalid - log error + error_collector.add_error( + file_name="unknown", # Will be filled in bulk operations + patient_id="unknown", + column=column, + original_value=original_val, + error_message=f"Value '{original_val}' not in allowed values: {allowed_values}", + error_code="invalid_value", + function_name="validate_allowed_values", + ) + + if replace_invalid: + value_replacements[original_val] = settings.error_val_character + else: + value_replacements[original_val] = original_val + + # Apply all replacements at once using pl.when().then() chain + # This ensures we replace with canonical values even if they match + if value_replacements: + expr = pl.col(column) + for original, replacement in value_replacements.items(): + expr = pl.when(pl.col(column) == original).then(pl.lit(replacement)).otherwise(expr) + + df = df.with_columns(expr.alias(column)) + + return df + + +def validate_column_from_rules( + df: pl.DataFrame, + column: str, + rules: dict[str, Any], + error_collector: ErrorCollector, + file_name_col: str = "file_name", + patient_id_col: str = "patient_id", +) -> pl.DataFrame: + """Validate column using rules from validation_rules.yaml. + + Args: + df: Input DataFrame + column: Column name to validate + rules: Validation rules for this column (from validation_rules.yaml) + Structure: {allowed_values: [...], replace_invalid: bool} + error_collector: ErrorCollector instance + file_name_col: Column containing file name for error tracking + patient_id_col: Column containing patient ID for error tracking + + Returns: + DataFrame with column validated and cleaned + + Example: + >>> rules = load_validation_rules() + >>> collector = ErrorCollector() + >>> df = validate_column_from_rules( + ... df=df, + ... column="status", + ... rules=rules["status"], + ... error_collector=collector, + ... ) + """ + if column not in df.columns: + return df + + # Extract validation parameters from simplified rules + allowed_values = rules.get("allowed_values", []) + replace_invalid = rules.get("replace_invalid", True) + allow_csv_subset = rules.get("allow_csv_subset", False) + aliases = rules.get("aliases") + + df = validate_allowed_values( + df=df, + column=column, + allowed_values=allowed_values, + error_collector=error_collector, + replace_invalid=replace_invalid, + file_name_col=file_name_col, + patient_id_col=patient_id_col, + allow_csv_subset=allow_csv_subset, + aliases=aliases, + ) + + return df + + +def validate_province( + df: pl.DataFrame, + error_collector: ErrorCollector, + file_name_col: str = "file_name", + patient_id_col: str = "patient_id", +) -> pl.DataFrame: + """Validate province column against allowed provinces from YAML. + + Uses the shared allowed_provinces.yaml file to validate province values, + sanitizing both sides for comparison and setting an unrecognised province + to "Undefined". Sanitizing folds accents, which recovers spellings a + stricter comparison loses (`Thái Nguyễn` -> `Thái Nguyên`); this is safe + because the 209-entry province list has no two entries that sanitize + alike, and validate_allowed_values raises if any two ever do. + + Args: + df: Input DataFrame + error_collector: ErrorCollector instance + file_name_col: Column containing file name for error tracking + patient_id_col: Column containing patient ID for error tracking + + Returns: + DataFrame with province validated + + Example: + >>> collector = ErrorCollector() + >>> df = validate_province(df, collector) + """ + from a4d.reference.provinces import load_canonical_provinces + + if "province" not in df.columns: + return df + + # Load canonical province names (with proper casing) for validation + allowed_provinces = load_canonical_provinces() + + # Use generic validator with loaded provinces + df = validate_allowed_values( + df=df, + column="province", + allowed_values=allowed_provinces, + error_collector=error_collector, + replace_invalid=True, + file_name_col=file_name_col, + patient_id_col=patient_id_col, + ) + + return df + + +def validate_all_columns( + df: pl.DataFrame, + error_collector: ErrorCollector, + file_name_col: str = "file_name", + patient_id_col: str = "patient_id", +) -> pl.DataFrame: + """Validate all columns that have rules in data_cleaning.yaml. + + Args: + df: Input DataFrame + error_collector: ErrorCollector instance + file_name_col: Column containing file name for error tracking + patient_id_col: Column containing patient ID for error tracking + + Returns: + DataFrame with all columns validated + + Example: + >>> collector = ErrorCollector() + >>> df_clean = validate_all_columns(df, collector) + >>> len(collector) # Number of validation errors found + """ + rules = load_validation_rules() + + for column, column_rules in rules.items(): + if column in df.columns: + df = validate_column_from_rules( + df=df, + column=column, + rules=column_rules, + error_collector=error_collector, + file_name_col=file_name_col, + patient_id_col=patient_id_col, + ) + + # Validate province separately (not in validation_rules.yaml) + df = validate_province( + df=df, + error_collector=error_collector, + file_name_col=file_name_col, + patient_id_col=patient_id_col, + ) + + # Fix patient_id LAST (other functions use it for logging) + df = fix_patient_id( + df=df, + error_collector=error_collector, + patient_id_col=patient_id_col, + ) + + return df + + +def _is_one_edit_apart(a: str, b: str) -> bool: + """Whether one insertion, deletion or substitution turns ``a`` into ``b``. + + A bounded Levenshtein check rather than a full distance: recovery only ever + accepts distance 1, so nothing is gained by computing the rest. + """ + if abs(len(a) - len(b)) > 1: + return False + + if len(a) == len(b): + return sum(x != y for x, y in zip(a, b, strict=True)) == 1 + + shorter, longer = (a, b) if len(a) < len(b) else (b, a) + i = j = 0 + skipped = False + while i < len(shorter) and j < len(longer): + if shorter[i] == longer[j]: + i += 1 + j += 1 + elif skipped: + return False + else: + skipped = True + j += 1 + return True + + +def recover_patient_id(malformed: str, known_ids: set[str]) -> str | None: + """The ID this tracker spells correctly elsewhere, if exactly one fits. + + Ticket 47: 2023_NPH's `Sep23` sheet types a stray H into four IDs + (`KH_NPH026`), while its Patient List and every other month sheet write + `KH_NP026`. Rather than guess at the intended patient, this accepts a + candidate only when the same tracker already carries a well-formed ID one + edit away, and only when there is exactly one such candidate -- two + candidates mean the intended patient is unknowable from the evidence. + + Args: + malformed: The ID that failed format validation, hyphens already + normalized. + known_ids: The well-formed IDs present elsewhere in the same tracker. + + Returns: + The recovered ID, or None if no unambiguous candidate exists. + """ + candidates = [known for known in known_ids if _is_one_edit_apart(malformed, known)] + return candidates[0] if len(candidates) == 1 else None + + +def fix_patient_id( + df: pl.DataFrame, + error_collector: ErrorCollector, + patient_id_col: str = "patient_id", +) -> pl.DataFrame: + """Validate and fix patient ID format. + + - Valid format: XX_YY### (e.g., "KD_EW004") + - 2 uppercase letters, underscore, 2 uppercase letters, 3 digits + - Normalizes hyphens to underscores: "KD-EW004" → "KD_EW004" + - Recovers a one-character typo against the tracker's own well-formed IDs + - Replaces with the error value otherwise + + **An over-long ID is never truncated (ticket 47).** Cutting anything + longer than 8 characters to its first 8 published `KH_NPH02` -- an + identifier present in no source workbook -- and filed four different + patients' September records under it, detaching them from their own Oct-Dec + history (2023_NPH, whose Sep23 sheet types a stray `H` into four IDs its + own Patient List spells correctly). An unrecoverable ID is sentinelled + instead, the way a too-short one already was, so the pipeline never + publishes an identity no tracker contains. + + Every malformed ID is reported to the error collector whether it was + recovered or sentinelled, since either way the source workbook needs + correcting. + + This function should be called LAST in the validation pipeline because + other functions use patient_id for error logging. + + Args: + df: Input DataFrame + error_collector: ErrorCollector for tracking validation errors + patient_id_col: Column name for patient ID (default: "patient_id") + + Returns: + DataFrame with validated/fixed patient IDs + + Example: + >>> df = fix_patient_id(df, error_collector) + >>> # "KD_EW004" → "KD_EW004" (valid) + >>> # "KD-EW004" → "KD_EW004" (normalized) + >>> # "KH_NPH026" → "KH_NP026" (recovered, if the tracker has one) + >>> # "KD_EW004XY" → "Undefined" (no candidate) + """ + import re + + from a4d.config import settings + + if patient_id_col not in df.columns: + return df + + # Store original values for error reporting + original_col = f"{patient_id_col}_original" + df = df.with_columns(pl.col(patient_id_col).alias(original_col)) + + # Valid format: XX_YY### (2 letters, underscore, 2 letters, 3 digits) + valid_pattern = re.compile(r"^[A-Z]{2}_[A-Z]{2}\d{3}$") + + # The recovery universe is this tracker's own well-formed IDs. The Patient + # List is joined onto the monthly rows during extraction rather than kept + # as rows of its own, so an ID misspelled on every sheet of a tracker is + # not recoverable here -- it is reported and sentinelled instead. + known_ids = { + value.replace("-", "_") + for value in df[patient_id_col].drop_nulls().unique().to_list() + if valid_pattern.match(value.replace("-", "_")) + } + + def fix_single_id(patient_id: str | None) -> str | None: + """Fix a single patient ID value.""" + if patient_id is None: + return None + + # Step 1: Replace hyphens with underscores + patient_id = patient_id.replace("-", "_") + + # Step 2: Check if it matches the valid pattern + if valid_pattern.match(patient_id): + return patient_id + + # Step 3: Invalid format - recover from the tracker's own spelling, + # or sentinel. Never truncate: see the divergence note above. + return recover_patient_id(patient_id, known_ids) or settings.error_val_character + + # Apply transformation + df = df.with_columns( + pl.col(patient_id_col) + .map_elements(fix_single_id, return_dtype=pl.String) + .alias(patient_id_col) + ) + + # Now collect errors for changed values + for row in df.iter_rows(named=True): + original = row[original_col] + fixed = row[patient_id_col] + + if original != fixed and original is not None: + # Normalize original to check if it's just hyphen replacement + normalized = original.replace("-", "_") + + if normalized != fixed: + if fixed != settings.error_val_character: + error_collector.add_error( + file_name="", + patient_id=fixed, + column=patient_id_col, + original_value=original, + error_message=( + f"Patient ID {original!r} is malformed; recovered as {fixed!r} " + f"from this tracker's own spelling. The source workbook " + f"needs correcting." + ), + error_code="invalid_value", + ) + else: + error_collector.add_error( + file_name="", + patient_id=original, + column=patient_id_col, + original_value=original, + error_message=( + "Invalid patient ID format (expected XX_YY###) and no " + "unambiguous match in this tracker" + ), + error_code="invalid_value", + ) + + # Drop the temporary column + df = df.drop(original_col) + + return df diff --git a/src/a4d/cli.py b/src/a4d/cli.py new file mode 100644 index 0000000..9357f1e --- /dev/null +++ b/src/a4d/cli.py @@ -0,0 +1,1569 @@ +"""Command-line interface for A4D pipeline.""" + +import warnings +from datetime import datetime +from pathlib import Path +from typing import Annotated + +import polars as pl +import typer +import typer.core +from rich.console import Console +from rich.table import Table + +from a4d.pipeline.patient import ( + discover_tracker_files, + process_patient_tables, + run_patient_pipeline, +) +from a4d.pipeline.product import process_product_tables, run_product_pipeline +from a4d.state import filter_unchanged_trackers, load_previous_manifest +from a4d.tables.errors import create_table_errors +from a4d.tables.logs import create_table_logs + +# google-crc32c has no pre-built C wheel for Python 3.14 yet; the pure-Python +# fallback is correct, just slightly slower. Suppress the noisy runtime warning +# before any google SDK calls are made (those happen lazily inside commands). +warnings.filterwarnings( + "ignore", message="As the c extension couldn't be imported", category=RuntimeWarning +) + + +class _SortedTyperGroup(typer.core.TyperGroup): + """Lists subcommands alphabetically instead of Typer's default creation order.""" + + def list_commands(self, ctx) -> list[str]: + return sorted(super().list_commands(ctx)) + + +app = typer.Typer( + name="a4d", + help="A4D medical tracker data processing pipeline", + no_args_is_help=True, + cls=_SortedTyperGroup, +) + +# Command groups, named after the process they perform (run / create / upload / +# download), not the object they act on — `a4d run patient`, not `a4d patient run`. +# `run_app` uses invoke_without_command so bare `a4d run` (no subcommand) runs the +# full pipeline, while `a4d run patient` / `a4d run product` run a single arm. +create_app = typer.Typer(help="Rebuild tables from existing cleaned output.", cls=_SortedTyperGroup) +download_app = typer.Typer( + help="Download pipeline input (trackers, reference data).", cls=_SortedTyperGroup +) +run_app = typer.Typer( + help="Run the pipeline: full run by default, or a single arm.", + invoke_without_command=True, + cls=_SortedTyperGroup, +) +upload_app = typer.Typer(help="Upload pipeline output (tables, files).", cls=_SortedTyperGroup) + +app.add_typer(create_app, name="create") +app.add_typer(download_app, name="download") +app.add_typer(run_app, name="run") +app.add_typer(upload_app, name="upload") + +console = Console() + + +def _display_tables_summary(tables: dict[str, Path]) -> None: + """Display summary table of created tables with record counts. + + Args: + tables: Dictionary mapping table name to output path + """ + if not tables: + return + + console.print("\n[bold green]Created Tables:[/bold green]") + tables_table = Table(title="Created Tables") + tables_table.add_column("Table", style="cyan") + tables_table.add_column("Path", style="green") + tables_table.add_column("Records", justify="right", style="magenta") + + # Add patient tables first, then product, then logs/errors/clinic/metadata + for name in [ + "static", + "monthly", + "annual", + "product_data", + "logs", + "errors", + "clinic_data_static", + "tracker_metadata", + ]: + if name in tables: + path = tables[name] + try: + record_count = f"{pl.read_parquet(path).__len__():,}" + except Exception: + record_count = "?" + tables_table.add_row(name, str(path.name), record_count) + + console.print(tables_table) + console.print() + + +def _render_pipeline_header( + data_root: str | Path, + output_root: str | Path, + workers: int, + *, + skip_tables: bool = False, + extras: list[tuple[str, str]] | None = None, +) -> None: + """Render the per-command header banner. + + `extras` carries the bare `run`-only fields (Project / Dataset / Drive / + Download / Upload / Product) so run patient and run product can + omit them. All labels are padded to a 13-character column to match the + pre-refactor output exactly. + """ + rows: list[tuple[str, str]] = [ + ("Data root", str(data_root)), + ("Output root", str(output_root)), + ("Workers", str(workers)), + ] + if skip_tables: + rows.append(("Tables", "skipped")) + if extras: + rows.extend(extras) + for label, value in rows: + console.print(f"{label + ':':<13}{value}") + console.print() + + +def _render_pipeline_results_summary( + result, + tables: dict[str, Path], + total_errors: int, + files_with_errors: int, +) -> None: + """Render the 7-row Summary table used by run patient / run product.""" + summary_table = Table(title="Summary") + summary_table.add_column("Metric", style="cyan") + summary_table.add_column("Value", style="green") + + summary_table.add_row("Total Trackers", str(result.total_trackers)) + summary_table.add_row("Successful", str(result.successful_trackers)) + summary_table.add_row("Failed", str(result.failed_trackers)) + summary_table.add_row("Tables Created", str(len(tables))) + summary_table.add_row("", "") + summary_table.add_row("Data Quality Errors", f"{total_errors:,}") + summary_table.add_row("Files with Errors", str(files_with_errors)) + + console.print(summary_table) + + +def _render_error_type_breakdown(result, total_errors: int) -> None: + """Render the per-error-type frequency table used by run patient / run product. + + No-op when there are no errors. + """ + if total_errors <= 0: + return + + console.print("\n[bold yellow]Error Type Breakdown:[/bold yellow]") + + error_type_totals: dict[str, int] = {} + for tr in result.tracker_results: + if tr.error_breakdown: + for error_type, count in tr.error_breakdown.items(): + error_type_totals[error_type] = error_type_totals.get(error_type, 0) + count + + error_type_table = Table() + error_type_table.add_column("Error Type", style="yellow") + error_type_table.add_column("Count", justify="right", style="red") + error_type_table.add_column("Percentage", justify="right", style="cyan") + + sorted_error_types = sorted(error_type_totals.items(), key=lambda x: x[1], reverse=True) + for error_type, count in sorted_error_types: + percentage = (count / total_errors) * 100 + error_type_table.add_row(error_type, f"{count:,}", f"{percentage:.1f}%") + + console.print(error_type_table) + + +def _render_top_files_by_error(result, total_errors: int) -> None: + """Render the top-10-files-by-error-count table used by run patient / run product. + + No-op when there are no errors. + """ + if total_errors <= 0: + return + + console.print("\n[bold yellow]Top Files by Error Count:[/bold yellow]") + + files_by_errors = sorted( + [ + (tr.tracker_file.name, tr.cleaning_errors) + for tr in result.tracker_results + if tr.cleaning_errors > 0 + ], + key=lambda x: x[1], + reverse=True, + )[:10] + + errors_table = Table() + errors_table.add_column("File", style="yellow") + errors_table.add_column("Errors", justify="right", style="red") + for filename, error_count in files_by_errors: + errors_table.add_row(filename, f"{error_count:,}") + + console.print(errors_table) + + +def _resolve_tracker_files( + file: Path | None, + data_root_arg: Path | None, + incremental: bool, + output_root: Path, +) -> tuple[list[Path] | None, str]: + """Resolve the tracker-file list for a CLI invocation. + + Returns ``(tracker_files, display_str)``. ``tracker_files`` is ``None`` when + the orchestrator should discover trackers itself (the default + non-incremental "process everything in data_root" path). An empty list means + discovery + incremental filtering produced no work; the caller should + short-circuit. + + --file always wins; --incremental + --file is a no-op (logged warning), + matching the design that single-file is an explicit user override. + """ + if file: + if incremental: + console.print("[yellow]Warning: --incremental is ignored when --file is set[/yellow]") + return [file], f"{file} (single file)" + + from a4d.config import settings as _settings + + if data_root_arg is not None: + files = discover_tracker_files(data_root_arg) + if not files: + console.print( + f"[bold red]Error: No tracker files found in {data_root_arg}[/bold red]\n" + ) + raise typer.Exit(1) + display = str(data_root_arg) + elif incremental: + files = discover_tracker_files(_settings.data_root) + display = str(_settings.data_root) + else: + # Default: orchestrator discovers everything from settings.data_root. + return None, str(_settings.data_root) + + if incremental: + manifest = load_previous_manifest(output_root) + files, summary = filter_unchanged_trackers(files, manifest) + console.print( + f"[cyan]Incremental filter: queued {summary.queued}, " + f"skipped {summary.skipped} unchanged " + f"(new={summary.new}, changed={summary.changed}, " + f"incomplete={summary.previously_incomplete})[/cyan]\n" + ) + + return files, display + + +# Output table files worth reporting row counts for, in display order. +_KNOWN_TABLE_FILES = [ + "patient_data_static.parquet", + "patient_data_monthly.parquet", + "patient_data_annual.parquet", + "product_data.parquet", + "clinic_data_static.parquet", + "table_logs.parquet", + "table_errors.parquet", + "tracker_metadata.parquet", +] + + +def _scan_table_row_counts(tables_dir: Path) -> list[tuple[str, int]]: + """Row counts for whichever known output tables exist in `tables_dir`. + + Uses a lazy scan + `pl.len()` rather than reading each table fully into + memory — these can be large (hundreds of thousands of rows for logs). + """ + counts = [] + for filename in _KNOWN_TABLE_FILES: + path = tables_dir / filename + if path.exists(): + count = pl.scan_parquet(path).select(pl.len()).collect().item() + counts.append((filename, count)) + return counts + + +def _scan_distinct_count(path: Path, column: str) -> int | None: + """Distinct non-null value count for `column` in a parquet file, or None if missing.""" + if not path.exists(): + return None + lf = pl.scan_parquet(path) + if column not in lf.collect_schema().names(): + return None + return lf.select(pl.col(column).n_unique()).collect().item() + + +def _render_dataset_overview(tables_dir: Path) -> None: + """Render row counts for every output table plus distinct clinic/patient counts. + + The sanity-check signal for "does this run look right" — e.g. a clinic + count that suddenly drops is worth catching here, before anything is + uploaded. + """ + row_counts = _scan_table_row_counts(tables_dir) + if not row_counts: + return + + console.print("\n[bold blue]Dataset Overview:[/bold blue]") + overview_table = Table() + overview_table.add_column("Table", style="cyan") + overview_table.add_column("Rows", justify="right", style="green") + for filename, count in row_counts: + overview_table.add_row(filename, f"{count:,}") + + distinct_clinics = _scan_distinct_count(tables_dir / "clinic_data_static.parquet", "clinic_id") + distinct_patients = _scan_distinct_count( + tables_dir / "patient_data_static.parquet", "patient_id" + ) + if distinct_clinics is not None: + overview_table.add_row("[dim]distinct clinics[/dim]", f"[dim]{distinct_clinics:,}[/dim]") + if distinct_patients is not None: + overview_table.add_row("[dim]distinct patients[/dim]", f"[dim]{distinct_patients:,}[/dim]") + console.print(overview_table) + + +def _render_combined_run_summary(patient_result, product_result, tables_dir: Path) -> None: + """Render a combined patient+product view of a full `run` execution. + + Crosses each tracker file's patient and product outcome (both ok / one + arm failed / lost entirely) and merges both arms' per-file error counts. + Only meaningful once both arms have actually run — silently skips + the outcome/error tables otherwise (e.g. --skip-patient / --skip-product, + or the product arm crashing before producing a result). The dataset + overview still renders regardless, since it reads from disk rather than + depending on both arms' in-memory results. + """ + if patient_result is None or product_result is None: + _render_dataset_overview(tables_dir) + return + + patient_by_name = {tr.tracker_name: tr for tr in patient_result.tracker_results} + product_by_name = {tr.tracker_name: tr for tr in product_result.tracker_results} + all_names = sorted(set(patient_by_name) | set(product_by_name)) + + both_ok = patient_only_failed = product_only_failed = both_failed = 0 + attention_rows: list[tuple[str, str, str]] = [] + error_counts: dict[str, tuple[int, int]] = {} + + for name in all_names: + p = patient_by_name.get(name) + q = product_by_name.get(name) + p_ok = p.success if p is not None else True + q_ok = q.success if q is not None else True + + if p_ok and q_ok: + both_ok += 1 + else: + p_err = p.error if p is not None and not p_ok else None + q_err = q.error if q is not None and not q_ok else None + attention_rows.append((name, p_err or "-", q_err or "-")) + if not p_ok and not q_ok: + both_failed += 1 + elif not p_ok: + patient_only_failed += 1 + else: + product_only_failed += 1 + + patient_errors = p.cleaning_errors if p else 0 + product_errors = q.cleaning_errors if q else 0 + if patient_errors or product_errors: + error_counts[name] = (patient_errors, product_errors) + + console.print("\n[bold blue]Combined Run Summary[/bold blue]") + summary_table = Table() + summary_table.add_column("Outcome", style="cyan") + summary_table.add_column("Files", justify="right", style="green") + summary_table.add_row("Both arms OK", str(both_ok)) + summary_table.add_row("Patient failed only", str(patient_only_failed)) + summary_table.add_row("Product failed only", str(product_only_failed)) + summary_table.add_row("Lost entirely (both arms failed)", str(both_failed)) + console.print(summary_table) + + if attention_rows: + console.print("\n[bold yellow]Files needing attention:[/bold yellow]") + detail_table = Table() + detail_table.add_column("File", style="red") + detail_table.add_column("Patient error") + detail_table.add_column("Product error") + for name, p_err, q_err in attention_rows: + detail_table.add_row(name, p_err, q_err) + console.print(detail_table) + + if error_counts: + console.print("\n[bold yellow]Top Files by Error Count:[/bold yellow]") + top_table = Table() + top_table.add_column("File", style="cyan") + top_table.add_column("Patient", justify="right", style="green") + top_table.add_column("Product", justify="right", style="yellow") + top_table.add_column("Total", justify="right", style="magenta") + ranked = sorted(error_counts.items(), key=lambda kv: -(kv[1][0] + kv[1][1])) + for name, (patient_errors, product_errors) in ranked[:20]: + top_table.add_row( + name, + str(patient_errors), + str(product_errors), + str(patient_errors + product_errors), + ) + console.print(top_table) + + _render_dataset_overview(tables_dir) + + +def _render_failed_trackers( + result, + *, + mode: str, + truncate: int | None = 100, + title: str = "Failed Trackers", + leading_newline: bool = True, +) -> None: + """Render the failed-trackers section. + + `mode="table"` matches run patient / run product (Rich Table, + error truncated). `mode="bullets"` matches `run` (bullet list, + full error). `truncate` is ignored in bullets mode. + """ + if result.failed_trackers <= 0: + return + prefix = "\n" if leading_newline else "" + console.print(f"{prefix}[bold yellow]{title}:[/bold yellow]") + if mode == "table": + failed_table = Table() + failed_table.add_column("File", style="red") + failed_table.add_column("Error") + for tr in result.tracker_results: + if not tr.success: + error_text = str(tr.error) + if truncate is not None: + error_text = error_text[:truncate] + failed_table.add_row(tr.tracker_file.name, error_text) + console.print(failed_table) + else: # bullets + for tr in result.tracker_results: + if not tr.success: + console.print(f" • {tr.tracker_file.name}: {tr.error}") + console.print() + + +@run_app.command("patient") +def run_patient_cmd( + file: Annotated[ + Path | None, + typer.Option( + "--file", + "-f", + help="Process specific tracker file (if not set, processes all files in data_root)", + ), + ] = None, + workers: Annotated[ + int | None, + typer.Option( + "--workers", "-w", help="Number of parallel workers (default: A4D_MAX_WORKERS)" + ), + ] = None, + skip_tables: Annotated[ + bool, typer.Option("--skip-tables", help="Skip table creation (only extract + clean)") + ] = False, + data_root: Annotated[ + Path | None, + typer.Option( + "--data-root", "-d", help="Directory containing tracker files (default: from config)" + ), + ] = None, + output_root: Annotated[ + Path | None, typer.Option("--output", "-o", help="Output directory (default: from config)") + ] = None, + incremental: Annotated[ + bool, + typer.Option( + "--incremental", + help=( + "Skip trackers whose MD5 + completion state match the previous " + "run's manifest. Preserves prior outputs (clean_output disabled)." + ), + ), + ] = False, + force: Annotated[ + bool, + typer.Option( + "--force", + help=( + "Wipe prior outputs and reprocess every tracker. Same as the " + "default behavior; pass explicitly for self-documenting deploy " + "commands. Overrides --incremental if both are passed." + ), + ), + ] = False, +): + """Process patient data pipeline. + + \b + By default, output is cleaned before each run so tables reflect only the + current run's files. With --incremental, prior outputs are preserved and + only new/changed/previously-incomplete trackers are re-processed. + With --force, behaves as the default (wipe + reprocess) and overrides + --incremental if both are passed. + + Examples: + # Process all trackers in data_root (from config) + uv run a4d run patient + + # Process all trackers in a specific directory + uv run a4d run patient --data-root /path/to/trackers + + # Process specific file + uv run a4d run patient --file /path/to/tracker.xlsx + + # Parallel processing with 8 workers + uv run a4d run patient --workers 8 + + # Just extract + clean, skip tables + uv run a4d run patient --skip-tables + + # Skip trackers whose MD5 matches the previous run's manifest + uv run a4d run patient --incremental + + # Explicitly wipe outputs and reprocess everything + uv run a4d run patient --force + """ + from a4d.config import settings as _settings + + console.print("\n[bold blue]A4D Patient Pipeline[/bold blue]\n") + + if force and incremental: + console.print("[yellow]Warning: --incremental is ignored when --force is set[/yellow]") + incremental = False + + _output_root = output_root or _settings.output_root + _workers = workers if workers is not None else _settings.max_workers + + tracker_files, data_root_display = _resolve_tracker_files( + file, data_root, incremental, _output_root + ) + + if tracker_files is not None and len(tracker_files) == 0: + console.print("[bold green]✓ No trackers need reprocessing — exiting[/bold green]\n") + raise typer.Exit(0) + + _render_pipeline_header(data_root_display, _output_root, _workers, skip_tables=skip_tables) + + # Step 1: Extract + clean (table creation handled below for visible progress) + console.print("[bold]Step 1/4:[/bold] Extracting and cleaning tracker files...") + try: + result = run_patient_pipeline( + tracker_files=tracker_files, + max_workers=_workers, + output_root=output_root, + skip_tables=True, # tables created below with console feedback + clean_output=force + or not incremental, # incremental keeps prior outputs; --force always wipes + show_progress=True, + console_log_level="ERROR", + ) + except Exception as e: + console.print(f"\n[bold red]Error: {e}[/bold red]\n") + raise typer.Exit(1) from e + + # Steps 2-4: Table and log/error creation with console feedback + tables: dict[str, Path] = {} + if not skip_tables and result.successful_trackers > 0: + cleaned_dir = _output_root / "patient_data_cleaned" + tables_dir = _output_root / "tables" + logs_dir = _output_root / "logs" + + console.print("[bold]Step 2/4:[/bold] Creating patient tables...") + try: + tables = process_patient_tables(cleaned_dir, tables_dir) + except Exception as e: + console.print(f"[bold red]Error creating tables: {e}[/bold red]") + + if logs_dir.exists(): + console.print("[bold]Step 3/4:[/bold] Creating logs table...") + try: + logs_table_path = create_table_logs(logs_dir, tables_dir) + tables["logs"] = logs_table_path + except Exception as e: + console.print(f"[bold red]Error creating logs table: {e}[/bold red]") + + console.print("[bold]Step 4/4:[/bold] Creating errors table...") + try: + all_data_errors = [e for r in result.tracker_results for e in r.data_errors] + errors_table_path = create_table_errors(all_data_errors, tables_dir) + tables["errors"] = errors_table_path + except Exception as e: + console.print(f"[bold red]Error creating errors table: {e}[/bold red]") + elif skip_tables: + console.print("[dim]Steps 2–3: Skipped (--skip-tables)[/dim]") + console.print("[bold]Step 4/4:[/bold] Creating errors table...") + try: + tables_dir = _output_root / "tables" + all_data_errors = [e for r in result.tracker_results for e in r.data_errors] + errors_table_path = create_table_errors(all_data_errors, tables_dir) + tables["errors"] = errors_table_path + except Exception as e: + console.print(f"[bold red]Error creating errors table: {e}[/bold red]") + + # Display results + console.print("\n[bold]Pipeline Results[/bold]\n") + + # Calculate error statistics + total_errors = sum(tr.cleaning_errors for tr in result.tracker_results) + files_with_errors = sum(1 for tr in result.tracker_results if tr.cleaning_errors > 0) + + _render_pipeline_results_summary(result, tables, total_errors, files_with_errors) + _render_error_type_breakdown(result, total_errors) + _render_failed_trackers(result, mode="table") + _render_top_files_by_error(result, total_errors) + + # Show created tables + _display_tables_summary(tables) + + # Exit status + if result.success: + console.print("\n[bold green]✓ Pipeline completed successfully![/bold green]\n") + raise typer.Exit(0) + else: + console.print( + f"\n[bold red]✗ Pipeline completed with {result.failed_trackers} failures[/bold red]\n" + ) + raise typer.Exit(1) + + +@create_app.command("tables") +def create_tables_cmd( + output_root: Annotated[ + Path | None, + typer.Option( + "--output", "-o", help="Pipeline output root directory (default: from config)" + ), + ] = None, +): + """Rebuild all tables from existing cleaned output, without re-running extract/clean. + + Reads whichever of `patient_data_cleaned/`, `product_data_cleaned/`, and + `logs/` exist under the output root and rebuilds the corresponding + tables (patient static/monthly/annual, product, logs), plus the clinic + static table and tracker metadata table. At least one of the patient or + product cleaned directories must exist. + + Useful for: + - Re-creating tables after fixing table creation logic + - Creating tables from manually cleaned data + - Testing table creation independently + + \\b + Examples: + # Rebuild tables from the default output root (from config) + uv run a4d create tables + + # Rebuild from a specific output root + uv run a4d create tables --output /path/to/output + """ + from a4d.config import settings as _settings + from a4d.tables.clinic import create_table_clinic_static + from a4d.tables.metadata import create_table_tracker_metadata + + console.print("\n[bold blue]A4D Table Creation[/bold blue]\n") + + _output_root = output_root or _settings.output_root + patient_cleaned_dir = _output_root / "patient_data_cleaned" + product_cleaned_dir = _output_root / "product_data_cleaned" + logs_dir = _output_root / "logs" + tables_dir = _output_root / "tables" + + console.print(f"Output root: {_output_root}") + console.print(f"Tables directory: {tables_dir}\n") + + if not patient_cleaned_dir.exists() and not product_cleaned_dir.exists(): + console.print( + f"[bold red]Error: Neither {patient_cleaned_dir} nor {product_cleaned_dir} " + "exists — nothing to build tables from[/bold red]\n" + ) + raise typer.Exit(1) + + try: + tables: dict[str, Path] = {} + + if patient_cleaned_dir.exists(): + console.print(" • Creating patient tables...") + tables.update(process_patient_tables(patient_cleaned_dir, tables_dir)) + else: + console.print( + f" [yellow]Warning: {patient_cleaned_dir} not found, " + "skipping patient tables[/yellow]" + ) + + if product_cleaned_dir.exists(): + console.print(" • Creating product table...") + tables.update(process_product_tables(product_cleaned_dir, tables_dir)) + else: + console.print( + f" [yellow]Warning: {product_cleaned_dir} not found, " + "skipping product table[/yellow]" + ) + + if logs_dir.exists(): + console.print(" • Creating logs table...") + tables["logs"] = create_table_logs(logs_dir, tables_dir) + else: + console.print(f" [yellow]Warning: Logs directory not found at {logs_dir}[/yellow]") + + # Create clinic static table (reads reference_data/clinic_data.xlsx) + console.print(" • Creating clinic static table...") + tables["clinic_data_static"] = create_table_clinic_static(tables_dir) + + # Create tracker metadata table (MD5 + per-tracker output presence). + # Skipped if settings.data_root is unreachable — the table needs the + # raw .xlsx files, which create tables doesn't otherwise require. + if _settings.data_root.exists(): + console.print(" • Creating tracker metadata table...") + tables["tracker_metadata"] = create_table_tracker_metadata( + _settings.data_root, _output_root + ) + else: + console.print( + f" [yellow]Warning: data_root {_settings.data_root} not found, " + "skipping tracker metadata[/yellow]" + ) + + # Display results + console.print("\n[bold green]✓ Tables created successfully![/bold green]") + _display_tables_summary(tables) + + except Exception as e: + console.print(f"\n[bold red]Error creating tables: {e}[/bold red]\n") + raise typer.Exit(1) from e + + +@create_app.command("logs") +def create_logs_cmd( + output_root: Annotated[ + Path | None, + typer.Option( + "--output", "-o", help="Pipeline output root directory (default: from config)" + ), + ] = None, +): + """Rebuild only the logs table from existing pipeline log files. + + Faster than `create tables` when only the logs table is needed — e.g. + iterating on a log-analysis tool without rebuilding patient/product + tables too. + + \\b + Examples: + uv run a4d create logs + uv run a4d create logs --output /path/to/output + """ + from a4d.config import settings as _settings + + console.print("\n[bold blue]A4D Logs Table Creation[/bold blue]\n") + + _output_root = output_root or _settings.output_root + logs_dir = _output_root / "logs" + tables_dir = _output_root / "tables" + + if not logs_dir.exists(): + console.print(f"[bold red]Error: Logs directory not found: {logs_dir}[/bold red]\n") + raise typer.Exit(1) + + try: + logs_table_path = create_table_logs(logs_dir, tables_dir) + console.print(f"[bold green]✓ Logs table created: {logs_table_path}[/bold green]\n") + except Exception as e: + console.print(f"\n[bold red]Error creating logs table: {e}[/bold red]\n") + raise typer.Exit(1) from e + + +@run_app.command("product") +def run_product_cmd( + file: Annotated[ + Path | None, + typer.Option( + "--file", + "-f", + help="Process specific tracker file (if not set, processes all files in data_root)", + ), + ] = None, + workers: Annotated[ + int | None, + typer.Option( + "--workers", "-w", help="Number of parallel workers (default: A4D_MAX_WORKERS)" + ), + ] = None, + skip_tables: Annotated[ + bool, typer.Option("--skip-tables", help="Skip table creation (only extract + clean)") + ] = False, + data_root: Annotated[ + Path | None, + typer.Option( + "--data-root", "-d", help="Directory containing tracker files (default: from config)" + ), + ] = None, + output_root: Annotated[ + Path | None, typer.Option("--output", "-o", help="Output directory (default: from config)") + ] = None, + incremental: Annotated[ + bool, + typer.Option( + "--incremental", + help=( + "Skip trackers whose MD5 + completion state match the previous " + "run's manifest. Preserves prior outputs (clean_output disabled)." + ), + ), + ] = False, + force: Annotated[ + bool, + typer.Option( + "--force", + help=( + "Wipe prior outputs and reprocess every tracker. Same as the " + "default behavior; pass explicitly for self-documenting deploy " + "commands. Overrides --incremental if both are passed." + ), + ), + ] = False, +): + """Process product data pipeline. + + \b + By default, output is cleaned before each run so tables reflect only the + current run's files. With --incremental, prior outputs are preserved and + only new/changed/previously-incomplete trackers are re-processed. + With --force, behaves as the default (wipe + reprocess) and overrides + --incremental if both are passed. + + Examples: + # Process all trackers in data_root (from config) + uv run a4d run product + + # Process specific file + uv run a4d run product --file /path/to/tracker.xlsx + + # Parallel processing with 8 workers + uv run a4d run product --workers 8 + + # Just extract + clean, skip tables + uv run a4d run product --skip-tables + + # Skip trackers whose MD5 matches the previous run's manifest + uv run a4d run product --incremental + + # Explicitly wipe outputs and reprocess everything + uv run a4d run product --force + """ + from a4d.config import settings as _settings + + console.print("\n[bold blue]A4D Product Pipeline[/bold blue]\n") + + if force and incremental: + console.print("[yellow]Warning: --incremental is ignored when --force is set[/yellow]") + incremental = False + + _output_root = output_root or _settings.output_root + _workers = workers if workers is not None else _settings.max_workers + + tracker_files, data_root_display = _resolve_tracker_files( + file, data_root, incremental, _output_root + ) + + if tracker_files is not None and len(tracker_files) == 0: + console.print("[bold green]✓ No trackers need reprocessing — exiting[/bold green]\n") + raise typer.Exit(0) + + _render_pipeline_header(data_root_display, _output_root, _workers, skip_tables=skip_tables) + + console.print("[bold]Step 1/4:[/bold] Extracting and cleaning product data...") + try: + result = run_product_pipeline( + tracker_files=tracker_files, + max_workers=_workers, + output_root=output_root, + skip_tables=True, + clean_output=force + or not incremental, # incremental keeps prior outputs; --force always wipes + show_progress=True, + console_log_level="ERROR", + ) + except Exception as e: + console.print(f"\n[bold red]Error: {e}[/bold red]\n") + raise typer.Exit(1) from e + + # Steps 2-4 mirror process_patient_cmd: table, then logs, then errors. + tables: dict[str, Path] = {} + if not skip_tables and result.successful_trackers > 0: + cleaned_dir = _output_root / "product_data_cleaned" + tables_dir = _output_root / "tables" + logs_dir = _output_root / "logs" + + console.print("[bold]Step 2/4:[/bold] Creating product table...") + try: + tables = process_product_tables(cleaned_dir, tables_dir) + except Exception as e: + console.print(f"[bold red]Error creating tables: {e}[/bold red]") + + if logs_dir.exists(): + console.print("[bold]Step 3/4:[/bold] Creating logs table...") + try: + logs_table_path = create_table_logs(logs_dir, tables_dir) + tables["logs"] = logs_table_path + except Exception as e: + console.print(f"[bold red]Error creating logs table: {e}[/bold red]") + + console.print("[bold]Step 4/4:[/bold] Creating errors table...") + try: + all_data_errors = [e for r in result.tracker_results for e in r.data_errors] + errors_table_path = create_table_errors(all_data_errors, tables_dir) + tables["errors"] = errors_table_path + except Exception as e: + console.print(f"[bold red]Error creating errors table: {e}[/bold red]") + elif skip_tables: + console.print("[dim]Steps 2-3: Skipped (--skip-tables)[/dim]") + console.print("[bold]Step 4/4:[/bold] Creating errors table...") + try: + tables_dir = _output_root / "tables" + all_data_errors = [e for r in result.tracker_results for e in r.data_errors] + errors_table_path = create_table_errors(all_data_errors, tables_dir) + tables["errors"] = errors_table_path + except Exception as e: + console.print(f"[bold red]Error creating errors table: {e}[/bold red]") + + console.print("\n[bold]Pipeline Results[/bold]\n") + + total_errors = sum(tr.cleaning_errors for tr in result.tracker_results) + files_with_errors = sum(1 for tr in result.tracker_results if tr.cleaning_errors > 0) + + _render_pipeline_results_summary(result, tables, total_errors, files_with_errors) + _render_error_type_breakdown(result, total_errors) + _render_failed_trackers(result, mode="table") + _render_top_files_by_error(result, total_errors) + + _display_tables_summary(tables) + + if result.success: + console.print("\n[bold green]✓ Product pipeline completed successfully![/bold green]\n") + raise typer.Exit(0) + else: + console.print( + f"\n[bold red]✗ Product pipeline completed with " + f"{result.failed_trackers} failures[/bold red]\n" + ) + raise typer.Exit(1) + + +@upload_app.command("tables") +def upload_tables_cmd( + tables_dir: Annotated[ + Path, + typer.Option("--tables-dir", "-t", help="Directory containing parquet table files"), + ], + dataset: Annotated[ + str | None, + typer.Option("--dataset", "-d", help="BigQuery dataset name (default: from config)"), + ] = None, + project_id: Annotated[ + str | None, + typer.Option("--project", "-p", help="GCP project ID (default: from config)"), + ] = None, + append: Annotated[ + bool, + typer.Option("--append", help="Append to existing tables instead of replacing"), + ] = False, + only: Annotated[ + list[str] | None, + typer.Option( + "--only", + help="Restrict upload to a table group: patient, product, clinic, logs, " + "errors, metadata. Repeatable; default uploads all groups.", + ), + ] = None, +): + """Upload pipeline output tables to BigQuery. + + Loads parquet files from the tables directory into the configured + BigQuery dataset. By default, existing tables are replaced. + + \b + Examples: + # Upload tables from default output directory + uv run a4d upload tables --tables-dir output/tables + + # Upload to a specific dataset + uv run a4d upload tables --tables-dir output/tables --dataset tracker_dev + + # Append instead of replace + uv run a4d upload tables --tables-dir output/tables --append + + # Only upload the patient tables + uv run a4d upload tables --tables-dir output/tables --only patient + """ + from a4d.gcp.bigquery import TABLE_GROUPS, load_pipeline_tables + + console.print("\n[bold blue]A4D BigQuery Upload[/bold blue]\n") + console.print(f"Tables directory: {tables_dir}") + + if not tables_dir.exists(): + console.print(f"[bold red]Error: Directory not found: {tables_dir}[/bold red]\n") + raise typer.Exit(1) + + only_tables: set[str] | None = None + if only: + invalid = sorted(set(only) - set(TABLE_GROUPS)) + if invalid: + console.print( + f"[bold red]Error: unknown --only group(s): {', '.join(invalid)} " + f"(valid: {', '.join(sorted(TABLE_GROUPS))})[/bold red]\n" + ) + raise typer.Exit(1) + only_tables = set().union(*(TABLE_GROUPS[group] for group in only)) + console.print(f"Restricting to: {', '.join(sorted(only))}") + + try: + results = load_pipeline_tables( + tables_dir=tables_dir, + dataset=dataset, + project_id=project_id, + replace=not append, + only_tables=only_tables, + ) + + if results: + result_table = Table(title="Uploaded Tables") + result_table.add_column("Table", style="cyan") + result_table.add_column("Rows", justify="right", style="green") + result_table.add_column("Status", style="green") + + for table_name, job in results.items(): + result_table.add_row( + table_name, + f"{job.output_rows:,}" if job.output_rows else "?", + "✓", + ) + + console.print(result_table) + console.print( + f"\n[bold green]✓ Uploaded {len(results)} tables to BigQuery[/bold green]\n" + ) + else: + console.print("[bold yellow]No tables found to upload[/bold yellow]\n") + + except Exception as e: + console.print(f"\n[bold red]Error: {e}[/bold red]\n") + raise typer.Exit(1) from e + + +@download_app.command("trackers") +def download_trackers_cmd( + data_root: Annotated[ + Path, + typer.Option( + "--data-root", + "-d", + help="Local directory to download files to — same directory `run patient`/" + "`run product` later read via their own --data-root", + ), + ], + bucket: Annotated[ + str | None, + typer.Option("--bucket", "-b", help="GCS bucket name (default: from config)"), + ] = None, +): + """Download tracker files from Google Cloud Storage. + + \b + Examples: + # Download to local directory + uv run a4d download trackers --data-root /data/trackers + + # Download from specific bucket + uv run a4d download trackers --data-root /data/trackers --bucket my-bucket + """ + from a4d.gcp.storage import download_tracker_files + + console.print("\n[bold blue]A4D Tracker Download[/bold blue]\n") + console.print(f"Data root: {data_root}") + + try: + downloaded = download_tracker_files(destination=data_root, bucket_name=bucket) + console.print(f"\n[bold green]✓ Downloaded {len(downloaded)} files[/bold green]\n") + except Exception as e: + console.print(f"\n[bold red]Error: {e}[/bold red]\n") + raise typer.Exit(1) from e + + +@upload_app.command("output") +def upload_output_cmd( + source_dir: Annotated[ + Path, + typer.Option("--source", "-s", help="Output directory to upload"), + ], + bucket: Annotated[ + str | None, + typer.Option("--bucket", "-b", help="GCS bucket name (default: from config)"), + ] = None, + prefix: Annotated[ + str, + typer.Option("--prefix", help="Prefix for uploaded blob names"), + ] = "", +): + """Upload pipeline output to Google Cloud Storage. + + \b + Examples: + # Upload output directory + uv run a4d upload output --source output/ + + # Upload with prefix + uv run a4d upload output --source output/ --prefix 2024-01 + """ + from a4d.gcp.storage import upload_output + + console.print("\n[bold blue]A4D Output Upload[/bold blue]\n") + console.print(f"Source: {source_dir}") + + if not source_dir.exists(): + console.print(f"[bold red]Error: Directory not found: {source_dir}[/bold red]\n") + raise typer.Exit(1) + + try: + uploaded = upload_output(source_dir=source_dir, bucket_name=bucket, prefix=prefix) + console.print(f"\n[bold green]✓ Uploaded {len(uploaded)} files to GCS[/bold green]\n") + except Exception as e: + console.print(f"\n[bold red]Error: {e}[/bold red]\n") + raise typer.Exit(1) from e + + +@download_app.command("clinic-data") +def download_clinic_data_cmd() -> None: + """Download reference data files from Google Drive. + + Downloads clinic_data.xlsx from Google Drive into the reference_data/ + directory. Uses Application Default Credentials with Drive readonly scope. + + The service account must have at least Viewer access to the file. + """ + from a4d.gcp.drive import download_clinic_data + from a4d.reference.loaders import find_reference_data_dir + + console.print("\n[bold blue]A4D Reference Data Download[/bold blue]\n") + + reference_dir = find_reference_data_dir() + console.print(f"Destination: {reference_dir}\n") + + try: + console.print("Downloading clinic_data.xlsx from Google Drive...") + path = download_clinic_data(reference_dir) + size_kb = path.stat().st_size / 1024 + console.print( + f" [bold green]✓[/bold green] clinic_data.xlsx ({size_kb:.1f} KB) -> {path}\n" + ) + except Exception as e: + console.print(f" [bold red]✗ Download failed: {e}[/bold red]\n") + raise typer.Exit(1) from e + + +@run_app.callback(invoke_without_command=True) +def run_all_cmd( + ctx: typer.Context, + workers: Annotated[ + int | None, + typer.Option( + "--workers", "-w", help="Number of parallel workers (default: A4D_MAX_WORKERS)" + ), + ] = None, + skip_download: Annotated[ + bool, + typer.Option("--skip-download", help="Skip GCS download (use files already in data_root)"), + ] = False, + skip_upload: Annotated[ + bool, + typer.Option("--skip-upload", help="Skip GCS and BigQuery upload steps"), + ] = False, + skip_drive_download: Annotated[ + bool, + typer.Option( + "--skip-drive-download", + help="Skip Google Drive download of reference data (clinic_data.xlsx)", + ), + ] = False, + skip_product: Annotated[ + bool, + typer.Option("--skip-product", help="Skip the product pipeline arm."), + ] = False, + skip_patient: Annotated[ + bool, + typer.Option( + "--skip-patient", + help=( + "Skip the patient pipeline arm. Note: leaves " + "tracker_metadata.complete=False for all trackers, so the " + "next --incremental run will re-queue everything." + ), + ), + ] = False, + incremental: Annotated[ + bool, + typer.Option( + "--incremental", + help=( + "Skip trackers whose MD5 + completion state match the previous " + "run's manifest. Both arms see the same filtered queue." + ), + ), + ] = False, + force: Annotated[ + bool, + typer.Option( + "--force", + help=( + "Wipe prior local outputs (raw, cleaned, tables) before each " + "pipeline arm runs. Without this flag, `run` reuses any " + "existing per-tracker parquets on disk. Overrides --incremental " + "if both are passed." + ), + ), + ] = False, +): + """Run the full end-to-end A4D pipeline (default), or a single arm. + + Bare `a4d run` executes all pipeline stages in sequence: + 0. Download reference data (clinic_data.xlsx) from Google Drive + 1. Download tracker files from Google Cloud Storage + 2. Extract and clean all tracker files + 3. Create final tables (static, monthly, annual, clinic) + 4. Upload output files to Google Cloud Storage + 5. Ingest tables into BigQuery + + Use `a4d run patient` or `a4d run product` to run a single arm locally + without any of the GCS/Drive/BigQuery steps. + + All configuration is read from environment variables (A4D_*) or a .env file. + + \b + Examples: + # Full pipeline (download + process + upload) + uv run a4d run + + # Download latest files, process locally, skip upload + uv run a4d run --skip-upload + + # Process local files only, no download or upload + uv run a4d run --skip-download --skip-upload + + # Skip Drive download if clinic_data.xlsx is already current + uv run a4d run --skip-drive-download + + # Wipe prior outputs before each arm runs + uv run a4d run --force + """ + if ctx.invoked_subcommand is not None: + return + + from a4d.config import settings + from a4d.gcp.bigquery import load_pipeline_tables + from a4d.gcp.drive import download_clinic_data + from a4d.gcp.storage import download_tracker_files, upload_output + from a4d.reference.loaders import find_reference_data_dir + from a4d.tables.clinic import create_table_clinic_static + + if skip_patient and skip_product: + console.print( + "[bold red]Error: --skip-patient and --skip-product are mutually exclusive[/bold red]\n" + ) + raise typer.Exit(1) + + if force and incremental: + console.print("[yellow]Warning: --incremental is ignored when --force is set[/yellow]") + incremental = False + + _workers = workers if workers is not None else settings.max_workers + run_ts = datetime.now().strftime("%Y/%m/%d/%H%M%S") + + console.print("\n[bold blue]A4D Full Pipeline[/bold blue]\n") + extras = [ + ("Project", str(settings.project_id)), + ("Dataset", str(settings.dataset)), + ("Drive", "yes" if not skip_drive_download else "skipped (--skip-drive-download)"), + ("Download", "yes" if not skip_download else "skipped (--skip-download)"), + ("Upload", "yes" if not skip_upload else "skipped (--skip-upload)"), + ("Product", "yes" if not skip_product else "skipped (--skip-product)"), + ("Patient", "yes" if not skip_patient else "skipped (--skip-patient)"), + ("Incremental", "yes" if incremental else "no"), + ("Force", "yes" if force else "no"), + ] + _render_pipeline_header(settings.data_root, settings.output_root, _workers, extras=extras) + + # Step 0 – Download reference data from Google Drive + if not skip_drive_download: + console.print("[bold]Step 0/5:[/bold] Downloading reference data from Google Drive...") + try: + reference_dir = find_reference_data_dir() + path = download_clinic_data(reference_dir) + size_kb = path.stat().st_size / 1024 + console.print(f" ✓ clinic_data.xlsx ({size_kb:.1f} KB)\n") + except Exception as e: + console.print(f"\n[bold red]Error downloading reference data: {e}[/bold red]\n") + raise typer.Exit(1) from e + else: + console.print("[bold]Step 0/5:[/bold] Skipping Drive download (--skip-drive-download)\n") + + # Step 1 – Download tracker files from GCS + if not skip_download: + console.print("[bold]Step 1/5:[/bold] Downloading tracker files from GCS...") + try: + downloaded = download_tracker_files(destination=settings.data_root) + console.print(f" ✓ Downloaded {len(downloaded)} files\n") + except Exception as e: + console.print(f"\n[bold red]Error during download: {e}[/bold red]\n") + raise typer.Exit(1) from e + else: + console.print("[bold]Step 1/5:[/bold] Skipping GCS download (--skip-download)\n") + + # Resolve the tracker queue once. Without --incremental, pass None and let + # each orchestrator discover. With --incremental, discover + filter here so + # both arms see the same queue (single manifest load, coherent skip). + shared_tracker_files: list[Path] | None = None + if incremental: + all_trackers = discover_tracker_files(settings.data_root) + manifest = load_previous_manifest(settings.output_root) + shared_tracker_files, summary = filter_unchanged_trackers(all_trackers, manifest) + console.print( + f"[cyan]Incremental filter: queued {summary.queued}, " + f"skipped {summary.skipped} unchanged " + f"(new={summary.new}, changed={summary.changed}, " + f"incomplete={summary.previously_incomplete})[/cyan]\n" + ) + if not shared_tracker_files: + console.print( + "[bold green]✓ No trackers need reprocessing — exiting cleanly[/bold green]\n" + ) + raise typer.Exit(0) + + # Step 2+3 – Extract, clean and build tables. + # clean_output wiring is `force` here, not `force or not incremental` like + # run patient/run product. Reason: the bare `run`'s historical default + # (on `migration` and on this branch pre-change) was preserve-outputs — it + # never passed clean_output, inheriting the orchestrator's False default. + # --force on `migration` was a vestigial no-op (declared, plumbed, never + # read). With --force now actually wired through, opting in wipes both arms; + # without it, `run` keeps its prior preserve-outputs contract. + result = None + product_result = None + if not skip_patient: + console.print("[bold]Steps 2–3/5:[/bold] Processing tracker files...\n") + try: + result = run_patient_pipeline( + tracker_files=shared_tracker_files, + max_workers=_workers, + clean_output=force, + show_progress=True, + console_log_level="WARNING", + ) + + console.print( + f" ✓ Processed {result.total_trackers} trackers " + f"({result.successful_trackers} ok, {result.failed_trackers} failed)\n" + ) + + _render_failed_trackers( + result, + mode="bullets", + title="Failed trackers", + leading_newline=False, + ) + + if not result.success: + # Soft-fail posture, matching the product arm below: some trackers + # failing doesn't block the rest of the run (product arm, tables, + # upload) — the combined run summary below is what surfaces this. + console.print( + "[bold yellow]Warning: some patient trackers failed — " + "continuing with the rest of the run.[/bold yellow]\n" + ) + + except Exception as e: + console.print( + f"\n[bold yellow]Warning: patient pipeline failed: {e}[/bold yellow]\n" + "[yellow]Continuing with product/upload steps.[/yellow]\n" + ) + else: + console.print("[bold]Steps 2–3/5:[/bold] Skipping patient pipeline (--skip-patient)\n") + + tables_dir = settings.output_root / "tables" + logs_dir = settings.output_root / "logs" + + # Clinic static table — independent of tracker processing, always created + console.print("[bold]Step 3b/5:[/bold] Creating clinic static table...") + try: + create_table_clinic_static(tables_dir) + console.print(" ✓ Clinic static table created\n") + except Exception as e: + console.print(f" [bold red]Error creating clinic static table: {e}[/bold red]\n") + raise typer.Exit(1) from e + + # Product pipeline arm — soft failure posture: a crash here warns and + # continues so patient outputs (already on disk) still get uploaded. + if not skip_product: + console.print("[bold]Step 3c/5:[/bold] Running product pipeline...\n") + # Drop any stale product table from a prior run before re-running. + # Without --force, `run` preserves outputs (clean_output=False), + # so a crash mid-product would otherwise leave the previous run's + # parquet for upload. With --force the orchestrator wipes anyway, so + # this unlink is redundant in that case but harmless. + (settings.output_root / "tables" / "product_data.parquet").unlink(missing_ok=True) + try: + product_result = run_product_pipeline( + tracker_files=shared_tracker_files, + max_workers=_workers, + clean_output=force, + show_progress=True, + console_log_level="WARNING", + ) + console.print( + f" ✓ Processed {product_result.total_trackers} product trackers " + f"({product_result.successful_trackers} ok, " + f"{product_result.failed_trackers} failed)\n" + ) + _render_failed_trackers( + product_result, + mode="bullets", + title="Failed product trackers", + leading_newline=False, + ) + except Exception as e: + console.print( + f"[bold yellow]Warning: product pipeline failed: {e}[/bold yellow]\n" + "[yellow]Continuing with patient outputs only.[/yellow]\n" + ) + else: + console.print("[bold]Step 3c/5:[/bold] Skipping product pipeline (--skip-product)\n") + + # Errors table — written here, after both arms, rather than by either arm. + # The patient arm writes it from inside run_patient_pipeline and the product + # arm never wrote it at all, so `run` published a patient-only errors table + # (ticket 32, measured: 63,295 patient records, 0 product). The + # source-defect report derives from this table, so a missing arm is a + # missing half of the findings. + arm_errors = [ + error + for arm_result in (result, product_result) + if arm_result is not None + for tracker in arm_result.tracker_results + for error in tracker.data_errors + ] + if arm_errors: + console.print("[bold]Step 3d/5:[/bold] Creating errors table (both arms)...") + try: + create_table_errors(arm_errors, tables_dir) + console.print(f" ✓ Errors table created ({len(arm_errors):,} records)\n") + except Exception as e: + console.print(f" [bold yellow]Warning: errors table failed: {e}[/bold yellow]\n") + + # Tracker metadata table — MD5 + per-tracker output presence. + # Not a skip-gated step; it's cheap and summarises the run's final state. + if settings.data_root.exists(): + console.print("[bold]Step 3e/5:[/bold] Creating tracker metadata table...\n") + try: + from a4d.tables.metadata import create_table_tracker_metadata + + create_table_tracker_metadata(settings.data_root, settings.output_root) + console.print(" ✓ Tracker metadata table created\n") + except Exception as e: + console.print(f" [bold yellow]Warning: tracker metadata failed: {e}[/bold yellow]\n") + + # Step 3e – Product-patient link validation (logging-only, post-tables). + # Joins against patient_data_monthly (one row per patient per tracker + # file/month) — NOT + # patient_data_static, which collapses each patient to a single latest + # record and therefore only covers each patient's most recent file. That + # mismatch alone previously produced an 88% false-positive mismatch rate. + # Skips silently if either arm's table is missing (e.g. --skip-product), or + # when --skip-patient leaves a stale patient_data_monthly.parquet on disk + # whose contents don't match this run's product output. + product_table = tables_dir / "product_data.parquet" + patient_monthly = tables_dir / "patient_data_monthly.parquet" + if not skip_patient and product_table.exists() and patient_monthly.exists(): + console.print("[bold]Step 3f/5:[/bold] Validating product-patient links...") + try: + from a4d.tables.product import link_product_patient + + product_df = pl.read_parquet(product_table) + mismatched = link_product_patient(product_df, patient_monthly) + console.print(f" ✓ Link validation complete ({mismatched} unmatched product rows)\n") + except Exception as e: + console.print(f" [bold yellow]Warning: link validation failed: {e}[/bold yellow]\n") + + # Step 4 – Upload tables/ and logs/ to GCS under a timestamped prefix + # Each run gets an isolated path: YYYY/MM/DD/HHMMSS/tables/ and .../logs/ + # This avoids overwriting previous runs and keeps objectCreator permission sufficient. + if not skip_upload: + # BigQuery loads below are WRITE_TRUNCATE (full replace, not append) — + # show exactly what's about to be pushed before it happens. + row_counts = _scan_table_row_counts(tables_dir) + if row_counts: + console.print( + "[bold]About to upload (BigQuery tables are replaced, not appended):[/bold]" + ) + preview_table = Table() + preview_table.add_column("File", style="cyan") + preview_table.add_column("Rows", justify="right", style="green") + for filename, count in row_counts: + preview_table.add_row(filename, f"{count:,}") + console.print(preview_table) + console.print() + + console.print("[bold]Step 4/5:[/bold] Uploading output files to GCS...") + console.print(f" Prefix: {run_ts}/\n") + try: + uploaded: list[str] = [] + if tables_dir.exists(): + uploaded += upload_output(source_dir=tables_dir, prefix=f"{run_ts}/tables") + if logs_dir.exists(): + uploaded += upload_output(source_dir=logs_dir, prefix=f"{run_ts}/logs") + console.print( + f" ✓ Uploaded {len(uploaded)} files to gs://{settings.upload_bucket}/{run_ts}/\n" + ) + except Exception as e: + console.print(f"\n[bold red]Error during GCS upload: {e}[/bold red]\n") + raise typer.Exit(1) from e + else: + console.print("[bold]Step 4/5:[/bold] Skipping GCS upload (--skip-upload)\n") + + # Step 5 – Ingest tables into BigQuery + if not skip_upload: + console.print("[bold]Step 5/5:[/bold] Ingesting tables into BigQuery...") + try: + bq_results = load_pipeline_tables(tables_dir=tables_dir) + console.print(f" ✓ Loaded {len(bq_results)} tables into BigQuery\n") + except Exception as e: + console.print(f"\n[bold red]Error during BigQuery upload: {e}[/bold red]\n") + raise typer.Exit(1) from e + else: + console.print("[bold]Step 5/5:[/bold] Skipping BigQuery upload (--skip-upload)\n") + + # Rendered last, deliberately: earlier steps (tracker metadata, product- + # patient link validation, uploads) print their own progress/warning + # lines, which buried this summary when it ran mid-pipeline. + _render_combined_run_summary(result, product_result, tables_dir) + + console.print("[bold green]✓ Full pipeline completed successfully![/bold green]\n") + + +def main(): + """Entry point for CLI.""" + app() + + +if __name__ == "__main__": + main() diff --git a/src/a4d/config.py b/src/a4d/config.py new file mode 100644 index 0000000..b9dca21 --- /dev/null +++ b/src/a4d/config.py @@ -0,0 +1,67 @@ +"""Application configuration using Pydantic Settings.""" + +from pathlib import Path +from typing import Literal + +from dotenv import load_dotenv +from pydantic_settings import BaseSettings, SettingsConfigDict + +# Load .env into os.environ so non-prefixed vars like GOOGLE_APPLICATION_CREDENTIALS +# are visible to third-party SDKs (Google Auth, etc.) without requiring a manual export. +load_dotenv(override=False) + + +class Settings(BaseSettings): + """ + Application configuration with environment variable support. + + All settings can be overridden with environment variables prefixed with A4D_. + Example: A4D_DATA_ROOT=/path/to/data + """ + + model_config = SettingsConfigDict( + env_file=".env", + env_prefix="A4D_", + case_sensitive=False, + extra="ignore", + ) + + # Environment + environment: Literal["development", "production"] = "development" + + # GCP Configuration + project_id: str = "a4dphase2" + dataset: str = "tracker" + download_bucket: str = "a4dphase2_upload" + upload_bucket: str = "a4dphase2_output" + + # Paths + data_root: Path = Path("/Volumes/USB SanDisk 3.2Gen1 Media/a4d/a4dphase2_upload") + output_dir: Path = Path("output") + + # Processing settings + max_workers: int = 4 + + # Sentinels for 'recorded but unusable'. Distinct from null, which means + # nothing was recorded at all -- consumers need to tell the two apart. + error_val_numeric: float = 999999.0 + error_val_character: str = "Undefined" + error_val_date: str = "9999-09-09" + + # Accepted tracker year range (raise on out-of-range sheet-name or filename) + min_tracker_year: int = 2017 + max_tracker_year: int = 2030 + + @property + def output_root(self) -> Path: + """Computed output root path.""" + return self.data_root / self.output_dir + + @property + def tracker_root(self) -> Path: + """Tracker files root directory.""" + return self.data_root + + +# Global settings instance +settings = Settings() diff --git a/src/a4d/errors.py b/src/a4d/errors.py new file mode 100644 index 0000000..e0cf033 --- /dev/null +++ b/src/a4d/errors.py @@ -0,0 +1,235 @@ +"""Data quality error tracking for pipeline processing. + +This module provides the ErrorCollector class for tracking conversion failures, +validation errors, and other data quality issues. Errors are exported as +parquet files and aggregated into the logs table for BigQuery analysis. + +This is separate from operational logging (see a4d.logging) which tracks +pipeline execution and progress. +""" + +from datetime import datetime +from typing import Any, Literal + +import polars as pl +from pydantic import BaseModel, Field + +# Error code taxonomy. Every row-level data-quality finding carries one of +# these, so the logs table can be filtered by kind of problem. +ErrorCode = Literal[ + "type_conversion", # Failed to convert type (e.g., "abc" -> int) + "invalid_value", # Value outside allowed range or not in allowed list + "missing_value", # Required value is missing/NA + "missing_required_field", # Critical field (patient_id, status) is missing, row excluded + "invalid_tracker", # Tracker-level issues (missing columns, etc.) + "function_call", # Generic function execution error + "critical_abort", # Fatal error, tracker cannot be processed + "typo_rescued", # Known source-data typo substituted before parsing (informational) + "source_formula_error", # Source tracker's own formula errored (#NUM!, #DIV/0!) -- an + # input it depended on was never recorded, so no value could be computed (informational) + "balance_reconciliation", # Recomputed closing stock disagrees with the balance the + # tracker itself recorded -- the transactions and the recorded total do not add up + "glucose_unit_swapped", # A whole column labelled mg/dL holds mmol/L readings; values + # moved to the mmol column and rescaled. Reported once per column, not per row + "glucose_unit_suspect", # A single reading sits where the other unit's values land; + # kept as recorded, because a severe hypoglycaemic reading is indistinguishable + "date_recovered_from_text", # A date was read out of a clinical note rather than from a + # date-shaped cell -- carries the note, so the extraction stays auditable (informational) + "date_multiple_in_cell", # The cell named several dates and the first was published; + # a single date column cannot hold three admissions, so the workbook is what needs fixing + "date_year_inferred", # The cell named a day and a month but no year, so the tracker's + # own year was used -- the one component published that the source does not state + "buddhist_era_converted", # A Thai clinic's Buddhist-era date (BE = CE + 543) shifted to + # Gregorian in the cleaned stage -- the calendar the clinic uses, not an error it made, + # so the cell is recovered rather than sentinelled; logged so the shift stays auditable + "implausible_era_date", # A date past the Buddhist-era threshold that is not this + # tracker's own BE year -- a corrupt Excel serial (1,339,576 reads as 5567-08-19), so + # the cell is sentinelled rather than published as a date centuries in the future + "excel_error_patient_id", # A row's patient ID cell holds a broken formula (#REF!), so the + # patient cannot be identified and the row's measurements are dropped rather than pooled + # under the Undefined sentinel -- only the workbook can restore the identity +] + + +class DataError(BaseModel): + """Single data quality error record. + + Attributes: + file_name: Name of the tracker file where error occurred + patient_id: Patient ID (if applicable, else "unknown") + column: Column name where error occurred + original_value: Original value that caused the error + error_message: Human-readable error description + error_code: Error category for grouping/analysis + script: Script name where error occurred (e.g., "script2", "clean") + function_name: Function name where error occurred + timestamp: When the error was recorded + """ + + file_name: str + patient_id: str + column: str + original_value: str + error_message: str + error_code: ErrorCode + script: str = "clean" + function_name: str = "" + timestamp: datetime = Field(default_factory=datetime.now) + + +class ErrorCollector: + """Collects data quality errors for export to parquet. + + Errors are collected during processing and exported as a DataFrame + at the end. The DataFrame schema matches the logs table in BigQuery + for easy querying and dashboard visualization. + + Example: + >>> collector = ErrorCollector() + >>> collector.add_error( + ... file_name="clinic_001.xlsx", + ... patient_id="XX_YY001", + ... column="age", + ... original_value="invalid", + ... error_message="Could not convert 'invalid' to Int32", + ... error_code="type_conversion", + ... function_name="safe_convert_column" + ... ) + >>> # Or batch add: + >>> errors = [ + ... DataError(file_name="clinic_001.xlsx", patient_id="XX_YY001", ...), + ... DataError(file_name="clinic_001.xlsx", patient_id="XX_YY002", ...), + ... ] + >>> collector.add_errors(errors) + >>> df = collector.to_dataframe() + >>> df.write_parquet("output/clinic_001/errors.parquet") + """ + + def __init__(self): + """Initialize an empty error collector.""" + self.errors: list[DataError] = [] + + def add_error( + self, + file_name: str, + patient_id: str, + column: str, + original_value: Any, + error_message: str, + error_code: ErrorCode, + script: str = "clean", + function_name: str = "", + ) -> None: + """Add a data quality error to the collector. + + Args: + file_name: Name of the tracker file + patient_id: Patient ID (use "unknown" if not applicable) + column: Column name where error occurred + original_value: Original value that caused the error + error_message: Human-readable error description + error_code: Error category (type_conversion, invalid_value, etc.) + script: Script name (default: "clean") + function_name: Function name where error occurred + """ + error = DataError( + file_name=file_name, + patient_id=patient_id, + column=column, + original_value=str(original_value), + error_message=error_message, + error_code=error_code, + script=script, + function_name=function_name, + ) + self.errors.append(error) + + def add_errors(self, errors: list[DataError]) -> None: + """Add multiple errors at once. + + Args: + errors: List of DataError instances to add + + Example: + >>> errors = [ + ... DataError(file_name="clinic_001.xlsx", patient_id="XX_YY001", ...), + ... DataError(file_name="clinic_001.xlsx", patient_id="XX_YY002", ...), + ... ] + >>> collector.add_errors(errors) + """ + self.errors.extend(errors) + + def to_dataframe(self) -> pl.DataFrame: + """Export errors as a Polars DataFrame for parquet export. + + Returns: + Polars DataFrame with all error records, or empty DataFrame if no errors + + Schema: + - file_name: str + - patient_id: str + - column: str + - original_value: str + - error_message: str + - error_code: str (categorical) + - script: str (categorical) + - function_name: str (categorical) + - timestamp: datetime + """ + if not self.errors: + # Return empty DataFrame with correct schema + return pl.DataFrame( + schema={ + "file_name": pl.Utf8, + "patient_id": pl.Utf8, + "column": pl.Utf8, + "original_value": pl.Utf8, + "error_message": pl.Utf8, + "error_code": pl.Categorical, + "script": pl.Categorical, + "function_name": pl.Categorical, + "timestamp": pl.Datetime, + } + ) + + # Convert Pydantic models to dict records + records = [error.model_dump() for error in self.errors] + + # Create DataFrame and cast categorical columns for efficiency + df = pl.DataFrame(records) + df = df.with_columns( + [ + pl.col("error_code").cast(pl.Categorical), + pl.col("script").cast(pl.Categorical), + pl.col("function_name").cast(pl.Categorical), + ] + ) + + return df + + def __len__(self) -> int: + """Return number of errors collected.""" + return len(self.errors) + + def __bool__(self) -> bool: + """Return True if any errors have been collected.""" + return len(self.errors) > 0 + + def clear(self) -> None: + """Clear all collected errors.""" + self.errors.clear() + + def get_error_summary(self) -> dict[str, int]: + """Get summary of errors by error_code. + + Returns: + Dictionary mapping error_code to count + + Example: + >>> collector.get_error_summary() + {'type_conversion': 10, 'invalid_value': 5} + """ + summary: dict[str, int] = {} + for error in self.errors: + summary[error.error_code] = summary.get(error.error_code, 0) + 1 + return summary diff --git a/src/a4d/extract/__init__.py b/src/a4d/extract/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/a4d/extract/common.py b/src/a4d/extract/common.py new file mode 100644 index 0000000..9177b43 --- /dev/null +++ b/src/a4d/extract/common.py @@ -0,0 +1,183 @@ +"""Shared tracker-level extraction plumbing used by patient and product arms. + +Holds helpers that are not patient-specific or product-specific: detecting +month sheets in a workbook, parsing the tracker year, and parsing the +sheet-name month suffix. +""" + +import calendar +import re +from pathlib import Path + +import polars as pl +from loguru import logger + +from a4d.config import settings + +# Excel's own formula-error sentinels. Extraction deliberately preserves these +# in the raw output so the raw layer stays a faithful capture of the source +# file (ticket 27); a4d.clean.converters.normalize_excel_formula_errors is +# where they become null, with each one logged. +EXCEL_ERROR_STRINGS: tuple[str, ...] = ( + "#DIV/0!", + "#VALUE!", + "#REF!", + "#NAME?", + "#NUM!", + "#N/A", + "#NULL!", +) + + +def normalize_patient_id_expr(col: pl.Expr) -> pl.Expr: + """Fold a patient ID to the identity the clinic meant, as a Polars expression. + + Folds hyphens to underscores and drops any transfer-clinic suffix, so + ``LA-MH056`` and ``MY_SM003_SB`` become ``LA_MH056`` and ``MY_SM003``. + + Lives here rather than in the cleaning layer because both arms need it: + cleaning applies it to publish one identity per patient, and extraction + applies it to the *Patient List join key* only (ticket 58) -- the raw + ``patient_id`` column itself stays exactly as the source workbook wrote it, + which the comparison tool's row-alignment key depends on. + """ + normalized = col.str.replace_all("-", "_") + return ( + pl.when(normalized.str.contains("_")) + .then(normalized.str.extract(r"^([A-Z]+_[^_]+)", 1)) + .otherwise(normalized) + ) + + +def get_tracker_year(tracker_file: Path, month_sheets: list[str]) -> int: + """Extract tracker year from month sheet names or filename. + + Tries to parse year from month sheet names (e.g., "Jan24" -> 2024). + Falls back to extracting from filename if parsing fails. + Validates year is in reasonable range (2017-2030). + + Args: + tracker_file: Path to the tracker Excel file + month_sheets: List of month sheet names + + Returns: + Year of the tracker (e.g., 2024) + + Raises: + ValueError: If year cannot be determined or is out of valid range + + Example: + >>> get_tracker_year(Path("2024_Clinic.xlsx"), ["Jan24", "Feb24"]) + 2024 + """ + for sheet in month_sheets: + match = re.search(r"(\d{2})$", sheet) + if match: + year_suffix = int(match.group(1)) + year = 2000 + year_suffix # Assume 20xx until 2100 + logger.debug(f"Parsed year {year} from sheet name '{sheet}'") + + if not (settings.min_tracker_year <= year <= settings.max_tracker_year): + raise ValueError( + f"Year {year} is out of valid range " + f"({settings.min_tracker_year}-{settings.max_tracker_year}). " + f"Parsed from sheet name '{sheet}'" + ) + + return year + + match = re.search(r"(\d{4})", tracker_file.name) + if match: + year = int(match.group(1)) + logger.debug(f"Parsed year {year} from filename '{tracker_file.name}'") + + if not (settings.min_tracker_year <= year <= settings.max_tracker_year): + raise ValueError( + f"Year {year} is out of valid range " + f"({settings.min_tracker_year}-{settings.max_tracker_year}). " + f"Parsed from filename '{tracker_file.name}'" + ) + + return year + + raise ValueError( + f"Could not determine year from month sheets {month_sheets} or filename {tracker_file.name}" + ) + + +def find_month_sheets(workbook) -> list[str]: + """Find all month sheets in the tracker workbook. + + Month sheets are identified by matching against month abbreviations + (Jan, Feb, Mar, etc.) and sorted by month number for consistent processing. + + Args: + workbook: openpyxl Workbook object + + Returns: + List of month sheet names found in the workbook, sorted by month number + (Jan=1, Feb=2, ..., Dec=12) + + Example: + >>> wb = load_workbook("tracker.xlsx") + >>> find_month_sheets(wb) + ['Jan24', 'Feb24', 'Mar24', ...] + """ + month_abbrs = list(calendar.month_abbr)[1:] # ['Jan', 'Feb', ...] + month_sheets = [] + + for sheet_name in workbook.sheetnames: + if any(sheet_name.startswith(abbr) for abbr in month_abbrs): + month_sheets.append(sheet_name) + + def get_month_number(sheet_name: str) -> int: + """Extract month number from sheet name (Jan=1, ..., Dec=12).""" + month_prefix = sheet_name[:3] + try: + return month_abbrs.index(month_prefix) + 1 + except ValueError: + return 999 # Push unrecognized sheets to end + + month_sheets.sort(key=get_month_number) + + logger.info(f"Found {len(month_sheets)} month sheets (sorted by month): {month_sheets}") + return month_sheets + + +def extract_tracker_month(sheet_name: str) -> int: + """Extract month number (1-12) from sheet name. + + Args: + sheet_name: Sheet name like "Jan24", "Feb24", etc. + + Returns: + Month number (1 for January, 2 for February, etc.) + + Raises: + ValueError: If month cannot be extracted or is out of valid range + + Example: + >>> extract_tracker_month("Jan24") + 1 + >>> extract_tracker_month("Dec23") + 12 + """ + month_abbrs = list(calendar.month_abbr)[1:] # ['Jan', 'Feb', ...] + + # Check first 3 characters + month_prefix = sheet_name[:3] + + if month_prefix in month_abbrs: + month_num = month_abbrs.index(month_prefix) + 1 # +1 because index is 0-based + + # Validate month is in valid range (1-12) + # This should always be true given the logic above, but check anyway for safety + if not (1 <= month_num <= 12): + raise ValueError( + f"Month number {month_num} is out of valid range (1-12). " + f"Parsed from sheet name '{sheet_name}'" + ) + + return month_num + + raise ValueError(f"Could not extract month from sheet name '{sheet_name}'") diff --git a/src/a4d/extract/patient.py b/src/a4d/extract/patient.py new file mode 100644 index 0000000..d2f9122 --- /dev/null +++ b/src/a4d/extract/patient.py @@ -0,0 +1,1343 @@ +"""Patient data extraction from Excel tracker files. + +This module handles reading patient data from Excel trackers, which have +evolved over the years with different formats and structures. +""" + +import datetime +import re +import warnings +from dataclasses import dataclass, field +from pathlib import Path + +import polars as pl +from loguru import logger +from openpyxl import load_workbook +from openpyxl.utils import get_column_letter, range_boundaries +from openpyxl.utils.datetime import to_excel + +from a4d.errors import ErrorCollector +from a4d.extract.common import ( + extract_tracker_month, + find_month_sheets, + get_tracker_year, + normalize_patient_id_expr, +) +from a4d.reference.synonyms import ColumnMapper, load_patient_mapper + +__all__ = [ + "extract_tracker_month", + "find_month_sheets", + "get_tracker_year", +] + +# Suppress openpyxl warnings about unsupported Excel features +# We only read data, so these warnings are not actionable +warnings.filterwarnings("ignore", category=UserWarning, module=r"openpyxl\..*") + + +def find_data_start_row(ws) -> int: + """Find the first row containing patient data. + + Scans column A for the first numeric value (patient row numbers: 1, 2, 3...). + This skips any non-numeric values that may appear above the patient data + (e.g., spaces, text, product data). + + A clinician who clears a row number leaves a whitespace-only string behind, + and that row is still a patient row -- so the numeric block is extended back + over any such cells directly abutting it. Starting one row too late is not a + lost row but a lost sheet: the header rows are read from the data, no + patient_id survives harmonization, and the sheet is skipped entirely (2022 + Children's Hospital 2, Oct22). Only cells touching the block qualify: the + broader "first non-empty cell in column A" rule was measured across all 254 + trackers and would start 14 sheets at row 1 on a stray 'm'/'f'/'n' left up + there (2026 Gensan, 2025/2026 VNCH). The narrow rule moves exactly one + sheet in the corpus. + One row only, since a longer run of blanks has never been observed and + swallowing several would risk reading a header row as data instead. + + Args: + ws: openpyxl worksheet object + + Returns: + Row number (1-indexed) where patient data starts + + Raises: + ValueError: If no numeric data is found in column A + """ + # Sequential iter_rows scan, not repeated ws.cell() calls: on a read-only + # worksheet each ws.cell() re-parses the sheet's XML from row 1, making a + # per-row loop O(n^2) in the row count before data starts. + max_row = ws.max_row or 1000 + previous_blank_string_row: int | None = None + for row_idx, (cell_value,) in enumerate( + ws.iter_rows(min_row=1, max_row=max_row, min_col=1, max_col=1, values_only=True), + start=1, + ): + if isinstance(cell_value, (int, float)) and not isinstance(cell_value, bool): + if previous_blank_string_row == row_idx - 1: + return previous_blank_string_row + return row_idx + + if isinstance(cell_value, str) and cell_value.strip() == "": + previous_blank_string_row = row_idx + else: + previous_blank_string_row = None + + raise ValueError("No patient data found in column A (looking for numeric row numbers)") + + +def read_header_rows(ws, data_start_row: int, max_cols: int = 100) -> tuple[list, list]: + """Read and trim the two header rows above the data. + + Headers are located in the two rows immediately before data_start_row. + Reads up to max_cols columns and trims to the last non-None column. + + Args: + ws: openpyxl worksheet object + data_start_row: Row number where patient data starts + max_cols: Maximum number of columns to read (default: 100) + + Returns: + Tuple of (header_1, header_2) lists, trimmed to actual width + + Example: + >>> header_1, header_2 = read_header_rows(ws, 77) + >>> len(header_1) + 31 + """ + header_row_1 = data_start_row - 1 + header_row_2 = data_start_row - 2 + + # Read raw header rows + header_1_raw = list( + ws.iter_rows( + min_row=header_row_1, + max_row=header_row_1, + min_col=1, + max_col=max_cols, + values_only=True, + ) + )[0] + header_2_raw = list( + ws.iter_rows( + min_row=header_row_2, + max_row=header_row_2, + min_col=1, + max_col=max_cols, + values_only=True, + ) + )[0] + + last_col = max_cols + for i in range(len(header_1_raw) - 1, -1, -1): + if header_1_raw[i] is not None or header_2_raw[i] is not None: + last_col = i + 1 + break + + header_1 = list(header_1_raw[:last_col]) + header_2 = list(header_2_raw[:last_col]) + + return header_1, header_2 + + +_UPDATED_YEAR_MARKER = re.compile(r"^updated\s*(19|20)\d{2}$") + + +def _is_updated_year_marker(header_cell: object) -> bool: + """Is this header cell a bare "Updated " continuation marker? + + Whole-cell match only: "Level of Education Or Occupation Updated 2023" is a + real single-cell header (see synonyms_patient.yaml) and must not match. + """ + normalized = re.sub(r"\s+", " ", str(header_cell).replace("\n", " ")).strip().lower() + return bool(_UPDATED_YEAR_MARKER.match(normalized)) + + +_MERGE_CELLS_BLOCK = re.compile(rb"", re.S) +_MERGE_REF = re.compile(rb'ref="([A-Z0-9:]+)"') + + +def merged_header_spans(workbook, worksheet, upper_header_row: int) -> list[tuple[int, int]]: + """Horizontal cell merges covering the upper header row, as (first, last) columns. + + A merged title is the workbook's own statement that the columns beneath it + are one block -- the only structural evidence available for a sub-header + whose own upper cell is empty. openpyxl drops merge metadata in read_only + mode, and loading a workbook read-write costs the 6.6x that ticket 10 won + back, so the ranges are read straight out of the sheet XML in the archive + openpyxl already has open. + + Degrades to "no spans" -- i.e. the pre-existing forward-fill behaviour -- + if the private attributes it leans on ever move. + """ + try: + archive = workbook._archive + data = archive.read(worksheet._worksheet_path.lstrip("/")) + except AttributeError, KeyError, OSError: + return [] + + block = _MERGE_CELLS_BLOCK.search(data) + if not block: + return [] + + spans = [] + for ref in _MERGE_REF.findall(block.group(0)): + try: + min_col, min_row, max_col, max_row = range_boundaries(ref.decode()) + except ValueError: + continue + if max_col > min_col and min_row <= upper_header_row <= max_row: + spans.append((min_col, max_col)) + return spans + + +def _merge_header_pair( + header_1: list, + header_2: list, + mapper: ColumnMapper | None = None, +) -> list[str | None]: + """Merge two header rows using heuristic forward-fill with synonym validation. + + When h2=None but h1 exists: + 1. Try forward-fill: combine prev_h2 + h1 + 2. If mapper validates this as known column, use it + 3. Otherwise, treat h1 as standalone column + + This replaces Excel merge metadata detection with synonym-based validation, + eliminating the need for slow read_only=False workbook loading. + + Special case: If header_1 contains "Patient ID" (or known synonyms) and + header_2 appears to be a title row (mostly None), use only header_1. + + Args: + header_1: First header row (closer to data), 0-indexed + header_2: Second header row (further from data), 0-indexed + mapper: Optional ColumnMapper for validating forward-filled headers + + Returns: + List of merged header strings with whitespace normalized + """ + patient_id_indicators = ["patient id", "patient.id"] + has_patient_id_in_h1 = any( + str(h1).strip().lower() in patient_id_indicators for h1 in header_1 if h1 is not None + ) + + non_none_count_h2 = sum(1 for h2 in header_2 if h2 is not None) + + if has_patient_id_in_h1 and non_none_count_h2 <= 2: + logger.debug( + "Detected title row in header_2 with Patient ID in header_1, using header_1 only" + ) + headers = [str(h1).strip() if h1 is not None else None for h1 in header_1] + headers = [re.sub(r"\s+", " ", h.replace("\n", " ")) if h else None for h in headers] + return headers + + headers = [] + prev_h2 = None + + for h1, h2 in zip(header_1, header_2, strict=True): + if h2 is not None and _is_updated_year_marker(h2) and prev_h2: + # The 2022 template labels an update-date column "Updated 2022" + # instead of repeating its subject, so the marker alone is not a + # column name -- the subject sits in the column to its left. + # Without this, blood_pressure_updated and edu_occ_updated go + # unmapped on every 2022 tracker (7,165 values). + h2 = prev_h2 + + if h1 and h2: + headers.append(f"{h2} {h1}".strip()) + prev_h2 = str(h2).strip() + elif h2: + headers.append(str(h2).strip()) + prev_h2 = str(h2).strip() + elif h1: + # Try forward-fill with validation + if prev_h2: + candidate = f"{prev_h2} {h1}".strip() + if mapper and mapper.is_known_column(candidate): + headers.append(candidate) + else: + # Forward-fill not valid, use h1 standalone + headers.append(str(h1).strip()) + else: + headers.append(str(h1).strip()) + else: + headers.append(None) + prev_h2 = None # Reset on gap + + headers = [re.sub(r"\s+", " ", h.replace("\n", " ")) if h else None for h in headers] + + return headers + + +def merge_headers( + header_1: list, + header_2: list, + mapper: ColumnMapper | None = None, + merged_spans: list[tuple[int, int]] | None = None, +) -> list[str | None]: + """Merge two header rows into one column name per column. + + Forward-fill alone cannot cross a column whose *both* header cells are + empty -- it resets there, deliberately, so a title cannot leak into a block + it does not cover. Where the workbook says a title is merged across a span, + that reset throws away information the file actually carries: 2021 + Putrajaya merges "Complication Screening (Current Month Testing)" over five + columns, and the "Results"/"Date (mmm-yy)" sub-headers past the gap were + left bare and mapped nowhere, losing the recorded screening results. + + A merged title *qualifies a sub-header*; it never names a column outright. + A column blank in both header rows has no name for the title to qualify, so + calling it by the bare block title would be a guess -- and a harmful one: + in the 2022 template that guess maps a second column onto + `complication_screening`, which one already claims (290 sheets), and in the + 2022 "Insulin Regimen" merge it would comma-join two columns holding + near-duplicate values ("Basal-bolus MDI (AN/HI)" against "Basal-bolus + (AN/HI)") into one worse value -- 3,659 near-duplicate cells. So the first + column is kept alone in both cases. Two sub-headers that would qualify to + the same name are refused for the same reason. + + Args: + header_1: First header row (closer to data), 0-indexed + header_2: Second header row (further from data), 0-indexed + mapper: Optional ColumnMapper for validating forward-filled headers + merged_spans: (first, last) 1-indexed column pairs the upper header row + is merged across, from `merged_header_spans` + """ + headers = _merge_header_pair(header_1, header_2, mapper) + if not merged_spans: + return headers + + filled = list(header_2) + for first, last in merged_spans: + if first - 1 >= len(filled): + continue + title = header_2[first - 1] + if title is None: + continue + for col in range(first, min(last, len(filled)) + 1): + has_sub_header = col - 1 < len(header_1) and header_1[col - 1] is not None + if filled[col - 1] is None and has_sub_header: + filled[col - 1] = title + + propagated = _merge_header_pair(header_1, filled, mapper) + taken = {h for h in headers if h} + for index, name in enumerate(propagated): + if name is None or name == headers[index] or name in taken: + continue + headers[index] = name + taken.add(name) + return headers + + +def _carries_data_beyond_identifier(row: tuple) -> bool: + """Does a row hold anything past a repeat of its own patient identifier? + + Deliberately not a count threshold: the trackers repeat the identifier in + a second column, so "more than n non-empty cells" would be a guess about + layout, while "a value that isn't the identifier" is what actually + distinguishes a record from a leftover ID. + """ + identifier = row[1] + return any( + cell is not None and str(cell).strip() != "" and cell != identifier for cell in row[2:] + ) + + +# Excel's serial epoch starts at 1899-12-30, so a "date" in the first years +# of the 1900s is a small number that inherited a date number format from a +# neighbouring cell, not a date anyone could have entered. Raised from 1903 +# to 1906 (ticket 52): a bare four-digit year typed into a date-formatted +# cell is a serial of 1900-2100, which resolves into March-September 1905 and +# so escaped the old bound -- 2019 Yangon Children's writes 163 D.O.B. cells +# that way. Still nowhere near a real date: the earliest genuine one in the +# 254-tracker set is a 1956 D.O.B. +_IMPOSSIBLE_DATE_BEFORE = datetime.datetime(1906, 1, 1) + + +def _recover_number_typed_as_date(value: object) -> object: + """Undo Excel's date formatting of a plainly numeric entry. + + openpyxl honors each cell's own format, so a plain number sitting in a + date-formatted cell comes back as a datetime, which the numeric conversion + then rejects into a 999999 sentinel -- 24 systolic readings were reaching + BigQuery that way. Excel's 1899-12-30 epoch is what makes this recoverable: + such a "date" is a small number, so a pre-1903 datetime converts back to + the serial the clinician actually typed. Verified against the real source + Excel (2025 Hat Yai, Annual!H + for TH_HY035: a dd-mmm-yyyy-formatted cell holding datetime(1900, 4, 29), + i.e. the systolic 120 the neighbouring diastolic 74 belongs with). + + Only the impossible range is recovered. A date that could plausibly have + been typed stays a date even in a numeric column -- that is a different + cause, already settled on the product arm as + ``openpyxl_date_typed_stray_cell``. + """ + if not isinstance(value, datetime.datetime) or value >= _IMPOSSIBLE_DATE_BEFORE: + return value + serial = to_excel(value) + return int(serial) if float(serial).is_integer() else serial + + +def read_patient_rows(ws, data_start_row: int, num_columns: int) -> list[tuple]: + """Read patient data rows from the worksheet. + + Reads from data_start_row until either ws.max_row or the first completely + empty row. Skips rows where both the row number (column A) and patient_id + (column B) are None, but accepts rows where patient_id exists even if row + number is missing (handles data quality issues in Excel files). + + An unnumbered row must additionally carry a value that is not just a + repeat of its own identifier. Keeping every unnumbered row instead turned a + bare list of patient IDs left below the data block into invented monthly + records (2024_Vietnam National Children's Jul24, 24 rows of nothing but the + ID twice), which then picked up real-looking demographics from the Patient + List join. Bounding the block by the row-number column alone would be the + other obvious fix, but a 254-tracker sweep showed it loses a genuine record + -- 2024_Mahosot's Jun24 LA-MH088 is a complete patient row that simply lost + its row number. Requiring actual data keeps that case and drops the other. + + Args: + ws: openpyxl worksheet object + data_start_row: Row number where patient data starts + num_columns: Number of columns to read + + Returns: + List of tuples, each containing one row of patient data + + Example: + >>> rows = read_patient_rows(ws, 77, 31) + >>> len(rows) + 4 + """ + data = [] + for row in ws.iter_rows( + min_row=data_start_row, + max_row=ws.max_row, + min_col=1, + max_col=num_columns, + values_only=True, + ): + if all(cell is None for cell in row): + break + # Skip rows where both row number (col A) AND patient_id (col B) are missing + # This handles cases where Excel has missing row numbers but valid patient data + if row[0] is None and (len(row) < 2 or row[1] is None): + continue + if row[0] is None and not _carries_data_beyond_identifier(row): + continue + data.append(tuple(_recover_number_typed_as_date(cell) for cell in row)) + + return data + + +def merge_duplicate_columns_data( + headers: list[str], data: list[list] +) -> tuple[list[str], list[list]]: + """Merge data from duplicate column headers by concatenating with commas. + + When Excel cells are merged both horizontally and vertically, the forward-fill + logic in merge_headers() can create duplicate column names. This function + concatenates their values rather than keeping only the first, which was + discarding 2,489 recorded values across 27 trackers -- the 2023 template's + B.P./Kidney/Eye/Foot/Lipids screening sub-columns all share one canonical + name. + + Args: + headers: List of header strings (may contain duplicates) + data: List of data rows (each row is a list) + + Returns: + Tuple of (unique_headers, merged_data) + + Example: + >>> headers = ["ID", "DM Complications", "DM Complications", "DM Complications", "Age"] + >>> data = [["1", "A", "B", "C", "25"], ["2", "X", "Y", "Z", "30"]] + >>> merge_duplicate_columns_data(headers, data) + (['ID', 'DM Complications', 'Age'], [['1', 'A,B,C', '25'], ['2', 'X,Y,Z', '30']]) + """ + if len(headers) == len(set(headers)): + return headers, data + + from collections import defaultdict + + header_positions: dict[str, list[int]] = defaultdict(list) + for idx, header in enumerate(headers): + header_positions[header].append(idx) + + unique_headers = list(header_positions.keys()) + + duplicated = [h for h, positions in header_positions.items() if len(positions) > 1] + if duplicated: + logger.debug(f"Merging {len(duplicated)} duplicate column groups: {duplicated}") + + merged_data = [] + for row in data: + merged_row = [] + for header in unique_headers: + positions = header_positions[header] + if len(positions) == 1: + merged_row.append(row[positions[0]]) + else: + values = [str(row[pos]) if row[pos] is not None else "" for pos in positions] + values = [v for v in values if v] + merged_value = ",".join(values) if values else None + merged_row.append(merged_value) + merged_data.append(merged_row) + + return unique_headers, merged_data + + +def filter_valid_columns( + headers: list[str | None], data: list[tuple] +) -> tuple[list[str], list[list]]: + """Filter out columns with None headers and their corresponding data. + + Args: + headers: List of header strings (may contain None) + data: List of data rows + + Returns: + Tuple of (valid_headers, filtered_data) + + Example: + >>> headers = ["ID", None, "Name", None, "Age"] + >>> data = [("1", "x", "Alice", "y", "30")] + >>> filter_valid_columns(headers, data) + (['ID', 'Name', 'Age'], [['1', 'Alice', '30']]) + """ + valid_cols = [(i, h) for i, h in enumerate(headers) if h] + + if not valid_cols: + return [], [] + + valid_indices = [i for i, _ in valid_cols] + valid_headers = [h for _, h in valid_cols] + + filtered_data = [[row[i] for i in valid_indices] for row in data] + + return valid_headers, filtered_data + + +def _is_number(value: object) -> bool: + try: + float(str(value)) + except ValueError: + return False + return True + + +def find_dropped_data_columns( + headers: list[str | None], data: list[tuple] +) -> list[tuple[int, int]]: + """Find headerless columns that nonetheless carry data. + + `filter_valid_columns` drops any column whose header cell is blank, which is + right for the trackers' many spacer columns but silently loses real values + when a clinician left a header cell empty by mistake -- e.g. 2021 Kantha + Bopha's Mar21/Apr21 sheets, where the insulin-regimen column has data from + row 87 down and no header, while the same column in May21 reads + "Insulin Regime". Reporting these lets the source workbook be corrected; + the pipeline cannot name such a column on its own. + + All-numeric columns are excluded: the trackers put an unlabelled row counter + left of the patient data, which accounts for 189 of the 199 headerless + columns holding values across the 254-tracker set. + + Returns: + (column index, count of non-empty values) per affected column + """ + findings = [] + for i, header in enumerate(headers): + if header: + continue + values = [row[i] for row in data if i < len(row) and row[i] not in (None, "")] + if not values or all(_is_number(v) for v in values): + continue + findings.append((i, len(values))) + return findings + + +def recover_blank_headers( + headers: list[str | None], + data: list[tuple], + sibling_headers: list[list[str | None]], + merged_spans: list[tuple[int, int]] | None = None, +) -> list[str | None]: + """Name a headerless data column from the month sheets that do label it. + + A tracker's month sheets share one layout, so when a clinician leaves a + header cell empty in one sheet the same column is usually still labelled in + the others -- e.g. 2021 Kantha Bopha, where Mar21/Apr21 hold insulin-regimen + values under an empty header and May21 reads "Insulin Regime". Without this + the column is dropped and its values are lost on both pipelines. + + Deliberately self-limiting; it abstains unless the workbook itself settles + the answer: + + - the siblings must **agree**: two different candidate names is a guess + (2021 Putrajaya offers three at one position), + - a position no sibling names is left alone -- which is what keeps the 2022 + template's hidden merged-cell column, blank in every sheet and correctly + dropped, out of reach of this rule, + - a name this sheet already uses is refused, since it would collide, + - a position covered by a merged header is refused: the merge is this + sheet's own statement about what the column belongs to, and outranks a + sibling laid out differently. Putrajaya's Jul21 sheet carries "Patient + Observations" at the position Dec21 uses for a complication-screening + selection, so recovering by position filed a screening result under + observations, + - and only columns `find_dropped_data_columns` reports are eligible, so + spacer columns and the unlabelled row counter are untouched. + + Measured over the 254-tracker set: recovers 316 values across 5 trackers and + does nothing to the other 4,256. + """ + if not sibling_headers: + return headers + + recovered = list(headers) + taken = {h for h in headers if h} + covered = { + col - 1 for first, last in (merged_spans or []) for col in range(first + 1, last + 1) + } + + for index, _ in find_dropped_data_columns(headers, data): + if index in covered: + continue + donors: set[str] = set() + for sibling in sibling_headers: + if index < len(sibling): + candidate = sibling[index] + if candidate: + donors.add(candidate) + if len(donors) != 1: + continue + donor = donors.pop() + if donor in taken: + continue + recovered[index] = donor + taken.add(donor) + + return recovered + + +@dataclass(frozen=True) +class LayoutChange: + """One column position the month sheets of a tracker disagree about.""" + + index: int + headers: list[str] + canonical: list[str] + renames_only: bool + + +def find_layout_changes( + layouts_by_sheet: dict[str, list[str | None]], mapper: ColumnMapper +) -> list[LayoutChange]: + """Find column positions whose meaning is not stable across a tracker's sheets. + + A tracker is one workbook for one clinic-year and should not change shape + partway through it, so a position that means one thing in January and + another in June is a defect in the workbook worth reporting -- and a + dangerous one, because nothing downstream can see it. + + Two kinds, distinguished because only one is harmful: + + - **renames_only**: every spelling at that position maps to the same + canonical column (`Insulin regime` -> `Insulin regimen`). The synonym file + already absorbs these. + - the rest: the canonical column itself changes. Either a column was added + or dropped mid-year and everything to its right shifted (2018 CDA's Apr18 + has no `Insulin Regimen`, so BASAL dose sits where the regimen sits in the + other eleven sheets), or the header was edited over unchanged data (2020 + CDA relabels `Baseline FBG` from `mmol/dL` to `mg/dL` in June while the + values stay 67-500, i.e. mg/dL throughout, so five months are filed under + the wrong unit). + + A blank header on one sheet is not a change -- that is + `recover_blank_headers`' business. + """ + if len(layouts_by_sheet) < 2: + return [] + + changes = [] + width = max(len(headers) for headers in layouts_by_sheet.values()) + for index in range(width): + names: set[str] = set() + for headers in layouts_by_sheet.values(): + if index < len(headers): + header = headers[index] + if header: + names.add(header) + if len(names) < 2: + continue + canonical = {mapper.get_standard_name(name) for name in names} + changes.append( + LayoutChange( + index=index, + headers=sorted(names), + canonical=sorted(canonical), + renames_only=len(canonical) == 1, + ) + ) + return changes + + +@dataclass(frozen=True) +class SheetLayout: + """Where a sheet's data starts and what its merged header row says.""" + + data_start_row: int + headers: list[str | None] + merged_spans: list[tuple[int, int]] = field(default_factory=list) + + +def collect_sheet_layouts( + workbook, month_sheets: list[str], mapper: ColumnMapper | None = None +) -> dict[str, SheetLayout]: + """Resolve every month sheet's layout once per tracker. + + `recover_blank_headers` needs every *other* sheet's headers before any one + sheet can be finalized, so this pass has to happen up front. Its results are + then handed back to `extract_patient_data`, which would otherwise redo + `find_data_start_row` -- a full column-A scan per sheet, and the hot spot + ticket 10 rewrote to be linear. Reusing the layout keeps that pass off the + critical path instead of doubling it. + """ + layouts: dict[str, SheetLayout] = {} + for sheet_name in month_sheets: + try: + worksheet = workbook[sheet_name] + data_start_row = find_data_start_row(worksheet) + header_1, header_2 = read_header_rows(worksheet, data_start_row) + except ValueError, KeyError: + # A sheet whose layout cannot be read simply donates nothing. + continue + spans = merged_header_spans(workbook, worksheet, data_start_row - 2) + layouts[sheet_name] = SheetLayout( + data_start_row=data_start_row, + headers=merge_headers(header_1, header_2, mapper=mapper, merged_spans=spans), + merged_spans=spans, + ) + return layouts + + +def extract_patient_data( + tracker_file: Path, + sheet_name: str, + year: int, + mapper: ColumnMapper | None = None, + workbook=None, + layout: SheetLayout | None = None, + sibling_headers: list[list[str | None]] | None = None, +) -> pl.DataFrame: + """Extract patient data from a single sheet. + + Uses single read_only=True load with synonym-validated header merging. + + Args: + tracker_file: Path to the tracker Excel file + sheet_name: Name of the sheet to extract + year: Year of the tracker (currently unused, reserved for future use) + mapper: Optional ColumnMapper for validating forward-filled headers + workbook: Optional pre-loaded workbook for caching across sheets + layout: This sheet's already-resolved start row and headers, from + collect_sheet_layouts; recomputed here when not supplied + sibling_headers: Other month sheets' merged headers, used to name a + column whose own header cell is empty (see recover_blank_headers) + + Returns: + Polars DataFrame with patient data (all columns as strings) + + Example: + >>> df = extract_patient_data( + ... Path("2024_Clinic.xlsx"), + ... "Jan24", + ... 2024 + ... ) + >>> len(df) + 4 + >>> "Patient ID*" in df.columns + True + """ + if mapper is None: + mapper = load_patient_mapper() + + # Use cached workbook or load new one + close_wb = workbook is None + if workbook is None: + workbook = load_workbook( + tracker_file, + read_only=True, + data_only=True, + keep_vba=False, + keep_links=False, + ) + + ws = workbook[sheet_name] + + if layout is None: + data_start_row = find_data_start_row(ws) + logger.info("Processing headers...") + header_1, header_2 = read_header_rows(ws, data_start_row) + merged_spans = merged_header_spans(workbook, ws, data_start_row - 2) + headers = merge_headers(header_1, header_2, mapper=mapper, merged_spans=merged_spans) + else: + data_start_row, headers = layout.data_start_row, list(layout.headers) + merged_spans = layout.merged_spans + + logger.debug( + f"Sheet '{sheet_name}': Patient data found in rows {data_start_row} to {ws.max_row}" + ) + + valid_cols = [(i, h) for i, h in enumerate(headers) if h] + + if not valid_cols: + if close_wb: + workbook.close() + logger.bind(error_code="invalid_tracker").warning( + f"No valid headers found in sheet '{sheet_name}'" + ) + return pl.DataFrame() + + data = read_patient_rows(ws, data_start_row, len(headers)) + + if close_wb: + workbook.close() + + if sibling_headers: + recovered = recover_blank_headers(headers, data, sibling_headers, merged_spans=merged_spans) + for i, (before, after) in enumerate(zip(headers, recovered, strict=True)): + if before != after: + logger.info( + f"Sheet '{sheet_name}': column {get_column_letter(i + 1)} has an empty " + f"header cell; recovered '{after}' from the sheets that label it." + ) + headers = recovered + + for column_index, value_count in find_dropped_data_columns(headers, data): + logger.bind(error_code="blank_header_with_data").warning( + f"Sheet '{sheet_name}': column {get_column_letter(column_index + 1)} holds " + f"{value_count} values but its header cell is empty and no other sheet names " + "it, so the column is dropped. Fix the header in the source tracker to recover it." + ) + + valid_headers, filtered_data = filter_valid_columns(headers, data) + + valid_headers, filtered_data = merge_duplicate_columns_data(valid_headers, filtered_data) + + # Create DataFrame with ALL columns explicitly as String type to ensure consistent schema + # across all files and avoid type inference issues (Null vs String dtype) + df = pl.DataFrame( + { + header: pl.Series( + [str(row[i]) if row[i] is not None else None for row in filtered_data], + dtype=pl.String, + ) + for i, header in enumerate(valid_headers) + } + ) + + logger.info(f"Extracted {len(df)} rows x {len(df.columns)} cols from sheet '{sheet_name}'") + + return df + + +def harmonize_patient_data_columns( + df: pl.DataFrame, + mapper: ColumnMapper | None = None, + strict: bool = False, +) -> pl.DataFrame: + """Harmonize patient data columns using synonym mappings. + + Renames columns from their various synonyms (e.g., "Patient ID", "ID", + "Patient ID*") to standardized column names (e.g., "patient_id"). + + Args: + df: DataFrame with raw column names from tracker + mapper: ColumnMapper to use (if None, loads default patient mapper) + strict: If True, raise error if unmapped columns exist + If False, keep unmapped columns as-is (default) + + Returns: + DataFrame with standardized column names + + Raises: + ValueError: If strict=True and unmapped columns exist + + Example: + >>> raw_df = pl.DataFrame({ + ... "Patient ID*": ["MY_SU001", "MY_SU002"], + ... "Age": [25, 30], + ... }) + >>> harmonized = harmonize_patient_data_columns(raw_df) + >>> harmonized.columns + ['patient_id', 'age'] + """ + if mapper is None: + mapper = load_patient_mapper() + + renamed_df = mapper.rename_columns(df, strict=strict) + + logger.info( + f"Harmonized columns: {len(df.columns)} -> {len(renamed_df.columns)} " + f"({len(df.columns) - len(renamed_df.columns)} columns removed)" + if len(df.columns) != len(renamed_df.columns) + else f"Harmonized {len(renamed_df.columns)} columns" + ) + + return renamed_df + + +def join_static_sheet( + df_monthly: pl.DataFrame, + static_sheet: pl.DataFrame, + suffix: str, + sheet_name: str, +) -> pl.DataFrame: + """Attach a per-patient sheet's columns onto that patient's month rows. + + Used for both whole-tracker joins: the ``Patient List`` demographics and + the ``Annual`` sheet. + + Keyed on the *normalized* ID rather than the raw one (ticket 58). A month + sheet routinely spells an ID differently from the Patient List in the same + workbook -- 2023/2024 Mahosot write ``LA-MH056`` against a Patient List + entry of ``LA_MH056`` -- and on the raw key those rows kept their + measurements and silently lost every static column. Measured across the + real 254-tracker corpus, the raw key misses 695 Patient List rows in 6 + files (680 recoverable, 6,863 cells) and 132 Annual rows (19 recoverable, + 40 cells); the rest name a patient the sheet does not list at all, which is + a source defect rather than a key mismatch. + + The key is derived for the join only: ``patient_id`` in the returned frame + is still the spelling the month sheet used, which is what the raw layer + promises: the raw stage records what the workbook says, and only cleaning + resolves identity. + """ + key = "__static_join_key" + static = static_sheet.with_columns( + normalize_patient_id_expr(pl.col("patient_id")).alias(key) + ).drop("patient_id") + + # Normalization introduces no new key collisions anywhere in the corpus, + # but if a sheet ever does carry two entries folding to one key, a fan-out + # would silently duplicate that patient's month rows -- so collapse to the + # first entry rather than let the join multiply rows. + deduped = static.unique(subset=[key], keep="first", maintain_order=True) + if deduped.height != static.height: + logger.bind(error_code="invalid_tracker").warning( + f"'{sheet_name}' has {static.height - deduped.height} entries whose IDs " + "differ only by hyphen or transfer-clinic suffix; keeping the first of each" + ) + + return ( + df_monthly.with_columns(normalize_patient_id_expr(pl.col("patient_id")).alias(key)) + .join(deduped, on=key, how="left", suffix=suffix) + .drop(key) + ) + + +def read_all_patient_sheets( + tracker_file: Path, + mapper: ColumnMapper | None = None, + error_collector: ErrorCollector | None = None, +) -> pl.DataFrame: + """Read patient data from all month sheets in a tracker file. + + Orchestrates the complete extraction process: + 1. Find all month sheets + 2. Extract tracker year + 3. For each month sheet: + - Extract raw data + - Harmonize column names + - Merge duplicate columns + - Add metadata (sheet_name, tracker_month, tracker_year, file_name) + 4. Combine all sheets + 5. Filter invalid rows (no patient_id and no name) + + Args: + tracker_file: Path to the tracker Excel file + mapper: ColumnMapper to use (if None, loads default patient mapper) + error_collector: ErrorCollector for tracking data quality issues (optional) + + Returns: + Combined DataFrame with all patient data from all month sheets + + Raises: + ValueError: If no month sheets found or year cannot be determined + + Example: + >>> df = read_all_patient_sheets(Path("2024_Clinic.xlsx")) + >>> "patient_id" in df.columns + True + >>> "tracker_month" in df.columns + True + >>> "tracker_year" in df.columns + True + """ + logger.info(f"Reading all patient sheets from {tracker_file.name}") + + # Load mapper once for all sheets + if mapper is None: + mapper = load_patient_mapper() + + # Load workbook once and reuse across all sheets + wb = load_workbook( + tracker_file, read_only=True, data_only=True, keep_vba=False, keep_links=False + ) + + month_sheets = find_month_sheets(wb) + if not month_sheets: + wb.close() + raise ValueError(f"No month sheets found in {tracker_file.name}") + + year = get_tracker_year(tracker_file, month_sheets) + logger.info(f"Processing {len(month_sheets)} month sheets for year {year}") + + all_sheets_data = [] + + layouts = collect_sheet_layouts(wb, month_sheets, mapper=mapper) + + if mapper is None: + mapper = load_patient_mapper() + headers_by_sheet = {sheet: layout.headers for sheet, layout in layouts.items()} + for change in find_layout_changes(headers_by_sheet, mapper): + column = get_column_letter(change.index + 1) + if change.renames_only: + logger.info( + f"Column {column} is spelled differently across month sheets " + f"({', '.join(change.headers)}); all map to '{change.canonical[0]}'." + ) + else: + logger.bind(error_code="tracker_layout_changed").warning( + f"Column {column} does not mean the same thing in every month sheet: " + f"{', '.join(change.headers)} -> {', '.join(change.canonical)}. " + "A tracker should keep one layout for the whole year; check the workbook." + ) + + for sheet_name in month_sheets: + logger.info(f"Processing sheet: {sheet_name}") + + siblings = [layout.headers for name, layout in layouts.items() if name != sheet_name] + df_sheet = extract_patient_data( + tracker_file, + sheet_name, + year, + mapper=mapper, + workbook=wb, + layout=layouts.get(sheet_name), + sibling_headers=siblings, + ) + + if df_sheet.is_empty(): + logger.bind(error_code="invalid_tracker").warning( + f"Sheet '{sheet_name}' has no data, skipping" + ) + continue + + df_sheet = harmonize_patient_data_columns(df_sheet, mapper=mapper, strict=False) + + if "patient_id" not in df_sheet.columns: + logger.bind(error_code="invalid_tracker").warning( + f"Sheet '{sheet_name}' has no 'patient_id' column after harmonization, skipping" + ) + continue + + try: + month_num = extract_tracker_month(sheet_name) + except ValueError as e: + logger.bind(error_code="invalid_tracker").warning( + f"Could not extract month from '{sheet_name}': {e}, skipping" + ) + continue + + # Derived metadata (year, month) use Int64; text metadata (sheet_name, etc.) use String + clinic_id = tracker_file.parent.name + file_name = tracker_file.stem + df_sheet = df_sheet.with_columns( + [ + pl.lit(sheet_name, dtype=pl.String).alias("sheet_name"), + pl.lit(month_num, dtype=pl.Int64).alias("tracker_month"), + pl.lit(year, dtype=pl.Int64).alias("tracker_year"), + pl.lit(file_name, dtype=pl.String).alias("file_name"), + pl.lit(clinic_id, dtype=pl.String).alias("clinic_id"), + ] + ) + + all_sheets_data.append(df_sheet) + + if not all_sheets_data: + raise ValueError(f"No valid patient data found in any month sheets of {tracker_file.name}") + + # diagonal_relaxed: sheets within one workbook differ in which columns they + # carry and in inferred dtype (Null vs String for an all-blank column), and + # a strict concat would fail on either. + logger.info(f"Combining {len(all_sheets_data)} sheets...") + df_combined = pl.concat(all_sheets_data, how="diagonal_relaxed") + + initial_rows = len(df_combined) + + # Track rows with missing patient_id for error reporting + missing_patient_id_rows = df_combined.filter(pl.col("patient_id").is_null()) + missing_count = len(missing_patient_id_rows) + + if missing_count > 0: + logger.bind(error_code="invalid_value").error( + f"Found {missing_count} rows with missing patient_id in {tracker_file.name} - " + f"these rows will be excluded from processing" + ) + + # Log to ErrorCollector if available + if error_collector is not None: + for row in missing_patient_id_rows.iter_rows(named=True): + sheet_name = row.get("sheet_name", "unknown") + name_value = row.get("name", "") + error_collector.add_error( + file_name=tracker_file.stem, + patient_id="MISSING", + column="patient_id", + original_value=None, + error_message=( + f"Row in sheet '{sheet_name}' has missing patient_id (name: {name_value})" + ), + error_code="missing_required_field", + script="extract", + function_name="read_all_patient_sheets", + ) + + # Filter out ALL rows with missing patient_id + df_combined = df_combined.filter(pl.col("patient_id").is_not_null()) + + # Filter out empty rows (both patient_id and name are null/empty) + # This is redundant now but kept for clarity + if "name" in df_combined.columns: + df_combined = df_combined.filter( + ~( + (pl.col("patient_id").str.strip_chars() == "") + & (pl.col("name").is_null() | (pl.col("name").str.strip_chars() == "")) + ) + ) + + # Filter out rows where both patient_id and name are numeric zeros (0, 0.0, "0", "0.0", etc.) + if "name" in df_combined.columns: + df_combined = df_combined.filter( + ~( + pl.col("patient_id").str.strip_chars().is_in(["0", "0.0"]) + & pl.col("name").str.strip_chars().is_in(["0", "0.0"]) + ) + ) + + # Filter out rows with patient_id starting with "#" (Excel errors like #REF!). + # Dropped rather than sentinelled to "Undefined" the way fix_patient_id + # handles a misspelled ID: #REF! is not an identifier a clinic could + # reconcile against its records, so keeping the row would pool its + # measurements with every other unidentified patient under one group key. + # The measurements are real, though (2026 Preah Kossamak's May26 sheet loses + # a whole month this way), so each discard is reported for source correction + # instead of vanishing into the "filtered N invalid rows" count. + excel_error_id_rows = df_combined.filter(pl.col("patient_id").str.starts_with("#")) + if len(excel_error_id_rows) > 0: + logger.bind(error_code="excel_error_patient_id").error( + f"Dropped {len(excel_error_id_rows)} rows from {tracker_file.name} whose patient ID " + "cell holds an Excel formula error - the patient cannot be identified, so their " + "measurements are discarded; the workbook needs correcting" + ) + if error_collector is not None: + for row in excel_error_id_rows.iter_rows(named=True): + error_collector.add_error( + file_name=tracker_file.stem, + patient_id="MISSING", + column="patient_id", + original_value=row["patient_id"], + error_message=( + f"Row in sheet '{row.get('sheet_name', 'unknown')}' has an Excel formula " + f"error ({row['patient_id']}) where its patient ID should be; the row is " + "dropped because the patient cannot be identified" + ), + error_code="excel_error_patient_id", + script="extract", + function_name="read_all_patient_sheets", + ) + + df_combined = df_combined.filter(~pl.col("patient_id").str.starts_with("#")) + + filtered_rows = initial_rows - len(df_combined) + if filtered_rows > 0: + logger.info(f"Filtered out {filtered_rows} invalid rows total") + + # Excel formula-error strings are deliberately NOT stripped here (ticket + # 27): the raw layer records what the source cell actually contained. + # a4d.clean.converters.normalize_excel_formula_errors nulls and logs them. + + # Use already-loaded workbook for sheet checking + all_sheets = wb.sheetnames + + # Process Patient List sheet if it exists + if "Patient List" in all_sheets: + logger.info("Processing 'Patient List' sheet...") + try: + patient_list = extract_patient_data( + tracker_file, "Patient List", year, mapper=mapper, workbook=wb + ) + if not patient_list.is_empty(): + patient_list = harmonize_patient_data_columns( + patient_list, mapper=mapper, strict=False + ) + + if "patient_id" in patient_list.columns: + # Filter out rows with missing patient_id + patient_list = patient_list.filter(pl.col("patient_id").is_not_null()) + + # Filter out numeric zeros and Excel errors + if "name" in patient_list.columns: + patient_list = patient_list.filter( + ~( + pl.col("patient_id").str.strip_chars().is_in(["0", "0.0"]) + & pl.col("name").str.strip_chars().is_in(["0", "0.0"]) + ) + ) + + patient_list = patient_list.filter(~pl.col("patient_id").str.starts_with("#")) + + # Drop the monthly hba1c_baseline and name before joining: + # the Patient List is the authority for both, and keeping + # each side's copy collides the names on join. + df_monthly = ( + df_combined.drop("hba1c_baseline") + if "hba1c_baseline" in df_combined.columns + else df_combined + ) + patient_list_join = ( + patient_list.drop("name") + if "name" in patient_list.columns + else patient_list + ) + + df_combined = join_static_sheet( + df_monthly, patient_list_join, ".static", "Patient List" + ) + logger.info(f"Joined {len(patient_list)} Patient List records") + else: + logger.bind(error_code="invalid_tracker").warning( + "Patient List sheet has no 'patient_id' column after harmonization" + ) + else: + logger.bind(error_code="invalid_tracker").warning("Patient List sheet is empty") + except Exception as e: + logger.bind(error_code="invalid_tracker").warning( + f"Could not process Patient List sheet: {e}" + ) + + # Process Annual sheet if it exists + if "Annual" in all_sheets: + logger.info("Processing 'Annual' sheet...") + try: + annual_data = extract_patient_data( + tracker_file, "Annual", year, mapper=mapper, workbook=wb + ) + if not annual_data.is_empty(): + annual_data = harmonize_patient_data_columns( + annual_data, mapper=mapper, strict=False + ) + + if "patient_id" in annual_data.columns: + # Filter out rows with missing patient_id + annual_data = annual_data.filter(pl.col("patient_id").is_not_null()) + + # Filter out numeric zeros and Excel errors + if "name" in annual_data.columns: + annual_data = annual_data.filter( + ~( + pl.col("patient_id").str.strip_chars().is_in(["0", "0.0"]) + & pl.col("name").str.strip_chars().is_in(["0", "0.0"]) + ) + ) + + annual_data = annual_data.filter(~pl.col("patient_id").str.starts_with("#")) + + # status and name come from the Patient List, not here. + cols_to_drop = [col for col in ["status", "name"] if col in annual_data.columns] + annual_data_join = ( + annual_data.drop(cols_to_drop) if cols_to_drop else annual_data + ) + + df_combined = join_static_sheet( + df_combined, annual_data_join, ".annual", "Annual" + ) + logger.info(f"Joined {len(annual_data)} Annual records") + else: + logger.bind(error_code="invalid_tracker").warning( + "Annual sheet has no 'patient_id' column after harmonization" + ) + else: + logger.bind(error_code="invalid_tracker").warning("Annual sheet is empty") + except Exception as e: + logger.bind(error_code="invalid_tracker").warning( + f"Could not process Annual sheet: {e}" + ) + + # Close workbook after all processing + wb.close() + + logger.info( + f"Successfully extracted {len(df_combined)} total rows " + f"from {len(all_sheets_data)} month sheets" + ) + + # Reorder: metadata first, then patient data + # (tracker_year, tracker_month, clinic_id, patient_id) + priority_cols = ["tracker_year", "tracker_month", "clinic_id", "patient_id"] + existing_priority = [c for c in priority_cols if c in df_combined.columns] + other_cols = [c for c in df_combined.columns if c not in priority_cols] + df_combined = df_combined.select(existing_priority + other_cols) + + return df_combined + + +def export_patient_raw( + df: pl.DataFrame, + tracker_file: Path, + output_dir: Path, +) -> Path: + """Export raw patient data to parquet file. + + - Filename: {tracker_name}_patient_raw.parquet + - Location: output_dir/{tracker_name}_patient_raw.parquet + + Args: + df: Patient DataFrame to export + tracker_file: Path to original tracker file (used to extract tracker_name) + output_dir: Directory to write parquet file (e.g., data_root/output/patient_data_raw) + + Returns: + Path to the written parquet file + + Example: + >>> df = read_all_patient_sheets(Path("2024_Clinic.xlsx")) + >>> output_path = export_patient_raw( + ... df, + ... Path("2024_Clinic.xlsx"), + ... Path("output/patient_data_raw") + ... ) + >>> output_path.name + '2024_Clinic_patient_raw.parquet' + """ + # Extract tracker name (filename without extension) + tracker_name = tracker_file.stem + + # Create output filename: {tracker_name}_patient_raw.parquet + output_filename = f"{tracker_name}_patient_raw.parquet" + output_path = output_dir / output_filename + + # Ensure output directory exists + output_dir.mkdir(parents=True, exist_ok=True) + + # Write parquet file + logger.info(f"Writing {len(df)} rows to {output_path}") + df.write_parquet(output_path) + + logger.info(f"Successfully exported to {output_path}") + return output_path diff --git a/src/a4d/extract/product.py b/src/a4d/extract/product.py new file mode 100644 index 0000000..d84b43c --- /dev/null +++ b/src/a4d/extract/product.py @@ -0,0 +1,445 @@ +"""Product data extraction from Excel tracker files. + +Mirrors `src/a4d/extract/patient.py` structure but targets the product +section of each month sheet. The ``1.x`` step numbers below are this +module's own ordering vocabulary and are referenced by the tests. +""" + +import warnings +from pathlib import Path + +import openpyxl +import polars as pl +from loguru import logger + +from a4d.errors import ErrorCollector +from a4d.extract.common import ( + extract_tracker_month, + find_month_sheets, + get_tracker_year, +) +from a4d.extract.wide_format import handle_wide_format_cells, handle_wide_format_columns +from a4d.reference.synonyms import ColumnMapper, load_product_mapper + +warnings.filterwarnings("ignore", category=UserWarning, module=r"openpyxl\..*") + + +class ProductSectionNotFoundError(ValueError): + """Raised when a month sheet has no identifiable product section.""" + + +def find_product_section(ws) -> tuple[int, int]: + """Locate the start and end rows of the product data region (step 1.1). + + The start row is the header row containing the product/date/received + keywords. The end row is the row immediately before the patient + recruitment / patient data summary section. + + Returns (start_row, end_row) as 1-indexed Excel rows, both inclusive. + + Raises: + ProductSectionNotFoundError: if either bound cannot be located. + """ + max_row = ws.max_row or 0 + max_col = ws.max_column or 50 + + def _norm(raw_row) -> list[str]: + return [" ".join(str(cell or "").lower().split()) for cell in raw_row] + + start_row: int | None = None + end_row: int | None = None + + # 3-row sliding window over the streaming iterator: prev, curr, next. + # We evaluate curr (at curr_excel_row) against prev and next contexts. + prev_norm: list[str] | None = None + curr_norm: list[str] | None = None + curr_excel_row: int = 0 + + for excel_row, raw in enumerate( + ws.iter_rows(min_row=1, max_row=max_row, max_col=max_col, values_only=True), + start=1, + ): + next_norm = _norm(raw) + + if curr_norm is not None: + if start_row is None: + has_product = any("product" in cell for cell in curr_norm) + has_desc_support = any("description of support" in cell for cell in curr_norm) + has_date = any("date" in cell for cell in curr_norm) + has_received = any("received" in cell for cell in curr_norm) + has_units_received = any("units received" in cell for cell in curr_norm) + + matches_2024 = has_product and has_date and has_received + matches_2019_2021 = has_product and has_date and has_units_received + matches_2017_2018 = has_desc_support and has_date and has_units_received + + if matches_2024 or matches_2019_2021 or matches_2017_2018: + start_row = curr_excel_row + elif curr_excel_row > start_row: + has_patient_name = any("patient name" in cell for cell in next_norm) + has_patient_id = any("patient id" in cell for cell in next_norm) or any( + cell.strip() == "id" for cell in next_norm + ) + has_recruitment = any("patient recruitment" in cell for cell in curr_norm) + has_data_summary_above = prev_norm is not None and any( + "patient data summary" in cell for cell in prev_norm + ) + + if ( + (has_recruitment or has_data_summary_above) + and has_patient_name + and has_patient_id + ): + end_row = curr_excel_row - 1 + break + + prev_norm = curr_norm + curr_norm = next_norm + curr_excel_row = excel_row + + if start_row is None: + raise ProductSectionNotFoundError("Could not find start of product section") + if end_row is None: + raise ProductSectionNotFoundError("Could not find end of product section") + + return start_row, end_row + + +def extract_product_data(ws, start_row: int, end_row: int) -> pl.DataFrame: + """Read the product region and promote its first row to headers (step 1.2). + + Returns a DataFrame with all columns typed as ``pl.String``. Type + coercion is deferred to the cleaning phase (Sprint 3). + """ + max_col = ws.max_column or 50 + + all_rows = list( + ws.iter_rows(min_row=start_row, max_row=end_row, max_col=max_col, values_only=True) + ) + + if len(all_rows) < 2: + return pl.DataFrame() + + header_raw = all_rows[0] + data_rows = all_rows[1:] + + last_col = 0 + for i in range(len(header_raw) - 1, -1, -1): + if header_raw[i] is not None or any( + row[i] is not None for row in data_rows if i < len(row) + ): + last_col = i + 1 + break + + if last_col == 0: + return pl.DataFrame() + + from collections import defaultdict + + col_names: list[str] = [] + for col_idx in range(last_col): + raw_name = header_raw[col_idx] + col_names.append(str(raw_name) if raw_name is not None else f"_unnamed_{col_idx}") + + header_positions: dict[str, list[int]] = defaultdict(list) + for idx, name in enumerate(col_names): + header_positions[name].append(idx) + + duplicated = [h for h, positions in header_positions.items() if len(positions) > 1] + if duplicated: + logger.debug(f"Merging {len(duplicated)} duplicate column groups: {duplicated}") + + data_dict: dict[str, list[str | None]] = {} + for header, positions in header_positions.items(): + values: list[str | None] = [] + for row in data_rows: + if len(positions) == 1: + pos = positions[0] + raw = row[pos] if pos < len(row) else None + values.append(str(raw) if raw is not None else None) + else: + parts: list[str] = [] + for pos in positions: + raw = row[pos] if pos < len(row) else None + text = str(raw) if raw is not None else None + if text: + parts.append(text) + values.append(",".join(parts) if parts else None) + data_dict[header] = values + + return pl.DataFrame(data_dict, schema=dict.fromkeys(data_dict, pl.String)) + + +def add_product_metadata( + df: pl.DataFrame, + sheet_name: str, + tracker_month: int, + tracker_year: int, + file_name: str, + clinic_id: str, +) -> pl.DataFrame: + """Append sheet/tracker metadata columns (step 1.8).""" + return df.with_columns( + [ + pl.lit(f"{tracker_month:02d}", dtype=pl.String).alias("product_table_month"), + pl.lit(float(tracker_year), dtype=pl.Float64).alias("product_table_year"), + pl.lit(sheet_name, dtype=pl.String).alias("product_sheet_name"), + pl.lit(file_name, dtype=pl.String).alias("file_name"), + pl.lit(clinic_id, dtype=pl.String).alias("clinic_id"), + ] + ) + + +def remove_header_rows(df: pl.DataFrame) -> pl.DataFrame: + """Drop residual header rows and fully empty rows (step 1.6).""" + if df.height == 0: + return df + + # A formula-emptied Excel cell surfaces as "" rather than None, so a + # null-only test leaves the row in place and it reaches the output as a + # phantom stock movement. + blank_exprs = [ + pl.col(name).is_null() | (pl.col(name).str.strip_chars() == "") + if dtype == pl.String + else pl.col(name).is_null() + for name, dtype in df.schema.items() + ] + df = df.filter(~pl.all_horizontal(blank_exprs)) + + if "product" in df.columns: + df = df.filter( + pl.col("product").is_null() + | ~pl.col("product") + .str.strip_chars() + .str.to_lowercase() + .is_in(["product", "patient data summary"]) + ) + + return df + + +def replace_extra_totals(df: pl.DataFrame) -> pl.DataFrame: + """Null ``product_units_released`` after a ``Total`` column (step 1.9).""" + if df.height == 0: + return df + + if "product_released_to" in df.columns: + # openpyxl preserves whitespace on read, so strip both ends before the + # "Total" comparison -- otherwise a padded label fails to match. + df = df.with_columns( + pl.col("product_released_to").str.strip_chars().alias("product_released_to") + ) + + if "product_units_released" not in df.columns: + return df + + col_idx = df.columns.index("product_units_released") + if col_idx < 2: + return df + + prev1 = df.columns[col_idx - 1] + prev2 = df.columns[col_idx - 2] + + def _total_mask(col_name: str) -> pl.Expr: + if df.schema[col_name] == pl.String: + return ( + pl.col(col_name) + .str.to_lowercase() + .str.contains("total", literal=True) + .fill_null(False) + ) + return pl.lit(False) + + mask = _total_mask(prev1) | _total_mask(prev2) + return df.with_columns( + pl.when(mask) + .then(None) + .otherwise(pl.col("product_units_released")) + .alias("product_units_released") + ) + + +def _count_orphan_released_units( + df: pl.DataFrame, + sheet_name: str, + file_name: str, + error_collector: ErrorCollector | None, +) -> None: + """Warn on orphan ``product_units_released`` values. + + Counts harmonized rows where ``product_released_to`` is null/whitespace + yet ``product_units_released`` carries a value, and emits one + ErrorCollector entry per sheet. A quantity released to nobody is either a + subtotal row or a missing recipient, and both need the workbook correcting + rather than a guess here. + + Whitespace-only ``product_released_to`` cells count as orphan: openpyxl + returns ``""`` for blank cells while ``_normalize_empty_strings_to_null`` + runs in clean rather than extract, so the check folds them into the null + branch here. + """ + if error_collector is None: + return + if "product_released_to" not in df.columns or "product_units_released" not in df.columns: + return + + released_to_blank = pl.col("product_released_to").is_null() | ( + pl.col("product_released_to").cast(pl.String).str.strip_chars() == "" + ) + has_released_units = pl.col("product_units_released").is_not_null() & ( + pl.col("product_units_released").cast(pl.String).str.strip_chars() != "" + ) + + count = df.filter(released_to_blank & has_released_units).height + if count == 0: + return + + logger.bind(error_code="invalid_tracker").warning( + f"Sheet '{sheet_name}' has {count} rows where product_released_to " + f"is missing next to product_units_released." + ) + error_collector.add_error( + file_name=file_name, + patient_id="unknown", + column="product_released_to", + original_value=str(count), + error_message=( + f"Sheet '{sheet_name}' has {count} rows where product_released_to " + f"is missing next to product_units_released." + ), + error_code="invalid_tracker", + script="script1", + function_name="read_product_data_step1", + ) + + +def _harmonize( + df: pl.DataFrame, + mapper: ColumnMapper, + sheet_name: str, + error_collector: ErrorCollector | None, + file_name: str, +) -> pl.DataFrame: + """Rename columns via the mapper, then drop any column not in the synonym schema (step 1.5).""" + unknown = [ + col for col in df.columns if not mapper.is_known_column(col) and col not in mapper.synonyms + ] + if unknown: + logger.bind(error_code="invalid_tracker").warning( + f"Sheet {sheet_name}: unknown column names: {unknown}." + ) + if error_collector is not None: + for col in unknown: + error_collector.add_error( + file_name=file_name, + patient_id="unknown", + column=col, + original_value=col, + error_message=f"Sheet {sheet_name}: unknown column '{col}'", + error_code="invalid_tracker", + script="script1", + function_name="harmonize_input_data_columns", + ) + + df = mapper.rename_columns(df) + known = set(mapper.synonyms.keys()) + keep = [c for c in df.columns if c in known] + return df.select(keep) if keep else df.clear() + + +def read_all_product_sheets( + tracker_file: Path, + mapper: ColumnMapper | None = None, + error_collector: ErrorCollector | None = None, +) -> pl.DataFrame: + """Run steps 1.1-1.10 across every month sheet, returning one combined DataFrame.""" + tracker_file = Path(tracker_file) + mapper = mapper or load_product_mapper() + + wb = openpyxl.load_workbook( + tracker_file, read_only=True, data_only=True, keep_vba=False, keep_links=False + ) + + month_sheets = find_month_sheets(wb) + if not month_sheets: + raise ValueError(f"No month sheets found in {tracker_file.name}") + + year = get_tracker_year(tracker_file, month_sheets) + filename = tracker_file.stem + clinic_id = tracker_file.parent.name + + per_sheet: list[pl.DataFrame] = [] + for sheet_name in month_sheets: + ws = wb[sheet_name] + try: + start, end = find_product_section(ws) + except ProductSectionNotFoundError as exc: + logger.bind(error_code="invalid_tracker").warning( + f"Sheet {sheet_name}: {exc}. Skipping." + ) + if error_collector is not None: + error_collector.add_error( + file_name=filename, + patient_id="unknown", + column="", + original_value="", + error_message=f"Sheet {sheet_name}: {exc}", + error_code="invalid_tracker", + script="script1", + function_name="find_product_section", + ) + continue + + df = extract_product_data(ws, start, end) + if df.height == 0: + continue + + df = handle_wide_format_columns(df, filename) + df = handle_wide_format_cells(df, filename) + + df = _harmonize(df, mapper, sheet_name, error_collector, filename) + if df.height == 0 or df.width == 0: + continue + + try: + month = extract_tracker_month(sheet_name) + except ValueError as exc: + logger.warning(f"Sheet {sheet_name}: month unparseable ({exc}). Skipping.") + continue + + df = remove_header_rows(df) + df = add_product_metadata(df, sheet_name, month, year, filename, clinic_id) + _count_orphan_released_units(df, sheet_name, filename, error_collector) + df = replace_extra_totals(df) + + if df.height > 0: + per_sheet.append(df) + + wb.close() + + if not per_sheet: + logger.bind(error_code="empty_product_data").warning( + f"Empty product data: no product section found in any sheet of {filename}." + ) + return pl.DataFrame() + + # Excel formula-error strings are deliberately preserved here (ticket 27): + # the raw layer records what the source cell actually contained. + # a4d.clean.converters.normalize_excel_formula_errors nulls and logs them. + return pl.concat(per_sheet, how="diagonal_relaxed") + + +def export_product_raw( + df: pl.DataFrame, + tracker_file: Path, + output_dir: Path, +) -> Path: + """Write raw product DataFrame to ``{output_dir}/{tracker_name}_product_raw.parquet``.""" + tracker_file = Path(tracker_file) + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + output_path = output_dir / f"{tracker_file.stem}_product_raw.parquet" + logger.info(f"Writing {df.height} rows to {output_path}") + df.write_parquet(output_path) + return output_path diff --git a/src/a4d/extract/wide_format.py b/src/a4d/extract/wide_format.py new file mode 100644 index 0000000..891bc7a --- /dev/null +++ b/src/a4d/extract/wide_format.py @@ -0,0 +1,150 @@ +"""Wide-format Mandalay tracker reshaping (extraction steps 1.4a and 1.4b). + +Two distinct formats exist in the Mandalay Children's Hospital trackers +that need long-format conversion before the main extraction flow +continues. +""" + +import re + +import polars as pl + + +def _rows_to_df(rows: list[dict], schema: dict) -> pl.DataFrame: + """Build a DataFrame from row-dicts using explicit schema (avoids inference pitfalls). + + A source column that was entirely null (e.g. "Units Released" before the + comma-cell split populates it) infers dtype ``Null`` from ``df.schema``; + pinning the output to that dtype would silently drop the new string + values this function writes into it. Widen ``Null`` columns to ``Utf8`` + (same fix as ``tables/logs.py``'s ``schema_overrides``, same root cause). + """ + if not rows: + return pl.DataFrame(schema=schema) + safe_schema = {name: (pl.Utf8 if dtype == pl.Null else dtype) for name, dtype in schema.items()} + columns = {name: [row.get(name) for row in rows] for name in schema} + return pl.DataFrame(columns, schema=safe_schema) + + +_RELEASED_TO_WIDE = "Released To (select from drop down list)" +_TOTAL = "Total Units Released" +_PER_PERSON = "Units Released per person" + + +def handle_wide_format_columns(df: pl.DataFrame, filename: str) -> pl.DataFrame: + """Expand 2020-2021 Mandalay wide-format columns into rows (step 1.4a). + + Gate: both ``Total Units Released`` and ``Units Released per person`` + columns must be present. Otherwise return ``df`` unchanged. + """ + del filename # gate is column-based, not filename-based + + cols = df.columns + if _TOTAL not in cols or _PER_PERSON not in cols or _RELEASED_TO_WIDE not in cols: + return df + if df.height == 0: + return df + + start_col_idx = cols.index(_RELEASED_TO_WIDE) + 1 + end_col_idx = cols.index(_PER_PERSON) - 1 + if start_col_idx > end_col_idx: + return df + + intermediate_cols = cols[start_col_idx : end_col_idx + 1] + + date_col = "Date" if "Date" in cols else next((c for c in cols if "Date" in c), None) + + new_rows: list[dict] = [] + for row in df.iter_rows(named=True): + new_rows.append(dict(row)) + + total = row.get(_TOTAL) + per_person = row.get(_PER_PERSON) + + if total is None or per_person is None: + continue + if str(total).strip() == "" or str(per_person).strip() == "": + continue + if total == per_person: + continue + + for col in intermediate_cols: + cell = row.get(col) + if cell is None: + continue + if isinstance(cell, str) and cell.strip() == "": + continue + + new_row = dict.fromkeys(cols) + if date_col is not None: + new_row[date_col] = row.get(date_col) + new_row[_RELEASED_TO_WIDE] = cell + new_row[_PER_PERSON] = per_person + new_rows.append(new_row) + + return _rows_to_df(new_rows, dict(df.schema)) + + +def handle_wide_format_cells(df: pl.DataFrame, filename: str) -> pl.DataFrame: + """Split 2017-2019 Mandalay comma-separated ``Released To`` cells (step 1.4b).""" + if not any(pat in filename for pat in ("2017_Mandalay", "2018_Mandalay", "2019_Mandalay")): + return df + if df.height == 0: + return df + + def _find_col(substr: str) -> str | None: + return next((c for c in df.columns if substr in c), None) + + released_to = _find_col("Released To") + units_released = _find_col("Units Released") + date_col = _find_col("Date") + received_from = _find_col("Received From") + + # Individual None checks (not all([...])) so ty/mypy narrow each name to + # `str` below instead of leaving them as `str | None`. + if released_to is None or units_released is None or date_col is None or received_from is None: + return df + + pattern = re.compile(r"(-\s*\d)|(,\s*)") + if not any(v is not None and pattern.search(str(v)) for v in df[released_to].to_list()): + return df + + cols = df.columns + new_rows: list[dict] = [] + for row in df.iter_rows(named=True): + new_rows.append(dict(row)) + + cell = row.get(released_to) + if cell is None or "," not in str(cell): + continue + + for fragment in str(cell).split(","): + fragment = fragment.strip() + if not fragment: + continue + + if "-" in fragment: + name, qty = fragment.split("-", 1) + name = name.strip() + qty = qty.strip() + else: + name, qty = fragment, None + + new_row = dict.fromkeys(cols) + new_row[date_col] = row.get(date_col) + new_row[received_from] = row.get(received_from) + new_row[released_to] = name or None + new_row[units_released] = qty + new_rows.append(new_row) + + result = _rows_to_df(new_rows, dict(df.schema)) + + comma_mask = pl.col(released_to).str.contains(",", literal=True).fill_null(False) + result = result.with_columns( + [ + pl.when(comma_mask).then(None).otherwise(pl.col(units_released)).alias(units_released), + pl.when(comma_mask).then(None).otherwise(pl.col(released_to)).alias(released_to), + ] + ) + + return result diff --git a/src/a4d/gcp/__init__.py b/src/a4d/gcp/__init__.py new file mode 100644 index 0000000..89b75e0 --- /dev/null +++ b/src/a4d/gcp/__init__.py @@ -0,0 +1,21 @@ +from a4d.gcp.bigquery import ( + TABLE_CONFIGS, + get_bigquery_client, + load_pipeline_tables, + load_table, +) +from a4d.gcp.storage import ( + download_tracker_files, + get_storage_client, + upload_output, +) + +__all__ = [ + "TABLE_CONFIGS", + "download_tracker_files", + "get_bigquery_client", + "get_storage_client", + "load_pipeline_tables", + "load_table", + "upload_output", +] diff --git a/src/a4d/gcp/bigquery.py b/src/a4d/gcp/bigquery.py new file mode 100644 index 0000000..45aafc2 --- /dev/null +++ b/src/a4d/gcp/bigquery.py @@ -0,0 +1,300 @@ +"""BigQuery table loading from parquet files. + +Uses the google-cloud-bigquery Python client to load parquet files, with +per-table clustering configuration. +""" + +from pathlib import Path + +import polars as pl +from google.api_core.exceptions import GoogleAPIError, NotFound +from google.cloud import bigquery +from loguru import logger + +from a4d.config import settings + +# Clustering fields per table. These are the columns consumers filter on +# most, and clustering on them is what keeps a full-corpus query cheap. +TABLE_CONFIGS: dict[str, list[str]] = { + "patient_data_monthly": ["clinic_id", "patient_id", "tracker_date"], + "patient_data_annual": ["patient_id", "tracker_date"], + "patient_data_static": ["clinic_id", "patient_id", "tracker_date"], + "product_data": [ + "clinic_id", + "product_released_to", + "product_table_year", + "product_table_month", + ], + "clinic_data_static": ["clinic_id"], + "logs": ["level", "error_code", "file_name", "function"], + "errors": ["file_name", "error_code", "patient_id", "column"], + "tracker_metadata": ["file_name", "clinic_code"], +} + +# Groups of BigQuery table names selectable via `a4d upload tables --only `. +TABLE_GROUPS: dict[str, set[str]] = { + "patient": {"patient_data_static", "patient_data_monthly", "patient_data_annual"}, + "product": {"product_data"}, + "clinic": {"clinic_data_static"}, + "logs": {"logs"}, + "errors": {"errors"}, + "metadata": {"tracker_metadata"}, +} + +# Maps the pipeline output file names to BigQuery table names. +# Note: table_logs.parquet uses this name from create_table_logs() in tables/logs.py. +PARQUET_TO_TABLE: dict[str, str] = { + "patient_data_static.parquet": "patient_data_static", + "patient_data_monthly.parquet": "patient_data_monthly", + "patient_data_annual.parquet": "patient_data_annual", + "product_data.parquet": "product_data", + "clinic_data_static.parquet": "clinic_data_static", + "table_logs.parquet": "logs", + "table_errors.parquet": "errors", + "tracker_metadata.parquet": "tracker_metadata", +} + + +def get_bigquery_client(project_id: str | None = None) -> bigquery.Client: + """Create a BigQuery client. + + Authentication uses Application Default Credentials (ADC): + - In Cloud Run / GCE: automatic via metadata server + - Locally: via `gcloud auth application-default login` + - In CI: via GOOGLE_APPLICATION_CREDENTIALS environment variable + + Args: + project_id: GCP project ID (defaults to settings.project_id) + + Returns: + Configured BigQuery client + """ + return bigquery.Client(project=project_id or settings.project_id) + + +def load_table( + parquet_path: Path, + table_name: str, + client: bigquery.Client | None = None, + dataset: str | None = None, + project_id: str | None = None, + replace: bool = True, +) -> bigquery.LoadJob: + """Load a parquet file into a BigQuery table. + + 1. Optionally deletes the existing table (replace=True by default) + 2. Loads the parquet file with clustering fields + + Args: + parquet_path: Path to the parquet file to load + table_name: BigQuery table name (e.g., "patient_data_monthly") + client: BigQuery client (created if not provided) + dataset: Dataset name (defaults to settings.dataset) + project_id: GCP project ID (defaults to settings.project_id) + replace: If True, deletes and recreates the table (default) + + Returns: + Completed LoadJob + + Raises: + FileNotFoundError: If parquet file doesn't exist + ValueError: If table_name is not in TABLE_CONFIGS + google.api_core.exceptions.GoogleAPIError: On BigQuery API errors + """ + if not parquet_path.exists(): + raise FileNotFoundError(f"Parquet file not found: {parquet_path}") + + dataset = dataset or settings.dataset + project_id = project_id or settings.project_id + + if client is None: + client = get_bigquery_client(project_id) + + table_ref = f"{project_id}.{dataset}.{table_name}" + logger.info(f"Loading {parquet_path.name} → {table_ref}") + + # WRITE_TRUNCATE preserves the existing clustering and schema, so a + # deliberate change to either would silently not take effect -- delete + # first instead. + if replace: + try: + client.delete_table(table_ref) + logger.info(f"Deleted existing table {table_ref} for fresh creation") + except NotFound: + pass + + # Configure the load job + job_config = bigquery.LoadJobConfig( + source_format=bigquery.SourceFormat.PARQUET, + write_disposition=( + bigquery.WriteDisposition.WRITE_TRUNCATE + if replace + else bigquery.WriteDisposition.WRITE_APPEND + ), + ) + + # Add clustering if configured for this table + clustering_fields = TABLE_CONFIGS.get(table_name) + if clustering_fields: + job_config.clustering_fields = clustering_fields + logger.info(f"Clustering fields: {clustering_fields}") + + # Load the parquet file + with open(parquet_path, "rb") as f: + load_job = client.load_table_from_file(f, table_ref, job_config=job_config) + + # Wait for completion + load_job.result() + + logger.info( + f"Loaded {load_job.output_rows} rows into {table_ref} " + f"({parquet_path.stat().st_size / 1024 / 1024:.2f} MB)" + ) + return load_job + + +def load_pipeline_tables( + tables_dir: Path, + client: bigquery.Client | None = None, + dataset: str | None = None, + project_id: str | None = None, + replace: bool = True, + only_tables: set[str] | None = None, +) -> dict[str, bigquery.LoadJob]: + """Load pipeline output tables into BigQuery. + + Scans the tables directory for known parquet files and loads each one + into the corresponding BigQuery table. + + Args: + tables_dir: Directory containing parquet table files (e.g., output/tables/) + client: BigQuery client (created if not provided) + dataset: Dataset name (defaults to settings.dataset) + project_id: GCP project ID (defaults to settings.project_id) + replace: If True, replaces existing tables + only_tables: If set, restrict loading to these BigQuery table names + (see TABLE_GROUPS for the named groups CLI callers select from). + None loads every known table. + + Returns: + Dictionary mapping table name to completed LoadJob + + Raises: + FileNotFoundError: If tables_dir doesn't exist + """ + if not tables_dir.exists(): + raise FileNotFoundError(f"Tables directory not found: {tables_dir}") + + if client is None: + project_id = project_id or settings.project_id + client = get_bigquery_client(project_id) + + logger.info(f"Loading pipeline tables from: {tables_dir}") + + results: dict[str, bigquery.LoadJob] = {} + + for parquet_name, table_name in PARQUET_TO_TABLE.items(): + if only_tables is not None and table_name not in only_tables: + continue + parquet_path = tables_dir / parquet_name + if parquet_path.exists(): + try: + job = load_table( + parquet_path=parquet_path, + table_name=table_name, + client=client, + dataset=dataset, + project_id=project_id, + replace=replace, + ) + results[table_name] = job + except Exception: + logger.exception(f"Failed to load table: {table_name}") + else: + logger.warning(f"Table file not found, skipping: {parquet_name}") + + considered = ( + len(PARQUET_TO_TABLE) + if only_tables is None + else sum(1 for t in PARQUET_TO_TABLE.values() if t in only_tables) + ) + logger.info(f"Successfully loaded {len(results)}/{considered} tables") + return results + + +def select_tracker_metadata( + client: bigquery.Client | None = None, + dataset: str | None = None, + project_id: str | None = None, +) -> pl.DataFrame | None: + """Read ``file_name, clinic_code, md5, complete`` from BigQuery. + + Used by ``a4d.state.source.load_previous_manifest`` to query the previous + run's tracker manifest for incremental processing. + + Returns ``None`` (rather than raising) on: + + * authentication failure (``DefaultCredentialsError``) — caller falls back + to local parquet, + * missing table (``NotFound``) — first-ever run, no manifest yet, + * any other ``GoogleAPIError`` — network issues etc., fall back rather + than block the pipeline. + + Schema fallback: if the BQ table predates the ``complete`` column being + published, the query is retried without it and ``complete`` is synthesised + as ``False`` for every row — forces a full reprocess, which is the safe + default when manifest provenance is uncertain. + """ + project_id = project_id or settings.project_id + dataset = dataset or settings.dataset + + if client is None: + try: + client = get_bigquery_client(project_id) + except Exception as e: + logger.warning(f"BigQuery client unavailable, skipping metadata query: {e}") + return None + + table_ref = f"{project_id}.{dataset}.tracker_metadata" + full_query = f"SELECT file_name, clinic_code, md5, complete FROM `{table_ref}`" + + full_schema = { + "file_name": pl.Utf8, + "clinic_code": pl.Utf8, + "md5": pl.Utf8, + "complete": pl.Boolean, + } + + try: + rows = list(client.query(full_query).result()) + data = {col: [r[col] for r in rows] for col in full_schema} + return pl.DataFrame(data, schema=full_schema) + except NotFound: + logger.info(f"BigQuery table not found, no previous manifest: {table_ref}") + return None + except GoogleAPIError as e: + # Schema fallback: retry without `complete` if the column is missing + # in the deployed table. The synthesised complete=False forces a + # full reprocess. + message = str(e).lower() + if "complete" in message and ("unrecognized name" in message or "not found" in message): + logger.warning( + f"BigQuery {table_ref} missing 'complete' column; " + "retrying without it and forcing full reprocess" + ) + try: + rows = list( + client.query(f"SELECT file_name, clinic_code, md5 FROM `{table_ref}`").result() + ) + fallback_schema = {k: v for k, v in full_schema.items() if k != "complete"} + data = {col: [r[col] for r in rows] for col in fallback_schema} + df = pl.DataFrame(data, schema=fallback_schema) + return df.with_columns(pl.lit(False).alias("complete")) + except GoogleAPIError as retry_err: + logger.warning(f"BigQuery schema-fallback query failed: {retry_err}") + return None + logger.warning(f"BigQuery query failed for {table_ref}: {e}") + return None + except Exception as e: + logger.warning(f"Unexpected error querying {table_ref}: {e}") + return None diff --git a/src/a4d/gcp/drive.py b/src/a4d/gcp/drive.py new file mode 100644 index 0000000..be76521 --- /dev/null +++ b/src/a4d/gcp/drive.py @@ -0,0 +1,65 @@ +"""Google Drive download utilities. + +Authentication uses Application Default Credentials (ADC), same as +GCS/BigQuery. +""" + +from pathlib import Path + +import google.auth +import google.auth.transport.requests +from loguru import logger + +# Google Drive file ID for clinic_data.xlsx. Hardcoded because this is a +# single, long-lived document owned by A4D, not a per-run input. +CLINIC_DATA_FILE_ID = "1HOxi0o9fTAoHySjW_M3F-09TRBnUITOzzxGx2HwRMAw" + +_DRIVE_API_URL = "https://www.googleapis.com/drive/v3/files" +_XLSX_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +_CHUNK_SIZE = 8 * 1024 * 1024 # 8 MB + + +def download_clinic_data(destination: Path) -> Path: + """Download clinic_data.xlsx from Google Drive to the destination directory. + + Uses ADC with Drive readonly scope. In Cloud Run the service account must + have 'Viewer' access to the file (or the shared drive it lives in). + + Args: + destination: Directory to write clinic_data.xlsx into + + Returns: + Path to the downloaded file + + Raises: + requests.HTTPError: If the Drive API returns a non-2xx status + """ + destination.mkdir(parents=True, exist_ok=True) + output_path = destination / "clinic_data.xlsx" + + logger.info(f"Downloading clinic_data.xlsx from Google Drive (file ID: {CLINIC_DATA_FILE_ID})") + + credentials, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/drive.readonly"]) + session = google.auth.transport.requests.AuthorizedSession(credentials) + + # clinic_data is a Google Sheets file — must use export endpoint, not alt=media. + + url = ( + f"{_DRIVE_API_URL}/{CLINIC_DATA_FILE_ID}/export" + f"?mimeType={_XLSX_MIME}&supportsAllDrives=true" + ) + response = session.get(url, stream=True) + if not response.ok: + logger.error(f"Drive API error {response.status_code}: {response.text}") + response.raise_for_status() + + bytes_written = 0 + with open(output_path, "wb") as f: + for chunk in response.iter_content(chunk_size=_CHUNK_SIZE): + f.write(chunk) + bytes_written += len(chunk) + + size_kb = bytes_written / 1024 + logger.info(f"Downloaded clinic_data.xlsx: {size_kb:.1f} KB -> {output_path}") + + return output_path diff --git a/src/a4d/gcp/storage.py b/src/a4d/gcp/storage.py new file mode 100644 index 0000000..503e54e --- /dev/null +++ b/src/a4d/gcp/storage.py @@ -0,0 +1,163 @@ +"""Google Cloud Storage operations for tracker file download and output upload. + +Uses the google-cloud-storage +Python client library. +""" + +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +from google.cloud import storage +from loguru import logger + +from a4d.config import settings + +_GCS_WORKERS = 16 # parallel connections; GCS supports many concurrent requests + + +def get_storage_client(project_id: str | None = None) -> storage.Client: + """Create a GCS client. + + Authentication uses Application Default Credentials (ADC): + - In Cloud Run / GCE: automatic via metadata server + - Locally: via `gcloud auth application-default login` + - In CI: via GOOGLE_APPLICATION_CREDENTIALS environment variable + + Args: + project_id: GCP project ID (defaults to settings.project_id) + + Returns: + Configured storage client + """ + return storage.Client(project=project_id or settings.project_id) + + +def _download_blob(blob: storage.Blob, destination: Path) -> Path | None: + """Download a single blob, skipping if the local file is already current. + + Uses blob.size (available from list_blobs metadata at no extra cost) to + detect unchanged files without reading the file content. + + Returns the local path if downloaded, None if skipped. + """ + assert blob.name is not None, "blob from list_blobs() always has a name" + local_path = destination / blob.name + + if local_path.exists() and local_path.stat().st_size == blob.size: + logger.debug(f"Skipping (unchanged): {blob.name}") + return None + + local_path.parent.mkdir(parents=True, exist_ok=True) + logger.debug(f"Downloading: {blob.name}") + blob.download_to_filename(str(local_path)) + return local_path + + +def download_tracker_files( + destination: Path, + bucket_name: str | None = None, + client: storage.Client | None = None, +) -> list[Path]: + """Download tracker files from GCS bucket. + + Downloads in parallel and skips files whose local size already matches + the blob size (equivalent to gsutil -m cp -n). + + Args: + destination: Local directory to download files to + bucket_name: GCS bucket name (defaults to settings.download_bucket) + client: Storage client (created if not provided) + + Returns: + List of downloaded file paths (excludes skipped files) + """ + bucket_name = bucket_name or settings.download_bucket + + if client is None: + client = get_storage_client() + + bucket = client.bucket(bucket_name) + destination.mkdir(parents=True, exist_ok=True) + + logger.info(f"Downloading tracker files from gs://{bucket_name} to {destination}") + + blobs = [b for b in bucket.list_blobs() if not b.name.endswith("/")] + logger.info(f"Found {len(blobs)} objects in bucket") + + downloaded: list[Path] = [] + + with ThreadPoolExecutor(max_workers=_GCS_WORKERS) as executor: + futures = {executor.submit(_download_blob, blob, destination): blob for blob in blobs} + for future in as_completed(futures): + try: + result = future.result() + if result is not None: + downloaded.append(result) + except Exception: + blob = futures[future] + logger.error(f"Failed to download: {blob.name}") + + skipped = len(blobs) - len(downloaded) + logger.info(f"Downloaded {len(downloaded)} files, skipped {skipped} unchanged") + return downloaded + + +def _upload_file(bucket: storage.Bucket, file_path: Path, blob_name: str) -> str: + """Upload a single file to GCS.""" + logger.debug(f"Uploading: {blob_name}") + blob = bucket.blob(blob_name) + blob.upload_from_filename(str(file_path)) + return blob_name + + +def upload_output( + source_dir: Path, + bucket_name: str | None = None, + prefix: str = "", + client: storage.Client | None = None, +) -> list[str]: + """Upload output directory to GCS bucket in parallel. + + Args: + source_dir: Local directory to upload + bucket_name: GCS bucket name (defaults to settings.upload_bucket) + prefix: Optional prefix for uploaded blob names + client: Storage client (created if not provided) + + Returns: + List of uploaded blob names + + Raises: + FileNotFoundError: If source directory doesn't exist + """ + if not source_dir.exists(): + raise FileNotFoundError(f"Source directory not found: {source_dir}") + + bucket_name = bucket_name or settings.upload_bucket + + if client is None: + client = get_storage_client() + + bucket = client.bucket(bucket_name) + + logger.info(f"Uploading {source_dir} to gs://{bucket_name}/{prefix}") + + files = [f for f in source_dir.rglob("*") if f.is_file()] + + def _blob_name(file_path: Path) -> str: + relative = file_path.relative_to(source_dir) + return f"{prefix}/{relative}" if prefix else str(relative) + + uploaded: list[str] = [] + + with ThreadPoolExecutor(max_workers=_GCS_WORKERS) as executor: + futures = {executor.submit(_upload_file, bucket, f, _blob_name(f)): f for f in files} + for future in as_completed(futures): + try: + uploaded.append(future.result()) + except Exception: + file_path = futures[future] + logger.exception(f"Failed to upload: {file_path}") + + logger.info(f"Uploaded {len(uploaded)} files to gs://{bucket_name}") + return uploaded diff --git a/src/a4d/gcp/verify.py b/src/a4d/gcp/verify.py new file mode 100644 index 0000000..e2cba65 --- /dev/null +++ b/src/a4d/gcp/verify.py @@ -0,0 +1,108 @@ +"""Coarse sanity check of a production run against a `just backup-bq` snapshot. + +Not a cell-by-cell output comparison (that lives in scripts/compare_outputs.py) +-- this only catches gross regressions (a table going near-empty, losing clinics, +or changing shape) right after a real production run. +""" + +from dataclasses import dataclass + +from google.cloud import bigquery + +from a4d.config import settings + +VERIFIED_TABLES = [ + "patient_data_static", + "patient_data_monthly", + "patient_data_annual", + "product_data", +] + +ROW_COUNT_DROP_THRESHOLD = 0.10 + + +@dataclass(frozen=True) +class TableStats: + row_count: int + distinct_clinics: int | None + columns: frozenset[str] + + +@dataclass(frozen=True) +class TableAnomaly: + table: str + reason: str + + +def diff_table_stats( + before: dict[str, TableStats], after: dict[str, TableStats] +) -> list[TableAnomaly]: + anomalies = [] + for table, before_stats in before.items(): + after_stats = after.get(table) + if after_stats is None: + anomalies.append(TableAnomaly(table, "missing after the run")) + continue + + if after_stats.columns != before_stats.columns: + added = sorted(after_stats.columns - before_stats.columns) + removed = sorted(before_stats.columns - after_stats.columns) + anomalies.append( + TableAnomaly(table, f"schema changed (added={added}, removed={removed})") + ) + + if before_stats.row_count > 0: + drop = (before_stats.row_count - after_stats.row_count) / before_stats.row_count + if drop > ROW_COUNT_DROP_THRESHOLD: + anomalies.append( + TableAnomaly( + table, + f"row count dropped {drop:.0%} " + f"({before_stats.row_count} -> {after_stats.row_count})", + ) + ) + + if ( + before_stats.distinct_clinics is not None + and after_stats.distinct_clinics is not None + and after_stats.distinct_clinics < before_stats.distinct_clinics + ): + anomalies.append( + TableAnomaly( + table, + "distinct clinics dropped " + f"({before_stats.distinct_clinics} -> {after_stats.distinct_clinics})", + ) + ) + + return anomalies + + +def fetch_table_stats( + client: bigquery.Client, + table_name: str, + dataset: str | None = None, + project_id: str | None = None, +) -> TableStats: + dataset = dataset or settings.dataset + project_id = project_id or settings.project_id + table_ref = f"{project_id}.{dataset}.{table_name}" + + table = client.get_table(table_ref) + columns = frozenset(field.name for field in table.schema) + has_clinic_id = "clinic_id" in columns + + if has_clinic_id: + query = ( + f"SELECT COUNT(*) AS row_count, COUNT(DISTINCT clinic_id) AS distinct_clinics " + f"FROM `{table_ref}`" + ) + else: + query = f"SELECT COUNT(*) AS row_count FROM `{table_ref}`" + row = next(iter(client.query(query).result())) + + return TableStats( + row_count=row.row_count, + distinct_clinics=row.distinct_clinics if has_clinic_id else None, + columns=columns, + ) diff --git a/src/a4d/logging.py b/src/a4d/logging.py new file mode 100644 index 0000000..366997d --- /dev/null +++ b/src/a4d/logging.py @@ -0,0 +1,172 @@ +"""Operational logging configuration using loguru. + +This module provides logging infrastructure for monitoring and debugging +the pipeline execution. Logs are exported to BigQuery for dashboard analysis +(success rates, error counts, processing times, etc.). + +For data quality errors (conversion failures, validation errors), +use the ErrorCollector class from a4d.errors instead. + +Usage: + The loguru logger is a singleton. Once configured with setup_logging(), + all imports of 'from loguru import logger' will use the same configuration. + + >>> from a4d.logging import setup_logging, file_logger + >>> setup_logging(output_root=Path("output"), log_name="script1") + >>> + >>> # In processing code: + >>> from loguru import logger + >>> with file_logger("clinic_001_patient", output_root, tracker_year=2024, tracker_month=10): + ... logger.info("Processing started", rows=150) + ... logger.warning("Missing column", column="hba1c_updated_date") +""" + +import sys +import threading +from collections.abc import Generator +from contextlib import contextmanager +from pathlib import Path + +from loguru import logger + + +def _main_thread_only(record) -> bool: # noqa: ANN001 + """Filter that passes only log records from the main thread. + + Used on the console handler when running parallel workers so that + worker thread logs don't flood the console or break tqdm progress bars. + Worker logs still reach their per-tracker JSON file handlers. + """ + return threading.current_thread() is threading.main_thread() + + +def setup_logging( + output_root: Path, + log_name: str, + level: str = "INFO", + console: bool = True, + console_level: str | None = None, + console_main_thread_only: bool = False, +) -> None: + """Configure loguru for pipeline-wide operational logging. + + Creates both console (colored, human-readable) and file (JSON for BigQuery) + handlers. All logs in the JSON file include context variables from + contextualize() for analysis in Looker Studio. + + Args: + output_root: Root output directory (logs will be in output_root/logs/) + log_name: Base name for the log file (e.g., "script1_extract") + level: Minimum file log level (DEBUG, INFO, WARNING, ERROR) + console: Whether to add console handler (set False for CLI with progress bars) + console_level: Console log level (None = use level, or set to ERROR for quiet mode) + + Example: + >>> setup_logging(Path("output"), "script1_extract") + >>> logger.info("Processing started", total_trackers=10) + + >>> # Quiet mode for CLI with progress bars + >>> setup_logging(Path("output"), "pipeline", console_level="ERROR") + """ + log_dir = output_root / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + log_file = log_dir / f"main_{log_name}.log" + + # Remove default handler + logger.remove() + + # Console handler: pretty, colored output for monitoring + if console: + console_log_level = console_level if console_level is not None else level + logger.add( + sys.stdout, + level=console_log_level, + colorize=True, + filter=_main_thread_only if console_main_thread_only else None, + format=( + "{time:HH:mm:ss} | " + "{level: <8} | " + "{message}" + ), + ) + + # File handler: JSON output for BigQuery upload + # serialize=True means all context from contextualize() is included + logger.add( + log_file, + level="DEBUG", # Capture all levels in file + serialize=True, # JSON format with all fields + rotation="100 MB", + retention="30 days", + compression="zip", + ) + + if console: + logger.info("Logging initialized", log_file=str(log_file), level=level) + + +@contextmanager +def file_logger( + file_name: str, + output_root: Path, + tracker_year: int | None = None, + tracker_month: int | None = None, + level: str = "DEBUG", +) -> Generator: + """Context manager for per-tracker file logging with context. + + Creates a separate log file for a specific tracker and sets context + variables (file_name, tracker_year, tracker_month) that are automatically + included in all log records within this context. + + All logs are JSON formatted and will be aggregated for BigQuery upload. + + Args: + file_name: Name of the tracker file (e.g., "clinic_001_patient") + output_root: Root output directory (logs will be in output_root/logs/) + tracker_year: Year from the tracker (for dashboard filtering) + tracker_month: Month from the tracker (for dashboard filtering) + level: Minimum log level for this file handler + + Yields: + None (use logger directly within context) + + Example: + >>> with file_logger("clinic_001_patient", output_root, 2024, 10): + ... logger.info("Processing patient data", rows=150) + ... logger.warning("Missing column", column="hba1c_updated_date") + ... # All logs include file_name, tracker_year, tracker_month + """ + log_dir = output_root / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + log_file = log_dir / f"{file_name}.log" + + # Remove old log file if exists + if log_file.exists(): + log_file.unlink() + + # Add file-specific handler (JSON only, no console) + handler_id = logger.add( + log_file, + level=level, + serialize=True, # JSON format + ) + + # Build context dict (only include non-None values) + context = {"file_name": file_name} + if tracker_year is not None: + context["tracker_year"] = tracker_year + if tracker_month is not None: + context["tracker_month"] = tracker_month + + # Use contextualize to add file_name, tracker_year, tracker_month to all logs + with logger.contextualize(**context): + try: + yield + except Exception: + # Log exception with full traceback + logger.bind(error_code="critical_abort").exception("Processing failed") + raise + finally: + # Remove the handler + logger.remove(handler_id) diff --git a/src/a4d/migration/__init__.py b/src/a4d/migration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/a4d/migration/compare.py b/src/a4d/migration/compare.py new file mode 100644 index 0000000..61b250e --- /dev/null +++ b/src/a4d/migration/compare.py @@ -0,0 +1,3415 @@ +"""R-vs-Python output comparison. HISTORICAL -- retired, kept as a record. + +**This module and its docstrings are a snapshot of a finished piece of work, +not current guidance.** The R pipeline it compares against was retired on +2026-08-24; no new R output will ever be produced, so this tool will not be +run again. It is kept because its cause registry is the written evidence +behind the migration's conclusions -- every classifier below records a +divergence that was traced to a mechanism in real source workbooks, and +deleting them would delete the only durable record of why the Python pipeline +behaves as it does in the places where it deliberately differs. + +Read it accordingly: the R behaviour described here was true of the R pipeline +as it stood at retirement, and the R source is reachable at +``git show r-archive-removed^:r-archive/R/``. Every other module in this +package documents itself in its own terms -- if you are looking for why the +*Python* pipeline does something, look there, not here. + +Diffs two existing output directories (a Python run, the frozen R baseline) +across seven layers: shape (row count), ID divergence (identities present on +only one side), columns/dtypes, categorical divergence (label values present +on only one side, per column), aggregate totals, row-key overlap (rows whose +full row-alignment key found no partner on the other side, or fanned out via +a repeated key), and cell-by-cell -- the last two grouped together since both +depend on the same row-alignment key. This was migration-only tooling with a +defined end-of-life (R's retirement, now reached) -- deliberately never wired +into ``a4d.cli``. +""" + +import datetime +import itertools +import re +from collections import Counter +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Any + +import polars as pl + +from a4d.clean.date_parser import EXCEL_EPOCH, parse_date_flexible +from a4d.clean.glucose import ( + GLUCOSE_COLUMN_PAIRS, + MG_ANALYTICAL_MAX, + MG_ANALYTICAL_MIN, + MMOL_ANALYTICAL_MAX, + MMOL_ANALYTICAL_MIN, + MMOL_TO_MG_FACTOR, +) +from a4d.clean.glucose import ( + GLUCOSE_COLUMNS as GLUCOSE_UNIT_COLUMNS, +) +from a4d.clean.validators import load_numeric_ranges, load_validation_rules, sanitize_str +from a4d.config import settings +from a4d.extract.common import normalize_patient_id_expr + +SENTINEL_DATE = datetime.date(9999, 9, 9) + + +def normalize_date_column(df: pl.DataFrame, column: str) -> pl.DataFrame: + """Parse a raw string date column to a common ``date`` representation. + + R's raw extraction stores dates as unparsed source text -- an Excel + serial number (e.g. "42872.0") for cells Excel itself formats as dates, + or the literal free-text string otherwise -- while Python's raw + extraction already ISO-formats parsed dates. Comparing the two as raw + strings treats that representation difference as a false mismatch on + ~99% of product_entry_date rows (ticket 20). Reuses the same flexible + parser the cleaning stage already applies (including its Excel-serial + and typo handling), so a genuinely different underlying date -- not + just a different spelling of the same one -- still surfaces as a real + divergence. + + A column absent from ``df`` entirely is a no-op, matching + ``add_row_ordinal``'s convention for raw output that isn't + schema-normalized. + """ + if column not in df.columns: + return df + parsed = [parse_date_flexible(v) for v in df[column]] + return df.with_columns(pl.Series(column, parsed, dtype=pl.Date)) + + +def normalize_numeric_column(df: pl.DataFrame, column: str) -> pl.DataFrame: + """Parse a raw numeric-string column to ``float`` so float tolerance applies. + + R's and Python's own float-to-string conversions round a binary float's + trailing digits differently (e.g. "9.300000000000001" vs "9.3" for the + same value) -- a representation difference, not a real divergence, that + ``_values_differ``'s float tolerance already handles once both sides are + parsed back to ``float`` rather than compared as strings. A value that + isn't numeric (product_units_received's raw column also holds text like + "START BALANCE") passes through unchanged, so it still surfaces as an + ordinary string mismatch if it genuinely differs. + """ + if column not in df.columns: + return df + + def _try_float(value: Any) -> Any: + if value is None: + return None + try: + return float(value) + except TypeError, ValueError: + return value + + parsed = [_try_float(v) for v in df[column]] + return df.with_columns(pl.Series(column, parsed, dtype=pl.Object)) + + +def numeric_normalize_targets(df: pl.DataFrame, exclude: Sequence[str]) -> list[str]: + """Raw-stage columns to parse back to ``float`` before diffing. + + The cleaned schema is the wrong source for this list even though it is + derived rather than hand-written: it has no entry at all for the Patient + List join's ``.static`` copies, and it types a screening measurement as a + string because the same column can also hold "normal". Both still carry + plain floats whose string forms R and Python round differently, so both + were reported as mismatches. Since the raw stage stores every value as + text and ``normalize_numeric_column`` leaves non-numeric text untouched, + the correct scope is every column except the ones the row-alignment join + must keep as text. + """ + excluded = set(exclude) + return [c for c in df.columns if c not in excluded and not c.startswith("__")] + + +def string_numeric_normalize_targets(df: pl.DataFrame, exclude: Sequence[str]) -> list[str]: + """Cleaned-stage columns to parse back to ``float`` before diffing. + + Cleaning casts its numeric columns, which is why ticket 22 scoped numeric + normalization to the raw stage. The exception is a column the cleaned + schema types as a **string** -- a screening measurement that can also read + "normal", an hba1c that can read "<7" -- which cleaning therefore never + casts, so R's and Python's differing float-to-string rounding + (``4.8600000000000003`` against ``4.86``) survives into the cleaned output + exactly as it does at the raw stage. + + Derived from the frame's own dtypes rather than named, for the same reason + ``numeric_normalize_targets`` is: which columns are string-typed is the + schema's decision, and a hand-written copy of it drifts. + """ + excluded = set(exclude) + return [ + name + for name, dtype in df.schema.items() + if dtype == pl.String and name not in excluded and not name.startswith("__") + ] + + +def normalize_boolean_literal_column(df: pl.DataFrame, column: str) -> pl.DataFrame: + """Fold R's and Python's spellings of a boolean cell together. + + Where a source cell holds a genuine Excel boolean, readxl reads a logical + and R writes it as ``FALSE``; openpyxl reads a Python ``bool`` and Python + writes it as ``False``. Only an actual Excel boolean produces both + spellings at once, so this is a language convention, not a divergence -- + confirmed at the cleaned stage, where both sides already agree because + cleaning canonicalizes them (ticket 49: 20 raw-stage mismatches on + clinic_visit/remote_followup, 0 cleaned-stage). Only the exact literals + are folded, so free text that merely contains "false" is untouched. + """ + if column not in df.columns: + return df + lowered = pl.col(column).str.to_lowercase() + return df.with_columns( + pl.when(lowered.is_in(["true", "false"])) + .then(lowered) + .otherwise(pl.col(column)) + .alias(column) + ) + + +def whitespace_normalize_targets(df: pl.DataFrame, exclude: Sequence[str]) -> list[str]: + """Raw-stage columns to strip and line-ending-normalize before diffing. + + Scoped like ``numeric_normalize_targets`` and for the same reason: the + cleaned patient schema, though derived rather than hand-written, is the + wrong source at the raw stage. ``get_string_columns()`` omits raw-only + columns entirely (``dm_complications`` has no cleaned equivalent) and + types as ``Float64`` columns the raw stage still holds as text + (``insulin_injections``, ``hba1c_updated``), so readxl's ``trim_ws`` and + its ``\\r\\n`` line endings kept surfacing on exactly those columns as + mismatches while every cleaned-schema string column was already handled. + Since the raw stage stores every value as text, the correct scope is + every string column. + """ + excluded = set(exclude) + return [ + name + for name, dtype in df.schema.items() + if name not in excluded and not name.startswith("__") and dtype == pl.String + ] + + +def normalize_whitespace_column(df: pl.DataFrame, column: str) -> pl.DataFrame: + """Strip leading/trailing whitespace from a raw string column. + + readxl's ``read_xlsx`` (R's raw extraction) trims whitespace from every + character column by default (``trim_ws = TRUE``) and represents an + embedded line break as ``\\r\\n``; openpyxl-based Python extraction + preserves a cell's leading/trailing whitespace exactly as stored and + normalizes embedded breaks to ``\\n`` alone. Both defaults are + representation differences, not real divergences -- verified against + real raw output (ticket 22), where together they account for the large + majority of non-numeric, non-date raw-stage product column mismatches. + A whitespace-only cell strips to an empty string, which R's own + ``trim_ws`` reduces further to NA, so this normalizes that too. + """ + if column not in df.columns: + return df + stripped = pl.col(column).str.replace_all("\r\n", "\n").str.strip_chars() + return df.with_columns(pl.when(stripped == "").then(None).otherwise(stripped).alias(column)) + + +@dataclass(frozen=True) +class ShapeResult: + r_rows: int + py_rows: int + match: bool + + +def compare_shape(r_df: pl.DataFrame, py_df: pl.DataFrame) -> ShapeResult: + r_rows, py_rows = len(r_df), len(py_df) + return ShapeResult(r_rows=r_rows, py_rows=py_rows, match=r_rows == py_rows) + + +@dataclass(frozen=True) +class TotalsMismatch: + column: str + r_total: float + py_total: float + + +TOTALS_REL_TOL = 1e-9 +TOTALS_ABS_TOL = 1e-6 + + +def compare_totals( + r_df: pl.DataFrame, py_df: pl.DataFrame, numeric_cols: list[str] +) -> list[TotalsMismatch]: + mismatches = [] + for col in numeric_cols: + if col not in r_df.columns or col not in py_df.columns: + continue + r_total = float(r_df[col].sum() or 0.0) + py_total = float(py_df[col].sum() or 0.0) + tolerance = max(TOTALS_REL_TOL * max(abs(r_total), abs(py_total)), TOTALS_ABS_TOL) + if abs(r_total - py_total) > tolerance: + mismatches.append(TotalsMismatch(column=col, r_total=r_total, py_total=py_total)) + return mismatches + + +@dataclass(frozen=True) +class ColumnsResult: + only_in_r: list[str] + only_in_py: list[str] + dtype_mismatches: list[tuple[str, pl.DataType, pl.DataType]] + + +def compare_columns(r_df: pl.DataFrame, py_df: pl.DataFrame) -> ColumnsResult: + r_cols, py_cols = set(r_df.columns), set(py_df.columns) + common = sorted(r_cols & py_cols) + dtype_mismatches = [ + (col, r_df.schema[col], py_df.schema[col]) + for col in common + if r_df.schema[col] != py_df.schema[col] + ] + return ColumnsResult( + only_in_r=sorted(r_cols - py_cols), + only_in_py=sorted(py_cols - r_cols), + dtype_mismatches=dtype_mismatches, + ) + + +@dataclass(frozen=True) +class IdOverlapResult: + only_in_r: list[Any] + only_in_py: list[Any] + common_count: int + + +def compare_id_overlap(r_df: pl.DataFrame, py_df: pl.DataFrame, id_col: str) -> IdOverlapResult: + r_ids = set(r_df[id_col].drop_nulls().to_list()) + py_ids = set(py_df[id_col].drop_nulls().to_list()) + return IdOverlapResult( + only_in_r=sorted(r_ids - py_ids), + only_in_py=sorted(py_ids - r_ids), + common_count=len(r_ids & py_ids), + ) + + +@dataclass(frozen=True) +class CategoricalOverlap: + column: str + only_in_r: list[Any] + only_in_py: list[Any] + + +def compare_categorical_overlap( + r_df: pl.DataFrame, py_df: pl.DataFrame, categorical_cols: list[str] +) -> list[CategoricalOverlap]: + mismatches = [] + for col in categorical_cols: + if col not in r_df.columns or col not in py_df.columns: + continue + overlap = compare_id_overlap(r_df, py_df, col) + if overlap.only_in_r or overlap.only_in_py: + mismatches.append( + CategoricalOverlap( + column=col, only_in_r=overlap.only_in_r, only_in_py=overlap.only_in_py + ) + ) + return mismatches + + +# Name of the positional row-alignment column add_row_ordinal creates. +ROW_ORDINAL_COL = "__row_ordinal" + + +def add_row_ordinal( + df: pl.DataFrame, group_cols: list[str], ordinal_col: str = ROW_ORDINAL_COL +) -> tuple[pl.DataFrame, list[str]]: + """Add a within-group ordinal-position row-alignment key (ticket 17). + + Product's natural key -- (clinic_id, product, product_sheet_name, + product_entry_date) -- collapses onto far fewer distinct values than rows + exist wherever product_entry_date is null, causing join fan-out. Ordinal + position within (clinic_id, product_sheet_name), taken in each file's + existing row order, is a substitute validated directly against the real + R/Python output pair: ~99.99% of product rows align cleanly on it, with + the handful of exceptions being real, separately diagnosable divergences + (a clinic_id typo, un-trimmed sheet names) rather than a key design flaw. + + Group columns are whitespace-normalized for the join key only -- the + original column values are untouched, so e.g. a trailing space in R's + product_sheet_name still surfaces as an ordinary cell mismatch on that + column instead of silently breaking alignment for the whole group. + + A group column absent from ``df`` entirely (raw output isn't + schema-normalized -- a columnless file is possible, e.g. a + pre-product-tracking tracker year) keys as null rather than raising; + such a file is empty anyway and never reaches the join. + """ + group_key_cols = [] + exprs = [] + for col in group_cols: + key_col = f"__key_{col}" + if col not in df.columns: + exprs.append(pl.repeat(None, df.height, dtype=pl.Null).alias(key_col)) + group_key_cols.append(key_col) + continue + expr = pl.col(col) + if df.schema[col] == pl.Utf8: + expr = expr.str.strip_chars() + exprs.append(expr.alias(key_col)) + group_key_cols.append(key_col) + df = df.with_columns(exprs) + df = df.with_columns(pl.int_range(pl.len()).over(group_key_cols).alias(ordinal_col)) + return df, [*group_key_cols, ordinal_col] + + +# A duplicate group is brute-forced over all pairings, so it is bounded. The +# real data's largest two-sided patient duplicate group is 4 rows (2023_NPH); +# anything beyond this keeps positional pairing rather than costing 5040+ +# permutations for a group that is almost certainly a source defect of a +# different kind. +MAX_DUPLICATE_GROUP_FOR_MATCHING = 6 + + +def _pairing_cost( + r_df: pl.DataFrame, py_df: pl.DataFrame, value_cols: list[str], pairs: Sequence[tuple[int, int]] +) -> int: + return sum( + 1 + for r_index, py_index in pairs + for col in value_cols + if _values_differ(r_df[col][r_index], py_df[col][py_index]) + ) + + +def align_duplicate_rows( + r_df: pl.DataFrame, py_df: pl.DataFrame, group_cols: list[str] +) -> tuple[pl.DataFrame, pl.DataFrame, list[str]]: + """Align two frames on an identity key, breaking ties by content (ticket 45). + + ``add_row_ordinal`` alone makes a repeated identity key unique again, but + it does so by *position*, which assumes both sides emit a duplicated + patient's rows in the same order. They do at the raw stage and demonstrably + do not at the cleaned stage, where a positional tie-break pairs the wrong + two copies of the same patient and reports every differing column twice + over. + + Within a patient-and-sheet group the row order carries no meaning, so + duplicates are paired to minimise the number of differing cells -- + brute-forced, since such groups are tiny and rare (24 groups in 5 of 250+ + files). Position is kept unless some other pairing is *strictly* better, so + a group whose rows genuinely both changed is never permuted into looking + like agreement. Groups present on only one side, or larger than + ``MAX_DUPLICATE_GROUP_FOR_MATCHING``, keep positional order. + """ + r_keyed, key_cols = add_row_ordinal(r_df, group_cols) + py_keyed, _ = add_row_ordinal(py_df, group_cols) + group_key_cols = key_cols[:-1] + value_cols = [c for c in r_df.columns if c in py_df.columns and c not in group_cols] + + def _rows_by_group(df: pl.DataFrame) -> dict[tuple[Any, ...], list[int]]: + groups: dict[tuple[Any, ...], list[int]] = {} + for index, row in enumerate(df.select(group_key_cols).iter_rows()): + groups.setdefault(row, []).append(index) + return groups + + r_groups, py_groups = _rows_by_group(r_keyed), _rows_by_group(py_keyed) + py_ordinals = py_keyed[ROW_ORDINAL_COL].to_list() + + for group, r_indices in r_groups.items(): + py_indices = py_groups.get(group, []) + if len(r_indices) < 2 or len(r_indices) != len(py_indices): + continue + if len(r_indices) > MAX_DUPLICATE_GROUP_FOR_MATCHING: + continue + positional = list(zip(r_indices, py_indices, strict=True)) + best_cost = _pairing_cost(r_keyed, py_keyed, value_cols, positional) + best: Sequence[tuple[int, int]] = positional + for permutation in itertools.permutations(py_indices): + candidate = list(zip(r_indices, permutation, strict=True)) + cost = _pairing_cost(r_keyed, py_keyed, value_cols, candidate) + if cost < best_cost: + best_cost, best = cost, candidate + for ordinal, (_, py_index) in enumerate(best): + py_ordinals[py_index] = ordinal + + py_keyed = py_keyed.with_columns(pl.Series(ROW_ORDINAL_COL, py_ordinals)) + return r_keyed, py_keyed, key_cols + + +UnmatchedKey = tuple[tuple[Any, ...], int] + + +@dataclass(frozen=True) +class RowKeyOverlap: + """How many rows found a partner via the row-alignment key -- and which + did not. + + The counts alone proved insufficient in practice (ticket 47): the four + patient rows this measure newly flagged could only be named by querying + the parquets by hand, because a row that finds no partner never reaches a + cell comparison and so appears in no other sheet of the report. The keys + are therefore carried alongside the counts, each with the number of + surplus rows on that side, so a divergence can be triaged from the report + itself. + """ + + matched: int + r_unmatched: int + py_unmatched: int + r_only_keys: tuple[UnmatchedKey, ...] = () + py_only_keys: tuple[UnmatchedKey, ...] = () + + +def compare_row_key_overlap( + r_df: pl.DataFrame, py_df: pl.DataFrame, key_cols: list[str] +) -> RowKeyOverlap: + r_counts = Counter(r_df.select(key_cols).iter_rows()) + py_counts = Counter(py_df.select(key_cols).iter_rows()) + + matched = 0 + r_unmatched = 0 + py_unmatched = 0 + r_only: list[UnmatchedKey] = [] + py_only: list[UnmatchedKey] = [] + for key in r_counts.keys() | py_counts.keys(): + r_count, py_count = r_counts.get(key, 0), py_counts.get(key, 0) + paired = min(r_count, py_count) + matched += paired + r_unmatched += r_count - paired + py_unmatched += py_count - paired + if r_count > paired: + r_only.append((key, r_count - paired)) + if py_count > paired: + py_only.append((key, py_count - paired)) + + # Sorted on the rendered key so a report diffs cleanly run over run, and + # so a key holding nulls or mixed types cannot raise on comparison. + def rendered(item: UnmatchedKey) -> tuple[str, ...]: + return tuple(str(part) for part in item[0]) + + return RowKeyOverlap( + matched=matched, + r_unmatched=r_unmatched, + py_unmatched=py_unmatched, + r_only_keys=tuple(sorted(r_only, key=rendered)), + py_only_keys=tuple(sorted(py_only, key=rendered)), + ) + + +@dataclass(frozen=True) +class CellMismatch: + key: dict[str, Any] + column: str + r_value: Any + py_value: Any + # Set by compare_cells when order_group_cols is given: True if r_value + # appears anywhere in the Python side's own group for this column -- + # i.e. the mismatch could be a within-group row-order divergence + # (ticket 21) rather than a genuine content difference. Diagnostic only; + # never suppresses a mismatch, only informs its classification. + row_order_candidate: bool = False + # Set by compare_cells when order_group_cols is given: True if r_value, + # read as a Buddhist-era date and shifted to Gregorian, appears in the + # Python side's own group for this column (ticket 61). The cleaned stage's + # conversion re-sorts the converted row, and R's unconverted value is by + # construction absent from Python's group -- so plain value membership + # cannot see this re-sort, only the shifted value can. + era_shift_row_order_candidate: bool = False + # Set by compare_cells when order_group_cols is given and the key is + # positional: True if this column's value on the group's LAST row agrees + # on both sides. For a derived running total (product_balance), which is + # recomputed per row and so cannot travel with its row under a re-sort, + # this is the order-independent evidence row_order_candidate cannot + # supply -- the ledgers took different paths to the same closing figure + # (ticket 36). Diagnostic only, like row_order_candidate. + group_endpoint_matches: bool = False + # Set by compare_cells: True if any *other* column on this same row is + # itself a bare-year mismatch (ticket 52). age and t1d_diagnosis_age are + # derived from dob and t1d_diagnosis_date, so when one of those carries a + # bare year the two pipelines derive from different birth years -- a + # cascade of an already-decided cause, not a separate divergence. Keyed on + # the mechanism rather than on the derived values' shape, which alone + # cannot tell this apart from a genuine age disagreement. + row_has_bare_year_date: bool = False + # Set by compare_cells when unit_swapped_columns is given (ticket 44): + # True if this cell's glucose column pair was unit-swapped in this file. + # Whether a column is really recorded in mmol/L is a property of the whole + # column, so no per-cell test can establish it -- ticket 57 left six cells + # unclassified for exactly this reason. The fact is read back from the + # pipeline's own glucose_unit_swapped error records, not re-derived here. + column_unit_swapped: bool = False + # Set by compare_cells when static_join_missed_ids is given (ticket 63): + # True if this row's patient is one whose month sheets spell the ID + # differently from the Patient List in the same workbook. The raw stage + # reads that off the key itself, but cleaning normalizes patient_id on both + # sides, so by then the only surviving discriminator is a per-file fact + # derived from the raw output -- the same shape column_unit_swapped uses + # for a property of a column rather than of a cell. + static_join_missed_id: bool = False + # Set by compare_cells (ticket 63): True if R holds no value for this + # column anywhere in this file. R's column-level failures are file-level by + # construction -- a header it cannot map is unmapped for every row at once + # (ticket 62 measured recruitment_date's xml:space defect as exactly this: + # 91 files reading zero, none reading some) -- so this separates "R lost + # the column" from "R lost this row", which are otherwise the same null. + r_column_empty_in_file: bool = False + # Set by compare_cells (ticket 63): True if this row's dob is null on R and + # present on Python. age is derived from dob rather than published from the + # sheet, so a birth date only one side holds moves the derived column too -- + # a cascade, keyed on the mechanism like row_has_bare_year_date rather than + # on the derived values' shape, which cannot tell it from a real + # disagreement about someone's age. + row_dob_recovered: bool = False + # Set by compare_cells (ticket 44): for a mmol glucose column, the + # (r_value, py_value) of the same row's mg sibling. The mmol column is not + # independently recorded -- both pipelines derive it from the mg cell -- so + # the sibling is what says whether an mmol divergence is its own or an + # already-named one restated. + mg_sibling: tuple[Any, Any] | None = None + # Set by compare_cells when tracker_year_col is given (ticket 32): the + # tracker's own calendar year, taken from the Python side's own column + # rather than parsed out of the file name. It is what lets a classifier ask + # whether Python's *own* value is plausible for this tracker, instead of + # only describing how the two sides differ. + tracker_year: int | None = None + + +CELL_FLOAT_REL_TOL = 1e-9 +CELL_FLOAT_ABS_TOL = 1e-9 + + +def _values_differ(r_value: Any, py_value: Any) -> bool: + if r_value is None or py_value is None: + return r_value != py_value + if isinstance(r_value, float) and isinstance(py_value, float): + tolerance = max(CELL_FLOAT_REL_TOL * max(abs(r_value), abs(py_value)), CELL_FLOAT_ABS_TOL) + return abs(r_value - py_value) > tolerance + return r_value != py_value + + +def compare_cells( + r_df: pl.DataFrame, + py_df: pl.DataFrame, + key_cols: list[str], + order_group_cols: list[str] | None = None, + unit_swapped_columns: set[str] | None = None, + tracker_year_col: str | None = None, + static_join_missed_ids: set[str] | None = None, +) -> list[CellMismatch]: + """Diff matched rows cell-by-cell. + + ``static_join_missed_ids`` (ticket 63) is this file's set of normalized + patient IDs whose month rows spell the ID differently from the Patient + List, derived from the raw output by ``static_join_missed_ids``. Supplied + on the cleaned stage only, where normalization has erased the spelling the + raw stage discriminates on. + + ``unit_swapped_columns`` (ticket 44) is this file's set of mg/dL-labelled + glucose columns that ``resolve_glucose_units`` found to be recorded in + mmol/L, taken from the run's own error records. Both halves of an affected + pair are flagged, since the correction moves values between them. + + ``order_group_cols`` (ticket 21) opts into an extra diagnostic: for a + positional row-alignment key (``add_row_ordinal``'s ``__row_ordinal``, + not a real identity key), a within-group sort-order divergence between R + and Python -- e.g. R falling back to raw input order when + ``product_entry_date`` fails to parse, while Python sorts chronologically + -- produces a cascade of cell mismatches that are really the *same* + value landing on a different row, not a content difference. Per column, + each mismatched cell is checked against a multiset of the Python side's + own values across its whole ``order_group_cols`` group (not just + adjacent rows, since a shift can be more than one position); a hit sets + ``row_order_candidate`` for the classifier registry to act on. Omit + ``order_group_cols`` for a genuine identity key (e.g. patient's + ``patient_id`` + ``sheet_name``), where this check doesn't apply. + """ + joined = r_df.join(py_df, on=key_cols, how="inner", suffix="_py") + value_cols = [c for c in r_df.columns if c not in key_cols and c in py_df.columns] + swapped_pair_columns = _swapped_pair_columns(unit_swapped_columns) + mg_by_mmol = { + mmol: mg for mg, mmol in GLUCOSE_COLUMN_PAIRS if mmol in value_cols and mg in value_cols + } + + group_value_counts: dict[tuple[Any, ...], dict[str, Counter[Any]]] = {} + if order_group_cols: + for row in py_df.iter_rows(named=True): + gkey = tuple(row[c] for c in order_group_cols) + per_column = group_value_counts.setdefault(gkey, {}) + for col in value_cols: + per_column.setdefault(col, Counter())[row[col]] += 1 + + # Measured on r_df rather than on the joined frame: a column R lost is a + # fact about R's whole output for this file, not about the rows that + # happened to pair. + r_empty_columns = {c for c in value_cols if r_df[c].null_count() == len(r_df)} + + group_endpoint_agrees = _group_endpoint_agreement( + joined, order_group_cols, key_cols, value_cols + ) + + # A value column is suffixed on the Python side by the join; a key column + # is shared, so it is not. + year_field = None + if tracker_year_col: + for candidate in (f"{tracker_year_col}_py", tracker_year_col): + if candidate in joined.columns: + year_field = candidate + break + + mismatches = [] + for row in joined.iter_rows(named=True): + key = {k: row[k] for k in key_cols} + row_static_join_missed = ( + static_join_missed_ids is not None and _key_patient_id(key) in static_join_missed_ids + ) + dob_recovered = "dob" in value_cols and row["dob"] is None and row["dob_py"] is not None + tracker_year = _as_year(row[year_field]) if year_field else None + gkey = tuple(row[c] for c in order_group_cols) if order_group_cols else None + has_bare_year = any( + _is_python_reads_bare_year( + CellMismatch(key=key, column=c, r_value=row[c], py_value=row[f"{c}_py"]) + ) + for c in value_cols + ) + for col in value_cols: + r_value, py_value = row[col], row[f"{col}_py"] + if _values_differ(r_value, py_value): + row_order_candidate = False + era_shift_row_order_candidate = False + if gkey is not None: + counts = group_value_counts.get(gkey, {}).get(col) + row_order_candidate = bool(counts) and counts[r_value] > 0 + shifted = _shift_from_buddhist_era(r_value) + era_shift_row_order_candidate = ( + bool(counts) and shifted is not None and counts[shifted] > 0 + ) + mismatches.append( + CellMismatch( + key=key, + column=col, + r_value=r_value, + py_value=py_value, + row_order_candidate=row_order_candidate, + era_shift_row_order_candidate=era_shift_row_order_candidate, + group_endpoint_matches=group_endpoint_agrees.get((gkey, col), False), + row_has_bare_year_date=has_bare_year, + column_unit_swapped=col in swapped_pair_columns, + static_join_missed_id=row_static_join_missed, + r_column_empty_in_file=col in r_empty_columns, + row_dob_recovered=dob_recovered, + mg_sibling=( + (row[mg_col], row[f"{mg_col}_py"]) + if (mg_col := mg_by_mmol.get(col)) is not None + else None + ), + tracker_year=tracker_year, + ) + ) + return mismatches + + +def _as_year(value: Any) -> int | None: + """The tracker year column is Float64 on one side and Int32 on the other.""" + try: + return int(value) + except TypeError, ValueError: + return None + + +def _swapped_pair_columns(unit_swapped_columns: set[str] | None) -> set[str]: + """Both halves of every swapped glucose pair. + + ``resolve_glucose_units`` records the swap against the mg/dL column it read + as mmol/L, but the correction empties that column into its mmol sibling, so + a cell on either side of the pair is affected by the same file-level fact. + """ + if not unit_swapped_columns: + return set() + swapped = set(unit_swapped_columns) + for mg_col, mmol_col in GLUCOSE_COLUMN_PAIRS: + if mg_col in unit_swapped_columns: + swapped.add(mmol_col) + return swapped + + +GLUCOSE_UNIT_SWAP_ERROR_CODE = "glucose_unit_swapped" + + +def load_glucose_unit_swaps(errors: pl.DataFrame) -> dict[str, set[str]]: + """Per tracker, which glucose columns the pipeline read as mmol/L. + + Derived from the run's own error table rather than restated in the + comparison: ``resolve_glucose_units`` already writes one + ``glucose_unit_swapped`` row per affected file-column, so a threshold + change in the pipeline reaches the comparison without a second edit. + """ + swapped = errors.filter(pl.col("error_code") == GLUCOSE_UNIT_SWAP_ERROR_CODE) + by_tracker: dict[str, set[str]] = {} + for row in swapped.iter_rows(named=True): + by_tracker.setdefault(row["file_name"], set()).add(row["column"]) + return by_tracker + + +def _key_patient_id(key: Any) -> str | None: + """The patient ID out of a row key, whatever add_row_ordinal prefixed it with. + + ``add_row_ordinal`` renames every key column to ``__key_``, so a bare + ``patient_id`` lookup silently finds nothing -- the same trap ``_sheet_year`` + documents. Matching on the suffix survives both spellings. + """ + if not isinstance(key, dict): + return None + return next( + (v for k, v in key.items() if str(k).endswith("patient_id") and isinstance(v, str)), + None, + ) + + +def static_join_missed_ids(raw_df: pl.DataFrame) -> set[str]: + """Which identities in one raw frame were spelled two ways in their workbook. + + Returns the *normalized* spelling, because that is what the cleaned stage's + row key holds -- the raw spelling exists only here (ticket 63). A patient + appears when its month rows carry a hyphen or a transfer-clinic suffix the + Patient List does not, which is exactly the condition under which R's + raw-keyed join comes back null for every static column at once. + + Read off the raw output rather than restated as a list of affected + trackers: the set is derived from the same run being compared, so a + workbook whose spelling is later corrected drops out without a second edit. + """ + # An all-null patient_id column comes back as pl.Null, where the string + # expressions normalization is built from do not apply at all. + if "patient_id" not in raw_df.columns or not len(raw_df): + return set() + if raw_df.schema["patient_id"] != pl.String: + return set() + respelled = raw_df.select( + pl.col("patient_id"), + normalize_patient_id_expr(pl.col("patient_id")).alias("__normalized"), + ).filter( + pl.col("patient_id").is_not_null() + & pl.col("__normalized").is_not_null() + & (pl.col("patient_id") != pl.col("__normalized")) + ) + return set(respelled["__normalized"].to_list()) + + +def _group_endpoint_agreement( + joined: pl.DataFrame, + order_group_cols: list[str] | None, + key_cols: list[str], + value_cols: list[str], +) -> dict[tuple[Any, str], bool]: + """Per (group, column): does the group's last positional row agree? + + Only meaningful for ``add_row_ordinal``'s positional key, so it is keyed + off ``ROW_ORDINAL_COL`` being present rather than off a new parameter -- + a genuine identity key has no "last row" to speak of. + """ + if not order_group_cols or ROW_ORDINAL_COL not in key_cols: + return {} + + last_ordinals: dict[tuple[Any, ...], int] = {} + for row in joined.iter_rows(named=True): + gkey = tuple(row[c] for c in order_group_cols) + ordinal = row[ROW_ORDINAL_COL] + if ordinal > last_ordinals.get(gkey, -1): + last_ordinals[gkey] = ordinal + + agreement: dict[tuple[Any, str], bool] = {} + for row in joined.iter_rows(named=True): + gkey = tuple(row[c] for c in order_group_cols) + if row[ROW_ORDINAL_COL] != last_ordinals[gkey]: + continue + for col in value_cols: + agreement[(gkey, col)] = not _values_differ(row[col], row[f"{col}_py"]) + return agreement + + +Classifier = Callable[[CellMismatch], bool] + + +def classify(mismatch: CellMismatch, registry: dict[str, Classifier]) -> str: + for cause_name, classifier in registry.items(): + if classifier(mismatch): + return cause_name + return "unclassified" + + +# Seeded from the four causes already identified in the parity-presentation PDF +# for Product_entry_date (docs/archive/Product pipeline parity presentation.pdf). +# These are heuristics on the parsed (r_value, py_value) pair alone -- refining +# them, or adding new named causes, against real flagged rows is ticket 15's job. +CE_TYPO_YEAR_THRESHOLD = 2100 + +# Mirrors the product pipeline's own BUDDHIST_ERA_OFFSET / YEAR_FLOOR_DELTA, so +# a date the pipeline treats as this tracker's Buddhist-era year is not then +# called absurd here. +BUDDHIST_ERA_OFFSET = 543 +YEAR_FLOOR_DELTA = 5 + +# Excel's 1900 date system counts a 29 February 1900 that never existed, so +# serials below 61 resolve one day apart depending on whether the reader +# reproduces the bug (openpyxl) or does plain epoch arithmetic. +EXCEL_LEAP_BUG_YEAR = 1900 +EXCEL_LEAP_BUG_CUTOFF = datetime.date(1900, 3, 1) + + +def _is_python_sentinel_r_extraction_gap(m: CellMismatch) -> bool: + """Python declined the cell and R never read it. + + Verified against source (ticket 32): 25 of the 78 current rows are + Sarawak's 2023 tracker, whose ``Dec23`` sheet genuinely carries December + *2024* entry dates. Python's beyond-tracker-year guard + (``_validate_entry_dates``) stamps the sentinel; R's cleaned side is null + for the same reason ``r_value_missing`` describes. Both halves are + already-decided mechanisms -- this names their intersection, which the + earlier ``sentinel_null`` described only by its shape. + """ + return m.r_value is None and m.py_value == SENTINEL_DATE + + +def _is_python_out_of_window_date_preserved(m: CellMismatch) -> bool: + """Python published a date from outside the tracker's own window. + + ``_validate_entry_dates``'s year-floor branch logs these and deliberately + keeps the parsed date, so the downstream sort matches R's. That is a + decision the pipeline already made and reports; what it must not do is + ride along inside ``r_value_missing``, which asserts the opposite -- that + Python read the cell correctly and R lost it. Ticket 32 found 19 such rows + there, including ``0202-06-20`` (Surat Thani) and seven ``2009-12-04`` in a + 2019 Mahosot tracker. Each is a corrupt source cell, so the workbook is the + fix (ticket 40), not the classifier. + """ + return ( + m.r_value is None + and isinstance(m.py_value, datetime.date) + and m.py_value != SENTINEL_DATE + and m.tracker_year is not None + and not (m.tracker_year - YEAR_FLOOR_DELTA <= m.py_value.year <= m.tracker_year) + ) + + +def _is_r_value_missing(m: CellMismatch) -> bool: + """R produced null where Python has a real value. + + Named generically rather than "typo_rescue" (its original name): ticket 18 + found the R source has no forward-fill for this column and, separately, + that R sometimes fails to extract a perfectly clean source value (see + 2018_Mahosot Hospital, Jan18) -- i.e. most cases in this bucket aren't + typo-driven at all. classify() only sees the parsed (r_value, py_value) + pair, not the raw source cell, so it can't distinguish a genuine + source-typo rescue from a plain R extraction gap; both look identical + here and are lumped together deliberately. + + Bounded by ``_is_python_out_of_window_date_preserved`` and + ``_is_python_absurd_excel_serial``, which run first: this cause asserts + Python read the cell correctly, so it may not cover a cell whose Python + value is itself implausible (ticket 32). + """ + return m.r_value is None and m.py_value is not None and m.py_value != SENTINEL_DATE + + +def _is_python_absurd_excel_serial(m: CellMismatch) -> bool: + """Python turned a corrupt Excel serial into a date centuries away. + + Replaces ``ce_typo``, which tested ``r_value.year > 2100`` and so fired on + ``normalize_date_column``'s own 9999 sentinel while saying nothing about + the Python side. Both current rows are junk source serials -- 1,339,576 + published as ``5567-08-19`` (2024 Chiang Mai) and 411,384 as + ``3026-04-30`` (2026 Penang). The cleaned stage no longer publishes these + (``implausible_era_date``, ticket 32), so the cause remains for the raw + stage, where extraction has no validation. + + A genuine Buddhist-era date is *not* this: BE serials (~243,933) land on + the tracker's own BE year and are excluded by the band check. + """ + if not isinstance(m.py_value, datetime.date) or m.py_value == SENTINEL_DATE: + return False + if m.py_value.year < CE_TYPO_YEAR_THRESHOLD: + return False + # Patient stages carry no tracker_year column, so the band would never + # apply there and the cause would out-claim genuine Buddhist-era dates + # (ticket 60). _sheet_year reads the same year off the row's sheet name. + tracker_year = m.tracker_year if m.tracker_year is not None else _sheet_year(m) + if tracker_year is not None: + buddhist_year = tracker_year + BUDDHIST_ERA_OFFSET + if buddhist_year - YEAR_FLOOR_DELTA <= m.py_value.year <= buddhist_year: + return False + return True + + +# Wired ahead of every patient date column's own causes (ticket 60). Safe to +# run first because the band check above excludes a genuine Buddhist-era year, +# and a junk serial is not evidence about R's parse orders -- it was +# `r_parse_order_cannot_read_cell` that claimed the five 2025 Surat Thani cells +# whose R side is the serial 1141523. +PATIENT_ABSURD_SERIAL_CLASSIFIERS: dict[str, Classifier] = { + "python_absurd_excel_serial": _is_python_absurd_excel_serial, +} + + +def _is_excel_1900_leap_serial(m: CellMismatch) -> bool: + """One day apart because Excel believes 1900 was a leap year. + + A serial below 61 predates the phantom 1900-02-29: openpyxl reproduces + Excel's own reading (serial 25 -> 1900-01-25) while plain epoch arithmetic + from 1899-12-30, which ``normalize_date_column`` does via + ``parse_date_flexible``, gives 1900-01-24. Both sides are reading the same + source integer (ticket 32), which is stray end-of-block summary residue in + both current rows -- the cleaned stage nulls it (``summary_residue_nulled``). + """ + return ( + isinstance(m.r_value, datetime.date) + and isinstance(m.py_value, datetime.date) + and m.r_value.year == EXCEL_LEAP_BUG_YEAR + and m.py_value.year == EXCEL_LEAP_BUG_YEAR + and m.py_value < EXCEL_LEAP_BUG_CUTOFF + and (m.py_value - m.r_value).days == 1 + ) + + +def _is_python_future_date_sentinel(m: CellMismatch) -> bool: + """Python replaced an out-of-tracker-year date with R's own error sentinel. + + ``_validate_entry_dates`` (clean/product.py) and its patient counterpart + ``_validate_dates`` (clean/patient.py) log a parsed date whose year is + beyond the tracker's own calendar year and substitute + ``error_val_date`` (9999-09-09), because a future date is genuinely + ambiguous -- a premature next-month entry or a typo -- and the sentinel + preserves that signal. R validates nothing and carries the bad date + through. Verified against the real source data (ticket 36): for Preah + Kossamak's 2023 tracker, sheet Aug23, *both* pipelines independently read + 2029-08-29 out of a sheet whose every other row is August 2023, so the + source really does hold the typo -- Python flags it, R propagates it. + """ + return m.py_value == SENTINEL_DATE and isinstance(m.r_value, datetime.date) + + +# An entry-date cell parsing to 1900 is not a date the clinician wrote: it is +# a tiny integer (a stray day-of-month, e.g. "30") landing in the date column +# on an end-of-block summary row and being read as an Excel serial. +_SUMMARY_RESIDUE_YEAR = 1900 + + +def _is_summary_residue_nulled(m: CellMismatch) -> bool: + """Python nulled end-of-block summary-row residue that R kept as a date. + + ``PRODUCT_DATE_NA_MARKERS`` and the tiny-int scrub (clean/product.py) + null these before parsing, so the cleaned output is NULL rather than + 1900-01-30. Python is the correct side: 1900-01-30 is not a date any + tracker records. + """ + return ( + m.py_value is None + and isinstance(m.r_value, datetime.date) + and m.r_value.year <= _SUMMARY_RESIDUE_YEAR + ) + + +# Order matters: the two bounds on r_value_missing must run before it, or it +# absorbs the cells they exist to expose (ticket 32). +PRODUCT_ENTRY_DATE_CLASSIFIERS: dict[str, Classifier] = { + "python_sentinel_r_extraction_gap": _is_python_sentinel_r_extraction_gap, + "excel_1900_leap_serial": _is_excel_1900_leap_serial, + "python_absurd_excel_serial": _is_python_absurd_excel_serial, + "python_out_of_window_date_preserved": _is_python_out_of_window_date_preserved, + "r_value_missing": _is_r_value_missing, + "python_future_date_sentinel": _is_python_future_date_sentinel, + "summary_residue_nulled": _is_summary_residue_nulled, +} + +# The patient arm's cleaned stage runs the same future-date guard, so its date +# columns need the same cause without the product-specific siblings above. +PATIENT_FUTURE_DATE_CLASSIFIERS: dict[str, Classifier] = { + "python_future_date_sentinel": _is_python_future_date_sentinel, +} + + +def _is_r_category_lookup_miss(m: CellMismatch) -> bool: + """R's category join missed a product Python resolved. + + ``add_product_categories`` (read_product_data.R:481) is a bare + ``dplyr::left_join`` on the raw ``product`` string; + ``load_product_reference_data`` lowercases only the *column names* of the + Stock_Summary sheet, never the values. Python's ``_add_product_categories`` + (clean/product.py) lowercases both sides before joining. + + Ticket 62 scanned the whole population (866 rows, 12 files, 29 distinct + product names) and found **two** mechanisms, not the one this docstring + used to name. Python is the correct side in both; R loses a category the + reference data can supply. + + - **CRLF vs LF, 652 rows over 26 names.** Not a property of R's join at + all -- the two sides never see the same string. The reference workbook + stores embedded line breaks as a bare ``\\n`` (executed: its + Stock_Summary sheet XML holds 29 LF bytes and **zero** CR bytes), while + the trackers store ``\\r\\n`` (2021 Surat Thani's sharedStrings holds 80 + CRLF pairs). readxl returns each faithfully, so R compares + ``'Accu-Chek Performa Test Strips \\r\\n(50s/ bottle)'`` against a + reference entry spelled with ``\\n`` and misses; openpyxl normalises + CRLF to LF on read, so Python's strings match byte-for-byte. This is + the same reader difference ``normalize_whitespace_column`` was added + for in ticket 22, surfacing here as a lookup failure. + - **Case, 214 rows over 3 names.** The documented mechanism, and the only + one it covers: ``'NovoFine Needles 4mm x 32G (singles)'`` against the + reference's ``'(Singles)'`` (98), ``'ACCU-CHEK Performa Glucometer Set'`` + against ``'Accu-Chek Performa Glucometer Set'`` (91), and the same + ``(singles)`` slip on NIPRO (25). + + Caveat worth carrying: Python matches the 26 CRLF names by luck, not by + design -- its join normalises case only, and it works solely because + openpyxl happens to fold the line endings. A reference entry typed with + CRLF against a tracker spelled with LF would miss on Python too. Nothing + in the current data triggers that, so it is recorded rather than fixed. + """ + return m.r_value is None and m.py_value is not None + + +PRODUCT_CATEGORY_CLASSIFIERS: dict[str, Classifier] = { + "r_category_lookup_miss": _is_r_category_lookup_miss, +} + + +def _is_row_order_divergence(m: CellMismatch) -> bool: + """R's value for this cell shows up elsewhere in Python's own group. + + Root-caused (ticket 21) to R's per-(clinic, sheet) sort falling back to + raw input order whenever ``product_entry_date`` fails to parse for a + row -- itself the same R date-extraction gap ticket 18 already + root-caused for the date column directly (see ``r_value_missing``). + Python parses the date and sorts chronologically instead, so both sides + land on the same total (e.g. same end-of-sheet balance) via a different + per-row order -- not a genuine content divergence, and not something to + "fix" toward R's order, since Python's is the one verified against the + real source Excel dates. + + **Verdict (ticket 60): Python's chronological sort is the correct side.** + The mechanism was established by ticket 21 and the evidence by ticket 25 -- + per-column multiset equality across all 2,283 ``(clinic_id, + product_sheet_name)`` groups for ``product_units_released``, and row-identity + preservation across all 11,649 product groups -- but the verdict itself was + never written down, only implied. Stating it plainly: both sides implement + the same documented rank algorithm, the difference is entirely in which + rows have a parsable ``product_entry_date`` to sort by, and Python parses + the ones R does not. Nothing is added or lost on either side; only the + order differs. + + Caveat on the flag, not on the verdict: ``row_order_candidate`` is a loose + value-membership test -- it asks whether R's value appears anywhere in + Python's own group -- so it false-fires on repeated small numeric values, + where a coincidental match means nothing. It is evidence of ordering, not + proof of it, and ``product_balance``'s cumulative running total cannot be + detected this way at all (already explained, and homed on ticket 36). + """ + return m.row_order_candidate + + +PRODUCT_ROW_ORDER_CLASSIFIERS: dict[str, Classifier] = { + "row_order_divergence": _is_row_order_divergence, +} + + +def _is_derived_running_total_row_order(m: CellMismatch) -> bool: + """Two ledgers, different accumulation order, same closing figure. + + ``product_balance`` is recomputed per row on both sides + (``_compute_running_balance``, step 2.15, mirroring R's + ``compute_balance``), so under the sort-order divergence ticket 21 + root-caused it cannot travel with its row -- Python's intermediate + balances are a chronological ledger, R's a data-entry-order one, and + neither side's intermediate values appear in the other's group. Value + membership therefore cannot detect it in principle; the group's *closing* + balance can, being order-independent. + + Measured across the real 248-tracker pair (ticket 36): 2,729 of the 2,740 + mismatching balance cells sit in groups whose closing balance agrees + exactly (2,281 of 2,283 groups). The 11 that do not are deliberately left + unclassified -- see the ticket: both are 2019 Sultanah Bahiyah groups + where R's ledger is corrupted by an Excel date serial leaking out of + "Units Received", so a disagreeing endpoint is a real signal worth + surfacing rather than a label to absorb it. + """ + return m.group_endpoint_matches + + +DERIVED_RUNNING_TOTAL_CLASSIFIERS: dict[str, Classifier] = { + "derived_running_total_row_order": _is_derived_running_total_row_order, +} + + +# Product rows are aligned by ordinal position within (clinic_id, +# product_sheet_name), so a within-group re-sort can move any column's value +# to a different row -- except these, which hold the same value for every row +# of a group and therefore cannot diverge by ordering alone. Everything else +# in the product schema must carry PRODUCT_ROW_ORDER_CLASSIFIERS; ticket 36's +# wiring test derives that list from this exclusion rather than restating it. +GROUP_INVARIANT_PRODUCT_COLUMNS = frozenset( + { + "clinic_id", # alignment group key + "product_sheet_name", # alignment group key + "file_name", # one value per compared file + "product_table_month", # derived from the sheet, so constant per group + "product_table_year", + } +) + + +def _is_r_extraction_gap(m: CellMismatch) -> bool: + """R produced null where Python has a real value. + + Three source-verified instances share this shape, in three different + tracker generations. classify() sees only the (r_value, py_value) pair, + not the file, so they carry one cause name; the mechanisms are: + + - ``recruitment_date`` (ticket 28): R's static "Patient List" + recruitment-date extraction fails to populate it for the large majority + of patients even where the tracker plainly records one (e.g. Quirino + Memorial Medical Center, patient PH_QM001, "Date of Recruitment" = + 2025-12-01 in the source). + - ``edu_occ_updated`` on 2022 trackers (ticket 37): R's own + "Updated 2022" header fixup (script1_helper_read_patient_data.R) + rewrites that cell to "Level of Education Or Occupation", but the + source cell reads " Updated \\n2022" with a *leading* space, which + survives the rewrite -- so R's merged header is " Level of Education Or + Occupation Date" and R's name sanitizer turns it into the junk column + ``xlevelofeducationoroccupationdate``, which no synonym matches. The + same fixup's blood-pressure branch has no leading space and works. + Verified against the real source Excel (2022 Kantha Bopha, Patient + List!M13/N13: "Law School - Year 2" updated 2022-11-04) and against R's + own raw parquet, which carries the junk column. + - ``blood_pressure_updated`` and ``edu_occ_updated`` on 2026 trackers + (ticket 37): the 2026 template moved the complication-screening block + to a new "Annual" sheet; R reads no values from it at all (0 non-null + for the whole file), while Python extracts them. Verified against the + real source Excel (06 YGH T1D Tracker_June_26, Annual!G/H/I for + MM_YG101: 2026-02-05, 100, 60). + + Ticket 50 adds three more, each verified against the source workbook and + against R's own raw parquet, which carries the column R could not name: + + - ``hospitalisation_date``/``hospitalisation_cause`` on 2022 Kantha Bopha: + the upper header cell (X71) opens with thirteen spaces, so R's merged + header starts with a space and its name sanitizer emits + ``xcurrentmonthhospitalisationdkahypootherdropdown``/``...date``, which no + synonym matches -- the same leading-space mechanism as ``edu_occ_updated`` + above. The source plainly records "DKA" and a date (Sept'22!X92/Y92 for + KH_KB020, Nov'22 and Dec'22 for KH_KB113). + - ``fbg_updated_date`` on 2019 Preah Kossamak: the Jul19 and Aug19 sheets + lost the header merges every other month sheet still has (only B63:B64 + survives), so the "Updated FBG" title no longer spans its "Date" + sub-header. R pastes the two header rows per column and is left with a + bare ``date`` column it cannot map; Python forward-fills the upper header + row and recovers the qualified name. Verified: Jul19!O66 = 2019-06-06 for + KH_PK002, exactly Python's value, and R has 0 non-null for both sheets + against Python's 13 and 12. + - ``clinic_visit`` on 2025 LWCH: the column holding the current-month visit + flag (D) has no header in *either* header row, so R drops it; Python + recovers the name from the sibling month sheets that do label it (ticket + 30's ``recover_blank_headers``). Verified: Oct25!D90 = "Y" for MY_LW014. + + Ticket 60 scanned the whole population rather than the columns already + named, and found it spans thirteen columns, not the seven above. Three + further mechanisms, each traced to the source workbook, each with Python + the correct side: + + - ``insulin_regimen`` on 2021 Kantha Bopha (194 rows): the ``Mar21`` and + ``Apr21`` sheets carry the patient header at row 85 with ``Q85``/``Q86`` + simply *empty*, where every other month sheet has ``Q13 = "Insulin + Regime"``. R publishes 0 non-null for those two sheets and 97 for each of + the others; the column still holds data (``Mar21!Q87 = "Self-mixed BD"`` + for KH_KB001), and Python recovers the name from the sibling sheets via + ``recover_blank_headers``. The same blank-header mechanism as + ``clinic_visit`` above, a second instance rather than a new kind. + - **A blank row number on the 2026 ``Annual`` sheet** (7 rows): 2026 ISDFI + rows 32-34 (PH_IS022-024) and 2026 Khon Kaen row 27 (TH_KN016) are + exactly the ``Annual`` rows whose column A is empty while every other row + is numbered. R drops them from the Annual join and publishes null across + ``edu_occ``, ``blood_pressure_sys_mmhg``, ``blood_pressure_dias_mmhg``, + ``other_issues`` and ``status``; the cells hold real values + (``Annual!E32 = "college graduate"``, ``H32``/``I32`` = 80/50). Note this + corrects the reason recorded against those same ISDFI cells further up + the registry, which attributes them to R reading nothing from the Annual + sheet: R reads that sheet for this file perfectly well (102 of 120 + ``edu_occ``), so the sheet is not the mechanism, the missing row number + is. + - ``edu_occ`` on 2026 Nakornping (42 rows, R 0 non-null for the whole + file): its ``Annual!E9`` reads ``"Level of Education\\nOr Occupation/ + อาชีพหรือชั้นเรียน"`` where ISDFI's and Khon Kaen's read ``"Level of + Education\\nOr Occupation"``. R's Unicode-aware sanitizer keeps the Thai + in the column name and no synonym matches; Python's ASCII folding drops + it. This is ``r_non_latin_header_miss``'s mechanism (ticket 49) being + absorbed here on registry order -- worth knowing, not a defect. + + Ticket 62 measured the two populations ticket 60 left, which are 98% of + the cause. Python is correct in both; the mechanism is sharper than the + entry above states. + + - ``recruitment_date`` (28,009 rows, 94 files). The claim above -- R's + extraction "fails to populate it for the large majority of patients" -- + is **false**: R populates 53,331 of its 82,771 raw rows (64%). The + divergence is perfectly file-level and all-or-nothing. Measured over the + 239 comparable files: 91 where R reads **zero** recruitment dates and + Python reads them, 148 with no divergence at all, and **zero** files + where R reads some and misses others. The cause is a header cell whose + text ends in a space, which forces Excel to write + ```` (executed: 2023 CDA ``Patient List!L8`` is + literally ``Date + of Recruitment ``). R reads its headers with + ``openxlsx::read.xlsx`` (script1_helper_read_patient_data.R:15), whose + inline-string parsing folds that attribute into the header text, so + ``make.names`` emits the column ``xmlspacepreservedateofrecruitmentmmmyy`` + and no synonym matches it. The correlation is exact: all 91 affected + files carry that column in R's own raw parquet, and none of the other + 148 do. openpyxl parses the attribute correctly and reads + ``'Date of Recruitment '``. The same defect eats at least one other + column R never maps + (``xmlspacepreserveinsurancestatusnanssfeqeqpending``). + - ``edu_occ_updated`` (2,770 rows, 16 files). Also whole-file rather than + per-patient (13 files where R reads zero), but unlike recruitment_date + it is not one mechanism: R's raw output carries three distinct junk + names for this column across the corpus -- + ``xlevelofeducationoroccupationdate`` (the leading-space fixup + documented above), ``levelofeducationoroccupationอาชพหรอชนเรยน`` and + ``...dateupdatedddmmmyyyy`` (ticket 60's Thai-header mechanism). Fifteen + files carry a junk name *and* still resolve the column, so the junk name + alone is not sufficient to predict the loss here the way it is for + recruitment_date. + + A genuine R limitation in every case, not a Python defect. + """ + return m.r_value is None and m.py_value is not None + + +PATIENT_R_EXTRACTION_GAP_CLASSIFIERS: dict[str, Classifier] = { + "r_extraction_gap": _is_r_extraction_gap, +} + + +def _is_r_static_join_misses_respelled_id(m: CellMismatch) -> bool: + """R attached the Patient List on the raw ID, so a respelled month row missed it. + + Both pipelines join the whole-tracker sheets (``Patient List``, ``Annual``) + onto the month rows during *extraction*, long before either fixes the ID. + Where a month sheet spells a patient differently from the Patient List in + the same workbook -- 2023/2024 Mahosot write ``LA-MH056`` against a Patient + List entry of ``LA_MH056``, 2024 CDA ``KH-CD016``, 2026 Surat Thani + ``TH-ST029`` -- the raw key cannot match, and every static column comes out + null. Ticket 58 made Python key on the normalized ID; R still does not, so + Python now carries values R has nothing for. + + Verified against the source workbooks: 2024 Mahosot's ``Patient List`` + holds a full row for ``LA_MH056`` (Vientiane Capital, M, D.O.B. + 2014-02-22, recruited 2021-06-22), and R's own cleaned output publishes + null for all four of that patient's twelve month rows. Python is the + correct side; the workbook's inconsistent spelling is a source defect + reported separately. + + **Two discriminators, one per stage** (ticket 63). At the raw stage the + month row's own ID still carries the hyphen or transfer-clinic suffix, so + the cell speaks for itself. Cleaning normalizes exactly that, on both + sides, so at the cleaned stage the spelling is gone and the cause would + otherwise fall through to ``r_extraction_gap``'s column-naming failures -- + the wrong reason over the right label. ``static_join_missed_id`` supplies + the missing fact instead: the per-file set of affected identities, derived + from the run's own raw output by ``static_join_missed_ids``. + + R-null and Python-present still gate both paths. The flag is a property of + the *patient*, not of the cell, so a flagged row whose column R populated + from the month sheet is not this cause. + + The cleaned path additionally declines a column R lost across the whole + file, because there the join miss is incidental: 2024 Preah Kossamak reads + **zero** ``recruitment_date`` in 863 rows and 2024 Yangon General zero in + 1,057, which is ``r_extraction_gap``'s file-level ``xml:space`` header + defect, while 2024 Mahosot reads 672 of 1,068 and so its nulls really are + per-patient. Without the guard the prepended cause takes 2,336 cells whose + reason is R's header, not R's key -- the ticket-32 failure mode of a right + label over a wrong mechanism. Only the cleaned stage needs it: at the raw + stage a column R never mapped is absent entirely and is reported as column + divergence, so no cell reaches here. + + **Its one measured over-claim is 5 cells, and it is not fixable at this + stage.** The flag is a property of the patient, so where a workbook spells + the *same* patient both ways in month rows -- 2026 Surat Thani carries both + ``TH_ST029`` and ``TH-ST029`` in ``May26`` -- normalization folds the two + rows together and neither can be told from the other. The raw stage labels + the canonical row ``r_extraction_gap`` instead, and that is the residue: + one row, five columns. Named rather than bounded out, because excluding the + identity would cost two correct classifications to avoid one wrong one. + """ + if m.r_value is not None or m.py_value is None: + return False + if m.static_join_missed_id: + return not m.r_column_empty_in_file + patient_id = _key_patient_id(m.key) + if patient_id is None: + return False + normalized = ( + pl.DataFrame({"patient_id": [patient_id]}) + .select(normalize_patient_id_expr(pl.col("patient_id"))) + .item() + ) + return normalized != patient_id + + +PATIENT_STATIC_JOIN_CLASSIFIERS: dict[str, Classifier] = { + "r_static_join_misses_respelled_id": _is_r_static_join_misses_respelled_id, +} + + +def _is_r_age_not_derived_without_dob(m: CellMismatch) -> bool: + """R keeps the month sheet's typed age, having no D.O.B. to recompute from. + + ``age`` is not published as the workbook wrote it: ``_fix_age_from_dob`` + (clean/patient.py) overrides the sheet's Age cell with the age derived from + ``dob`` whenever a birth date exists, and records an ``invalid_value`` + error saying so. Ticket 58's join fix gave Python a D.O.B. for patients + whose month rows are spelled differently from the Patient List; R's join + still misses, so R has no birth date and falls back to whatever the Age + cell said, or to nothing. + + Executed against the frozen baseline: R's cleaned 2023 Mahosot holds + ``dob`` null for **all 275 rows** of the affected patients, and publishes + LA_MH060 as 13 in ``Jan23`` where Python publishes 14 -- that patient's + recovered D.O.B. is 2009-01-16, so on any January 2023 sheet they are 14 + and R's figure is the clinic's own, updated a month late. Python is the + correct side; the divergence is a cascade of the recovered birth date, not + a disagreement about the patient. + + Keyed on ``row_dob_recovered`` rather than on the file-level identity set, + because it is the birth date the derivation actually consumes -- a row + whose D.O.B. R has too is not this cause however its ID is spelled. Both + directions of the population are covered: 53 cells where R publishes a + typed age that differs, and 60 where the Age cell is empty and R, unlike + Python, has nothing to derive from. + """ + if m.py_value is None: + return False + return m.row_dob_recovered + + +PATIENT_AGE_CASCADE_CLASSIFIERS: dict[str, Classifier] = { + "r_age_not_derived_without_dob": _is_r_age_not_derived_without_dob, +} + + +def _is_r_duplicate_header_selection_dropped(m: CellMismatch) -> bool: + """R keeps only the first selection of a multi-select block; Python keeps all. + + Where a merged screening header spans several "(Select)" sub-columns that + all map to one canonical name, R's ``make.names(unique = TRUE)`` suffixes + the duplicates and its mapping then keeps only the first -- verified in R's + own raw parquet for 2021 NPH, which carries the second selection in + ``complicationscreeningselect1`` rather than in ``complication_screening``. + Python unites the whole group (ticket 31), so it carries both. The source + records both (Dec21!AD84 "Dilated Eye Examination", AE84 "Foot Examination + (Nerves)" for KH_NP006), so Python is the correct side and R is losing a + recorded screening. + + Fires only where R's value is exactly Python's own first selection, so a + group whose *kept* selection disagrees stays unclassified. + """ + if m.r_value is None or m.py_value is None: + return False + py_parts = str(m.py_value).split(",") + return len(py_parts) > 1 and py_parts[0].strip() == str(m.r_value).strip() + + +PATIENT_SCREENING_SELECTION_CLASSIFIERS: dict[str, Classifier] = { + "r_duplicate_header_selection_dropped": _is_r_duplicate_header_selection_dropped, +} + + +def _is_r_non_latin_header_miss(m: CellMismatch) -> bool: + """R's header sanitizer keeps non-Latin script, so its exact match fails. + + ``harmonize_patient_data_columns`` (script1_helper_read_patient_data.R) + resolves a header by ``match()`` -- exact equality against the sanitized + synonym list -- after ``sanitize_str`` strips ``[^[:alnum:]]``. R's + character class is Unicode-aware, so a clinic that appends a local-language + translation to the English header keeps it: "Last Clinic \\nVisit\\n + " sanitizes to ``lastclinicvisit``, which equals no synonym, so + R drops the column and leaves every row null. Python's ``sanitize_str`` + (src/a4d/reference/synonyms.py) strips to ``[^a-z0-9]`` instead, reducing + the same header to ``lastclinicvisit`` and matching. + + Executed both implementations on the real header to confirm the divergence + rather than reading the two regexes. Verified against the real source Excel + (2022 Mukdahan, Mar22!F44/G44 carry Thai suffixes; F46 = 2022-02-17, which + is Python's 44609): Python recovers 44 real dates R discards. A genuine R + limitation, not a Python defect -- and the reason Python's own docstring + claim that its sanitizer "matches the R implementation" is not exactly + true here. + """ + return m.r_value is None and m.py_value is not None + + +PATIENT_NON_LATIN_HEADER_CLASSIFIERS: dict[str, Classifier] = { + "r_non_latin_header_miss": _is_r_non_latin_header_miss, +} + + +def _is_r_insulin_dedup_drop(m: CellMismatch) -> bool: + """R deletes the whole "TOTAL Insulin Units" column before it is ever read. + + ``extract_patient_data`` (r-archive/R/script1_helper_read_patient_data.R) + carries a hack for a merged-cell artefact that can produce two "insulin + regimen" columns: it greps the source headers for the literal ``Insulin`` + and drops every match after the first. Since the 2024 tracker redesign the + monthly sheet also carries "TOTAL Insulin Units per day", so that grep + matches two real, unrelated columns and the hack deletes the second. The + sibling insulin columns survive only because their sub-headers read + "Pre-mixed"/"Short-acting" (the "Human Insulin" group header sits in a row + R does not merge here) and "Number of insulin injections" has a lowercase + "i" the case-sensitive grep misses. + + Verified against the real source Excel (ticket 29, 06 500 NPT Children's + Hospital, Jan26!AA94-98: 20, 48, 24, 30, 20 -- exactly Python's values): + the column is absent from R's raw output entirely, so R's cleaned-stage + template leaves it null on all 81,859 rows. Python is not diverging, it is + the only side that reads the column at all. + """ + return m.r_value is None and m.py_value is not None + + +PATIENT_INSULIN_TOTAL_UNITS_CLASSIFIERS: dict[str, Classifier] = { + "r_insulin_dedup_drop": _is_r_insulin_dedup_drop, +} + + +def _is_r_join_suffix_collision(m: CellMismatch) -> bool: + """R's Patient List join renames *both* sides on a name collision. + + ``reading_patient_data`` (r-archive/R/script1_read_patient_data.R) joins + the monthly sheets to the static "Patient List" sheet with + ``suffix = c(".monthly", ".static")``. Where a column exists on both + sheets, dplyr renames both, so the unsuffixed name disappears -- and R's + cleaning stage, which reads ``fbg_baseline_mg``, then finds nothing and + leaves the schema column null for the whole file. R avoids this for + ``hba1c_baseline`` by explicitly dropping the monthly copy before the + join; it never does the same for the baseline FBG columns. + + Polars suffixes only the right-hand frame, so Python keeps the monthly + value under the base name and the Patient List copy as ``.static``. + Measured against the real drive data (ticket 29): 21 files collide on + ``fbg_baseline_mg`` and 3 on ``fbg_baseline_mmol``, and across their + ~10,000 rows the two copies are identical or numerically equal in 92.4%, + so the column Python keeps is very nearly the one R lost. The 663 rows + where the two sheets genuinely disagree are the source contradicting + itself, not a pipeline choice either side can resolve. + """ + return m.r_value is None and m.py_value is not None + + +PATIENT_JOIN_SUFFIX_COLLISION_CLASSIFIERS: dict[str, Classifier] = { + "r_join_suffix_collision": _is_r_join_suffix_collision, +} + + +def _is_r_numeric_error_sentinel(m: CellMismatch) -> bool: + """R stamps 999999 on a cell that never held a usable number; Python nulls it. + + R has no missing-value normalization before ``as.numeric()``, so a + clinician's "-", "NA" or free text ("2 months") fails to parse and R's + cleaning substitutes ``ERROR_VAL_NUMERIC`` (999999). + ``safe_convert_column`` (src/a4d/clean/converters.py) normalizes those + markers to null *before* conversion, deliberately: 999999 means "a value + was recorded but is invalid", which is the wrong claim for a blank, and a + magic number silently poisons every downstream mean. + + Checked exhaustively, not sampled (ticket 29): all 8,085 rows carrying + this shape were traced back to Python's own raw stage, and **not one** had + a clean number behind it. 3,530 are missing-value markers, 2,331 are Excel + formula-error strings, 1,719 are unparseable free text; 505 could not be + joined back to a raw row. No data is lost on Python's side -- the + information R keeps is the fact of a failed parse, which Python records in + the error log instead of in the data. + """ + if m.py_value is not None or m.r_value is None: + return False + try: + # The sentinel reaches here as a float from a numeric column and as + # text from a string-typed one, so compare numerically either way. + return float(m.r_value) == settings.error_val_numeric # type: ignore[arg-type] + except TypeError, ValueError: + return False + + +R_NUMERIC_ERROR_SENTINEL_CLASSIFIERS: dict[str, Classifier] = { + "r_numeric_error_sentinel": _is_r_numeric_error_sentinel, +} + + +def _as_date(value: Any) -> datetime.date | None: + """The date behind a cell value, whether it arrives as a date or a datetime. + + A ``pl.Date`` column reads back as ``date`` and a ``pl.Datetime`` one as + ``datetime``; ``datetime`` is a subclass of ``date`` but never equal to it, + so a plain ``==`` against the sentinel silently misses half the cases. + """ + if isinstance(value, datetime.datetime): + return value.date() + return value if isinstance(value, datetime.date) else None + + +def _shift_from_buddhist_era(value: Any) -> datetime.date | None: + """``value`` read as a Buddhist-era date and shifted to Gregorian, or None. + + None whenever the shift is not meaningful: the value is not a date, its + year is plausible as Gregorian already, or the shifted day does not exist + (a Buddhist leap day, which the pipeline declines to convert for the same + reason). + """ + as_date = _as_date(value) + if as_date is None or as_date.year < PATIENT_BUDDHIST_ERA_THRESHOLD: + return None + try: + return as_date.replace(year=as_date.year - BUDDHIST_ERA_OFFSET) + except ValueError: + return None + + +def _is_r_date_error_sentinel(m: CellMismatch) -> bool: + """R stamps 9999-09-09 on a date cell that recorded an absence; Python nulls it. + + The date-column twin of ``r_numeric_error_sentinel``. R's ``parse_dates`` + (script2_helper_patient_data_fix.R) tests ``is.na(date)``, which is false + for the *string* "NA", so the value falls through to ``lubridate::as_date``, + fails, and ``convert_to`` substitutes ``ERROR_VAL_DATE``. + + Python is the correct side, and the source says so explicitly: the tracker + template's own sub-header for ``hospitalisation_date`` reads "(Insert Date + or NA)", so a cell holding "NA" is the form being filled in as designed, + not a date that failed to parse. Verified on the real 254-tracker set + (ticket 38): of the 1,607 cells carrying this shape, 1,565 were traced back + to a raw value both pipelines agree on -- literal "NA" -- with the + remainder unmatched to a raw row rather than contradicting it. Checked + directly in the source workbook for 2020 Mahosot (Jan20, column 24, header + "Hospitalisation due to diabetes emergency or glucose control / (Insert + Date or NA)"), where "NA" is what the clinic wrote on nearly every row. + + Ticket 38 also widened Python's own missing-marker set on the date path + (``MISSING_VALUE_MARKERS`` / ``DATE_ABSENCE_MARKERS``, clean/date_parser.py), + so cells recording absence as "-", "Nil", "Unknown" or the template's + leftover placeholder text now land here too rather than agreeing with R's + sentinel by accident. + """ + return m.py_value is None and m.r_value is not None and _as_date(m.r_value) == SENTINEL_DATE + + +R_DATE_ERROR_SENTINEL_CLASSIFIERS: dict[str, Classifier] = { + "r_date_error_sentinel": _is_r_date_error_sentinel, +} + + +def _declared_aliases() -> dict[str, set[str]]: + """Canonical label -> the sanitized retired spellings it absorbs. + + Read from ``reference_data/validation_rules.yaml`` rather than restated + here, so this classifier cannot drift from the config that produced the + values it is explaining. + """ + resolved: dict[str, set[str]] = {} + for spec in load_validation_rules().values(): + if not isinstance(spec, dict): + continue + for canonical, spellings in (spec.get("aliases") or {}).items(): + resolved.setdefault(canonical, set()).update(sanitize_str(s) for s in spellings) + return resolved + + +ALIAS_CANONICAL_LABELS = _declared_aliases() + + +def _is_python_canonical_label(m: CellMismatch) -> bool: + """R emits a retired spelling of a label; Python emits the canonical one. + + Where a tracker generation renamed a status, both spellings appear across + the corpus -- "Active - Remote" in 2020-2023 trackers, "Active Remote" in + 2024+ ones, which is the only spelling the Lookup List dropdown + introduced with the 2024 template defines (verified against the real + source workbooks, ticket 29). R's own config lists both as allowed values + and R's first-match lookup collapses everything onto the older spelling. + Python declares one canonical label per status and folds the retired + spellings into it, so the reports carry one label per status rather than + two competing ones. A deliberate divergence from the frozen R baseline, + not a defect on either side -- and the source cell is unchanged in both. + """ + if not isinstance(m.r_value, str) or not isinstance(m.py_value, str): + return False + return sanitize_str(m.r_value) in ALIAS_CANONICAL_LABELS.get(m.py_value, set()) + + +PYTHON_CANONICAL_LABEL_CLASSIFIERS: dict[str, Classifier] = { + "python_canonical_label": _is_python_canonical_label, +} + + +def _is_r_validator_rejects_multivalue(m: CellMismatch) -> bool: + """R's own allowed-values validator rejects the multi-insulin CSV output + R's own derivation logic produces for 2024+ trackers, replacing it with + the "Undefined" sentinel -- confirmed in code (src/a4d/clean/patient.py's + _derive_insulin_fields docstring, ticket 28): Python's derivation is a + deliberate, documented correction of an R typo and validator bug, not a + parity gap to close. + + Ticket 60 measured the population and found the name covers **two** R + defects, not one. 570 of the 15,647 rows have a Python value with no comma + in it at all -- 510 of them a bare ``Pre-mixed`` -- which the multi-value + story cannot explain, since R accepts ``Pre-mixed`` elsewhere (5,426 rows + of its own output). The second mechanism is **NA propagation through R's + builder**: ``script2_process_patient_data.R`` composes the field as + ``ifelse(human_insulin_pre_mixed == "Y", "pre-mixed", "")`` pasted with one + ``ifelse`` per sibling column, and ``NA == "Y"`` is ``NA``, which ``paste`` + stringifies as the literal ``"NA"``. The subsequent ``.x[.x != ""]`` filter + removes empties but not ``"NA"``, so a single tick beside unfilled siblings + still yields ``"pre-mixed,NA,NA,NA,NA"`` and is rejected. Measured across + the corpus: where R publishes ``Pre-mixed``, the sibling columns are + non-null in 4,134 of 4,135 rows; where R publishes ``Undefined`` on a + pre-mixed tick, 575 have at least one null sibling. R's output contains no + comma-joined value at all, and no ``Rapid-acting`` whatsoever -- the latter + being the ``"rapic-acting"`` typo on line 105 of the same builder. + + Python is the correct side in both: it reads the same raw ticks (verified + identical on both sides for 2024 NPT, MM_NC009, Apr24) and validates each + token independently. + """ + return m.r_value == "Undefined" and m.py_value is not None + + +def _is_r_validator_rejects_untrimmed(m: CellMismatch) -> bool: + """R's allowed-value validator rejected a value for its whitespace alone. + + readxl does not trim these patient cells, so a source value like "F " (a + real one -- Kantha Bopha 2019, patient KH_KB023, verified in the source + Excel) misses R's allowed-value list and lands on the "Undefined" + character sentinel. Python's cleaning strips string cells before + validation (ticket 36), so it keeps the value. Python is recovering data + both pipelines previously lost, not diverging. + """ + return m.r_value == "Undefined" and m.py_value is not None + + +PATIENT_UNTRIMMED_VALIDATION_CLASSIFIERS: dict[str, Classifier] = { + "r_validator_rejects_untrimmed": _is_r_validator_rejects_untrimmed, +} + + +def _is_r_unicode_sanitizer_rejects_accent(m: CellMismatch) -> bool: + """A misaccented spelling misses R's allowed-value list but not Python's. + + The two sanitizers were meant to be the same function and are not. R's + (r-archive/R/script2_sanitize_str.R) strips ``[^[:alnum:]]``, which under + ICU is Unicode-aware, so accented letters survive; Python's + (clean/validators.py) strips ``[^a-z0-9]`` and folds them away. Both sides + sanitize the value and the allowed list before matching, so the difference + only shows where a source spelling differs from its canonical form in the + accents alone. + + The whole of this cause on the real 254-tracker set is one Vietnamese + province across five VNCH trackers: the source writes ``Thái Nguyễn`` (a + tilde on the second e) where the allowed list has ``Thái Nguyên``. R + sanitizes those to ``tháinguyễn`` and ``tháinguyên``, misses, and stamps + the "Undefined" character sentinel; Python sanitizes both to ``thinguyn`` + and recovers the canonical name. Python is right -- the province is real + and the accent is a typo -- and this is the value-side twin of ticket 49's + ``r_non_latin_header_miss``, where the same ASCII folding was what let + Python match a header R could not. + + The folding cannot silently merge two *different* provinces: + ``validate_allowed_values`` raises on any two allowed values that sanitize + alike, and the 209-entry province list has no such pair (executed). Note + what this does *not* fix: the same trackers also write ``Thai Nguyen`` + with no accents at all, which sanitizes to ``thainguyen`` on both sides + and so is "Undefined" in both pipelines -- a shared limitation the + comparison cannot see, not a divergence. + + The test is the mechanism: R sentinelled, and Python's canonical value + carries characters that only Python's sanitizer removes. + """ + if m.r_value != "Undefined" or not isinstance(m.py_value, str): + return False + unicode_aware = m.py_value.lower().replace(" ", "") + return sanitize_str(m.py_value) != unicode_aware + + +PATIENT_UNICODE_SANITIZER_CLASSIFIERS: dict[str, Classifier] = { + "r_unicode_sanitizer_rejects_accent": _is_r_unicode_sanitizer_rejects_accent, +} + + +PATIENT_INSULIN_SUBTYPE_CLASSIFIERS: dict[str, Classifier] = { + "r_validator_rejects_multivalue": _is_r_validator_rejects_multivalue, +} + + +def _is_r_drops_drug_name_tick(m: CellMismatch) -> bool: + """The clinic ticks an insulin column by naming the drug, and only Python + reads it. + + The 2024+ template's five insulin columns are tick boxes taking ``Y`` or + ``-``. 2024 Sarawak General writes the drug instead -- ``Novorapid`` in + ``analog_insulin_rapid_acting`` and ``Glargine``, ``Toujeo`` or + ``Ryzodeg`` in ``analog_insulin_long_acting``, across 56 rows. Both + pipelines tested for ``Y`` exactly, so both discarded the subtype and + Python's empty derivation was then published as ``Undefined``. + + ``_insulin_ticked`` (clean/patient.py) now reads any value that is not a + negative marker as a tick, which recovers ``Rapid-acting,Long-acting`` for + those rows. The negative markers are the closed set the data actually + holds: across all 248 trackers these columns contain only ``Y``, ``-``, + ``0``, the four drug names, and null (verified by a distinct-value count + over the raw output). Python is the correct side -- the clinic did record + which insulins the patient takes -- and the workbook is a ticket 40 + finding, since free text in a tick box is what made both pipelines lose it. + + Keyed on R holding nothing while Python holds a real subtype list, which + is the only shape this recovery produces. + """ + return ( + m.r_value is None + and isinstance(m.py_value, str) + and m.py_value not in ("", settings.error_val_character) + ) + + +PATIENT_INSULIN_DRUG_NAME_CLASSIFIERS: dict[str, Classifier] = { + "r_drops_drug_name_tick": _is_r_drops_drug_name_tick, +} + + +def _is_r_ifelse_na_propagation(m: CellMismatch) -> bool: + """R's derivation returns NA because one input is NA, not because the + answer is unknown. + + ``insulin_type`` is derived on both sides from the five 2024+ insulin + columns. R (script2_process_patient_data.R:94) writes + ``ifelse(pre_mixed == "Y" | short_acting == "Y" | intermediate == "Y", + "human insulin", "analog insulin")``. Under R's three-valued logic + ``FALSE | FALSE | NA`` is NA, so a row whose human columns are blank -- + but whose *analog* columns plainly read "Y" -- yields NA and R loses the + type entirely. ``_derive_insulin_fields`` (clean/patient.py) instead + derives whenever any of the five columns is populated, so it answers + "analog insulin" for exactly those rows. + + Verified against the real 248-tracker pair (ticket 37, 06 Baguio General + Hospital_Jun_26, May26, PH_BG001 and PH_BG004): both pipelines hold the + *same* five input values (human blank/N, rapid-acting Y, long-acting Y), + R's insulin_type is NULL and Python's is "Analog Insulin". Python is the + correct side -- the information is in the row, R's NA propagation drops + it. Not the same cause as ``r_validator_rejects_multivalue`` on the + sibling ``insulin_subtype`` column, which is a validator rejection of a + value R did produce. + """ + return m.r_value is None and m.py_value is not None + + +PATIENT_INSULIN_TYPE_CLASSIFIERS: dict[str, Classifier] = { + "r_ifelse_na_propagation": _is_r_ifelse_na_propagation, +} + + +def _excel_serial_to_datetime(serial: float) -> datetime.date | datetime.time | datetime.datetime: + """Convert an Excel serial to the exact value openpyxl would read for it. + + Delegates to openpyxl's own ``from_excel`` rather than hand-rolling the + epoch math, for two reasons verified against real flagged rows (ticket + 24): Excel (and Lotus 1-2-3 before it) treats 1900 as a leap year that + never existed, so serials 59 and 60 both resolve to 1900-02-28 -- a + plain ``EXCEL_EPOCH + timedelta`` (this module's own + ``parse_date_flexible`` convention) lands one day off for any serial in + that range; and a serial with no whole-day component (e.g. 0) comes back + as a bare ``datetime.time``, matching what openpyxl itself returns for a + cell formatted as time-of-day only. + """ + from openpyxl.utils.datetime import from_excel + + return from_excel(serial) + + +def _is_openpyxl_date_typed_stray_cell(m: CellMismatch) -> bool: + """A non-date column holds a cell carrying an Excel date/time number format. + + The two readers disagree about that cell and nothing else. R's readxl + infers a whole column's type from its majority values and hands back the + raw Excel serial; Python's openpyxl reads each cell individually and + honours its own number format, returning a ``datetime``/``time``. Every + cell behind this cause was opened in the source workbook and genuinely + carries such a format, so the *detection* is sound across the whole + population. + + Ticket 62 measured that population (100 rows, raw stage only, five + columns across both arms) and found the single verdict this docstring + used to state -- "Python's value is the faithful one, R's is a coercion + artifact" -- is right for only a third of it. Three shapes, each with its + own answer. Note "stray" in the cause name over-reads for two of them; + the name is kept because ``openpyxl``/``date_typed`` hold for all 100 and + renaming would orphan the ticket history. + + - **A real date typed into a quantity column** (14 rows): the serial sits + in the range a 2000-2050 date occupies (43,566-45,652). Verified at + Penang General 2019 ``Apr19!E36`` ("Units Received", ``d/m/yy``) by + ticket 24. Python is the correct side, and visibly so downstream -- + ``_is_stray_date_zeroed`` covers the cleaned face, where R carries + 43,708 into the stock ledger and Python zeroes it. + - **A quantity, or a zero, in a date-formatted cell** (66 rows: 48 with + an underlying 0 read back as ``00:00:00``, 18 with a genuine count of + 5-200 read back as a 1900 date). Here R's serial *is* the number the + clinician typed and Python's date is the misleading rendering -- the + old verdict was backwards. It costs nothing because the divergence does + not survive cleaning: verified against the real 254-tracker pair that + both sides publish identical cleaned values for the affected + (product, sheet) groups, e.g. 2023 Yangon General ``Apr23`` column G, + whose ``d-mmm-yy`` format sits on cells holding 150, 200, 30, 5 and 0. + A raw-stage representation artifact, not a data difference, so the + pipeline is deliberately not changed for it (ticket 32's precedent for + ``parse_date_flexible``'s epoch: do not move production output to fix a + raw-stage comparison row). + - **The patient arm** (20 rows, 5 source cells, 5 trackers), which this + docstring never mentioned even though ticket 50 wired it -- see the + ``blood_pressure_mmhg``/``testing_frequency`` note in + ``scripts/compare_outputs.py``. Each is a value Excel silently + auto-converted on entry: ``10/60`` becoming Oct-1960 in a + "Blood Pressure (mm HG)" column (2020 Mahosot ``Nov20!R152``/ + ``Dec20!R153``, 2022 Mahosot ``Jun22!Z323``), ``1-2`` becoming 1-Feb in + "Testing Frequency (per day)" (2021 Khon Kaen ``Mar21!P60``, whose own + column holds ``' 1 - 2'`` four rows above), and a diagnosis *date* in + "Age at Diagnosis" (2023 Chiang Mai ``Patient List!H14``, 2023 Yangon + General ``Patient List!H76``). Python is unambiguously correct: it + nulls them, while R publishes diagnosis ages of **20,668** and + **42,859** years. All five are source defects and already reach the + errors table as ``type_conversion`` with the original value intact, so + ticket 40 needs nothing added for them. + """ + try: + r_serial = float(m.r_value) + except TypeError, ValueError: + return False + expected = _excel_serial_to_datetime(r_serial) + if isinstance(expected, datetime.datetime): + expected_date, expected_time = expected.date(), expected.time() + elif isinstance(expected, datetime.date): + expected_date, expected_time = expected, datetime.time(0, 0) + else: + expected_date, expected_time = None, expected + py_str = str(m.py_value).strip() + try: + parsed = datetime.datetime.strptime(py_str, "%Y-%m-%d %H:%M:%S") + return parsed.date() == expected_date and parsed.time() == expected_time + except ValueError: + pass + try: + parsed_time = datetime.datetime.strptime(py_str, "%H:%M:%S").time() + return parsed_time == expected_time + except ValueError: + return False + + +STRAY_DATE_CLASSIFIERS: dict[str, Classifier] = { + "openpyxl_date_typed_stray_cell": _is_openpyxl_date_typed_stray_cell, +} + + +# An Excel serial in the range a real 2000-2050 date occupies. A unit count +# this large does not occur in the tracker data (the largest genuine +# product_units_received across the 248-tracker set is three digits), so a +# value here is a misplaced date, not a quantity. +_PLAUSIBLE_DATE_SERIAL_RANGE = (36526.0, 54789.0) + + +def _is_stray_date_zeroed(m: CellMismatch) -> bool: + """The cleaned-stage face of STRAY_DATE_CLASSIFIERS' cause (ticket 36). + + Where a clinician typed a date into a quantity column -- verified against + the real source Excel (2019 Sultanah Bahiyah, Aug19 rows 10-12: the entry + date sits in column E, "Units Received", with column D, "Date", empty) -- + R's readxl coerces the column to numeric and carries the raw serial into + its output, corrupting the ledger it feeds (ticket 36's addendum measured + R closing at 43,572 in five such groups). Python's cleaning fails the + float cast, emits a ``type_conversion`` error per row, and zeroes the + cell. Python is the correct side; the raw-stage classifier cannot see + this because it expects Python's value to still be a datetime. + """ + try: + r_serial = float(m.r_value) # type: ignore[arg-type] + except TypeError, ValueError: + return False + low, high = _PLAUSIBLE_DATE_SERIAL_RANGE + if not low <= r_serial <= high: + return False + try: + return float(m.py_value) == 0.0 # type: ignore[arg-type] + except TypeError, ValueError: + return False + + +STRAY_DATE_ZEROED_CLASSIFIERS: dict[str, Classifier] = { + "stray_date_zeroed": _is_stray_date_zeroed, +} + + +def _is_stray_date_dropped(m: CellMismatch) -> bool: + """The same cause again where the cleaned column is integer-typed. + + Ticket 57, one cell: 2021 Khon Kaen Mar21 records a date in + ``testing_frequency``. R carries the Excel serial (44228) into a column + meaning "tests per day"; Python's cast fails and leaves null rather than the + 0 its float columns get, so neither of the other two stray-date tests can + see it. Python is the correct side and the source cell is a defect for + ticket 40. + """ + try: + r_serial = float(m.r_value) # type: ignore[arg-type] + except TypeError, ValueError: + return False + low, high = _PLAUSIBLE_DATE_SERIAL_RANGE + return low <= r_serial <= high and m.py_value is None + + +STRAY_DATE_DROPPED_CLASSIFIERS: dict[str, Classifier] = { + "stray_date_dropped": _is_stray_date_dropped, +} + + +# Excel's own formula-error sentinels (ticket 27): a computed raw column +# (e.g. bmi, t1d_diagnosis_age -- both formula-derived in the source +# trackers) holds a literal "#DIV/0!"/"#VALUE!"/etc. string wherever the +# source formula could not compute -- an input it depends on was never +# recorded (height blank -> #DIV/0!; no diagnosis date -> #NUM!). Verified +# against real source Excel: the cell's openpyxl data_type is 'e'. +# +# Python's raw extraction preserves that text uniformly (ticket 27's own +# change -- the raw layer records what the source contained). R's does not: +# readxl guesses a column's type from its majority values, so the same error +# cell survives as a string in a column guessed character but becomes NA in +# a column guessed numeric -- the same type-guessing mechanism +# STRAY_DATE_CLASSIFIERS documents for a different symptom. So the mismatch +# appears in BOTH directions depending on which way readxl guessed, and +# neither direction is a Python defect: Python is the consistent side. +EXCEL_ERROR_STRINGS = frozenset( + {"#DIV/0!", "#VALUE!", "#NUM!", "#N/A", "#REF!", "#NAME?", "#NULL!"} +) + + +def _is_excel_formula_error(m: CellMismatch) -> bool: + """One side carries an Excel formula error string; the other has null. + + **Verdict (ticket 60): Python is the correct side, and the test's symmetry + is currently unexercised.** Measured over the real 254-tracker pair: all + 12,680 rows run one way -- Python holds the error string, R holds null -- + across ``t1d_diagnosis_age`` (8,495), ``bmi`` (4,170) and ``age`` (15), and + every one is at the **raw** stage. readxl coerces an Excel error cell to + NA; openpyxl returns the literal ``#DIV/0!``, which is what raw extraction + is for -- a faithful record of what the cell contains, including that it + contains a broken formula rather than a missing value. + + Nothing survives into cleaned output on either side (executed: 0 + error-string cells in all three columns across every cleaned parquet, both + pipelines), so the divergence is confined to the stage whose job is + fidelity, and Python's extra information costs the downstream tables + nothing. The reverse direction is kept in the test because a future tracker + could produce it, not because anything currently does. + """ + return (m.r_value in EXCEL_ERROR_STRINGS and m.py_value is None) or ( + m.py_value in EXCEL_ERROR_STRINGS and m.r_value is None + ) + + +EXCEL_FORMULA_ERROR_CLASSIFIERS: dict[str, Classifier] = { + "excel_formula_error": _is_excel_formula_error, +} + + +# Buddhist-era source-data typo (ticket 27): a clinician typed a Thai +# Buddhist-Era year (BE = CE + 543) directly into a Gregorian-calendar +# date cell -- verified against the real source Excel (06 Nakornping +# Hospital, patient TH_NK004, hba1c_updated_date: the cell is genuinely +# date-typed and formatted `dd-mmm-yyyy` but holds year 2569). R's raw +# extraction already rejects the implausible year and emits the sentinel; +# Python's raw extraction is a faithful pass-through of the cell's literal +# value, by design, so it carries the bad year through. Not a pipeline bug: +# the cleaned stage's own future-date guard (`_validate_dates`, +# `clean/patient.py`) independently replaces it with the same sentinel. +# +# Which side sentinels is not fixed (ticket 37): on `blood_pressure_updated` +# the direction reverses at the cleaned stage, because R's raw extraction +# carries the BE year through for that column and R's cleaning has no +# future-date guard, while Python's does. Same typo, same verdict -- the +# sentinelling side is the one that recognized an unusable year -- so the +# test is symmetric rather than two separately-named causes. Threshold +# matches the product pipeline's own `BUDDHIST_ERA_THRESHOLD` +# (`clean/product.py`) rather than a new one. +PATIENT_BUDDHIST_ERA_THRESHOLD = 2400 + + +def _is_buddhist_era_typo(m: CellMismatch) -> bool: + """One side sentinelled a Buddhist-era year the other carried through. + + Bounded by the sheet's own BE band (ticket 60). The bare + ``year >= PATIENT_BUDDHIST_ERA_THRESHOLD`` test this used to carry claimed + *any* far-future date as a Buddhist-era typo, which after ticket 61 is + exactly backwards: the cleaned stage now converts genuine BE dates, so a + year still sitting above the threshold is by construction not a + recoverable BE date. Both real populations it was claiming are ordinary + year typos in the source workbook, verified against the cells themselves + -- ``3035-03-01`` (2025 CDA, Mar25!O219) and ``5025-05-19`` (2025 Surat + Thani, May25!O70), each a genuine date-formatted Excel cell whose year is + neither Gregorian-plausible nor 543 off its tracker's. + + The band mirrors ``_is_python_absurd_excel_serial``'s. Where the sheet + year cannot be read the old behaviour stands, since an unknown tracker + year cannot disprove a BE date. + """ + sentinelled, carried = ( + (m.r_value, m.py_value) if m.r_value == SENTINEL_DATE else (m.py_value, m.r_value) + ) + if sentinelled != SENTINEL_DATE or not isinstance(carried, datetime.date): + return False + if carried.year < PATIENT_BUDDHIST_ERA_THRESHOLD: + return False + tracker_year = m.tracker_year if m.tracker_year is not None else _sheet_year(m) + if tracker_year is None: + return True + buddhist_year = tracker_year + BUDDHIST_ERA_OFFSET + return buddhist_year - YEAR_FLOOR_DELTA <= carried.year <= buddhist_year + + +def _is_python_buddhist_era_converted(m: CellMismatch) -> bool: + """R keeps the Buddhist-era year the clinic wrote; Python converts it. + + Ticket 61: a Thai clinic's tracker is kept in a Thai-locale Excel, so its + dates arrive with a Buddhist-era year (BE = CE + 543) -- the calendar the + clinic uses, not an error it made. The cleaned stage now shifts those to + Gregorian (``_convert_buddhist_era_dates``, clean/patient.py, and the BE + band in ``_validate_entry_dates``, clean/product.py), so downstream reads + one calendar. R does not convert, which makes this a deliberate divergence + rather than a defect on either side. + + On the patient arm it is also a recovery: before the conversion existed, + ``_validate_dates`` saw a year centuries ahead and clobbered the cell with + the sentinel, so 375 readings were destroyed rather than merely published + oddly. + + The test is the shift itself -- R's year is implausible as Gregorian, and + Python's date is exactly the same day 543 years earlier. Anything else + riding along (a day that also moves, a row-order shift) fails it, so a + second divergence cannot hide behind this name. + """ + r_date, py_date = _as_date(m.r_value), _as_date(m.py_value) + if r_date is None or py_date is None or SENTINEL_DATE in (r_date, py_date): + return False + return ( + r_date.year >= PATIENT_BUDDHIST_ERA_THRESHOLD + and r_date.year - BUDDHIST_ERA_OFFSET == py_date.year + and (r_date.month, r_date.day) == (py_date.month, py_date.day) + ) + + +# Kept as its own registry because both arms need it, and it has to be checked +# *before* the product entry-date causes: `python_out_of_window_date_preserved` +# would otherwise claim a converted cell, whose R side is genuinely outside the +# tracker window (ticket 61). +def _is_buddhist_era_conversion_row_order(m: CellMismatch) -> bool: + """The conversion moved the row, so the ordinal now pairs two different rows. + + Product rows are aligned by position within (clinic_id, sheet), and R sorts + an unconverted Buddhist-era date to the end of its group -- 2565-12-24 is + later than every Gregorian date beside it. Python converts first and sorts + 2022-12-24 chronologically, three rows earlier, which displaces everything + between. Verified on the real 254-tracker pair: 2022 Chiang Mai Maharaj + Nakorn, sheet Dec22, "Accu-Chek Instant Test Strips" -- R's ordinal 11 + holds 2565-12-24 where Python holds 2022-12-28, and Python's ordinal 9 + holds the converted 2022-12-24. + + This is ``row_order_divergence`` with the conversion as its trigger, and it + needs its own test because R's value is by construction absent from + Python's group -- Python converted it -- so value membership cannot fire. + """ + return m.era_shift_row_order_candidate + + +BUDDHIST_ERA_CONVERSION_CLASSIFIERS: dict[str, Classifier] = { + "python_buddhist_era_converted": _is_python_buddhist_era_converted, + "buddhist_era_conversion_row_order": _is_buddhist_era_conversion_row_order, +} + +PATIENT_BUDDHIST_ERA_CLASSIFIERS: dict[str, Classifier] = BUDDHIST_ERA_CONVERSION_CLASSIFIERS | { + "buddhist_era_typo": _is_buddhist_era_typo, +} + + +def _is_r_ymd_first_misparse(m: CellMismatch) -> bool: + """R reads a ``D.M.YY`` source string year-first, swapping day and year. + + ``parse_date_string`` (r-archive/R/script2_helper_dates.R) calls + ``lubridate::parse_date_time`` with ``orders = c("ymd", "dmy", "my")``. + lubridate takes the first order that parses, and "30.1.18" parses fine as + ymd -- year 30 -> 2030 -- so the dmy order is never reached. Every + date the source writes with a two-digit year and a day that is not also a + plausible-looking year lands on the wrong reading. + + Python is the correct side, on three independent grounds: + + * **Source.** 2018 Yangon Children's Hospital writes these dates as text + inside the value cell -- ``Jan18!L92`` is ``10.3 (22.8.17)`` and + ``Jan18!N92`` is ``223(30.1.18)``, day-first, matching Python's + 2017-08-22 and 2018-01-30 against R's 2022-08-17 and 2030-01-18. + * **Impossibility.** R's reading puts 1,236 of the 1,714 affected cells in + the *future* relative to the tracker's own year; Python's puts none + there and lands 1,713 of them within three years of it. + * **Out-of-range days.** Where the source day exceeds 12 (``31.5.16``), + only the day-first reading is even well-formed as a date the clinic + could have meant; R still produces 2031-05-16. + + The test is the digit swap itself -- same month, R's day is Python's + two-digit year and vice versa -- and excludes the case where the two agree + on the day, since a date whose day and year read alike (18.1.18) is + parsed identically by both orders and so cannot be this cause. + """ + r_date, py_date = _as_date(m.r_value), _as_date(m.py_value) + if r_date is None or py_date is None: + return False + if SENTINEL_DATE in (r_date, py_date): + return False + return ( + r_date.month == py_date.month + and r_date.day == py_date.year % 100 + and r_date.year % 100 == py_date.day + and r_date.day != py_date.day + ) + + +PATIENT_YMD_FIRST_CLASSIFIERS: dict[str, Classifier] = { + "r_ymd_first_misparse": _is_r_ymd_first_misparse, +} + + +def _is_python_reads_bare_year(m: CellMismatch) -> bool: + """A bare four-digit year in a date cell: Python reads the year, R the serial. + + Nine trackers type a year alone where the template asks for + ``(dd-mmm-yyyy)`` -- 590 D.O.B. cells across four Yangon Children's + trackers and 425 diagnosis dates across Sarawak, Uni Med Center, VNCH, + Heart of Jesus and MMMHMC. Both pipelines used to read that number as an + Excel serial, which lands in 1905; ``parse_date_flexible`` now resolves + it as the year (ticket 52), so only R still does. + + Python is right, and the source says so rather than the shape of the diff: + Sarawak General Hospital's **2024** workbook writes the same patients' + diagnoses as real 1-January dates (``Patient List!G10`` is 2011-01-01 for + MY_SW001) where its 2025 and 2026 workbooks write the bare year, so the + first of January is the clinic's own convention for a year with no day, + and the recovered ages agree across the three years. R's reading has every + such patient born or diagnosed in 1905, which drove ``age`` to the 999999 + sentinel and ``t1d_diagnosis_age`` negative. + + The test is the serial arithmetic itself: R's date must be exactly the + Excel serial of the year Python read, which no coincidentally-1-January + date can satisfy. + """ + r_date, py_date = _as_date(m.r_value), _as_date(m.py_value) + if r_date is None or py_date is None: + return False + if (py_date.month, py_date.day) != (1, 1): + return False + return r_date == EXCEL_EPOCH + datetime.timedelta(days=py_date.year) + + +PATIENT_BARE_YEAR_CLASSIFIERS: dict[str, Classifier] = { + "python_reads_bare_year": _is_python_reads_bare_year, +} + + +def _is_r_month_name_truncated_to_year(m: CellMismatch) -> bool: + """A spelled-out month-year cell R keeps only the year of. + + ``parse_dates`` (script2_helper_patient_data_fix.R) shortens any word of + four or more letters with ``sub("([[:alpha:]]{3})[[:alpha:]]", "\\\\1", date)``, + which deletes the *fourth* letter and leaves the rest of the word standing: + executed against R 4.5, "April-17" becomes "Aprl-17" and "August,2015" + becomes "Augst,2015". Neither is a month lubridate recognises, so + ``parse_date_time`` walks its order list down to the final ``"y"`` and + returns 1 January of the year -- 2017-01-01 and 2015-01-01, which is + exactly what R's frozen output carries for those cells. + + Python is right: ``parse_date_flexible``'s ``MONTH_NAME_PATTERN`` truncates + the same names to a real abbreviation and its month-year branch resolves + them to the first of the month the clinic actually wrote. Verified by + running R's own truncation and order list over the source strings taken + from the raw parquet (ticket 56), not inferred from the shape of the diff. + + The test is that R sits on 1 January of the same year Python read a + later month for. Python landing on 1 January too is the bare-year cause + above, and is excluded. + """ + r_date, py_date = _as_date(m.r_value), _as_date(m.py_value) + if r_date is None or py_date is None: + return False + if (r_date.month, r_date.day) != (1, 1) or r_date.year != py_date.year: + return False + return py_date.day == 1 and py_date.month != 1 + + +PATIENT_MONTH_NAME_TRUNCATED_CLASSIFIERS: dict[str, Classifier] = { + "r_month_name_truncated_to_year": _is_r_month_name_truncated_to_year, +} + + +def _is_r_reads_day_as_month(m: CellMismatch) -> bool: + """A month name R cannot read, so it reads the day number as the month. + + Once the alphabetic token fails, ``parse_date_time``'s order list reaches + ``"my"``, which reads the two remaining numbers as month and year. The day + is promoted to a month and the day itself is invented as the 1st: executed + against R, "9-Dce-20" gives 2020-09-01, "11-Mach-20" gives 2020-11-01, + "4-Okt-2023" gives 2023-04-01 and "10-MAC-2026" gives 2026-10-01 -- every + one matching R's frozen output for that cell. + + The month names behind this are two dropped or transposed letters ("Dce", + "ug"), R's own inability to keep "Mach" once truncation has run, and the + Bahasa Malaysia spellings the Malaysian clinics write ("Mac", "Mei", + "Okt"). Python is right on all of them: ``TYPO_REPLACEMENTS`` now maps each + to its English abbreviation before parsing (ticket 56), so Python publishes + the date the clinic wrote where R publishes a month it derived from the day. + + The test is that arithmetic identity -- R on the first of a month whose + number is Python's day, in the same year. + """ + r_date, py_date = _as_date(m.r_value), _as_date(m.py_value) + if r_date is None or py_date is None: + return False + if SENTINEL_DATE in (r_date, py_date): + return False + return r_date.day == 1 and r_date.year == py_date.year and py_date.day == r_date.month + + +def _is_r_parse_order_cannot_read_cell(m: CellMismatch) -> bool: + """R sentinels because none of its eight parse orders fits the cell. + + The same mechanism as ``r_reads_day_as_month`` at the point where it runs + out of road: with the month name unreadable, R's positional fallback needs + the day to be a valid month number, so a day past 12 leaves the whole order + list without a match and ``convert_to`` substitutes ERROR_VAL_DATE. + Executed against R: "25-Mach-20" returns NA, as do "05/15/2026" and + "Dec-22-2025" -- the month-first spellings no order in + ``c("dmy","dmY","dbY","by","bY","mY","my","y")`` can express. Python reads + all three. + + A day past 12 is the signature, and it is what separates this from the + other reasons R sentinels a date. It is deliberately not applied to + ``hospitalisation_date``, whose population round 6 measured as 100% clinical + notes: R sentinels those because it cannot read a sentence, which is + ticket 39's open question and not this cause. + """ + r_date, py_date = _as_date(m.r_value), _as_date(m.py_value) + if r_date != SENTINEL_DATE or py_date is None or py_date == SENTINEL_DATE: + return False + return py_date.day > 12 + + +PATIENT_UNREADABLE_MONTH_CLASSIFIERS: dict[str, Classifier] = { + "r_reads_day_as_month": _is_r_reads_day_as_month, + "r_parse_order_cannot_read_cell": _is_r_parse_order_cannot_read_cell, +} + + +def _is_age_from_bare_year(m: CellMismatch) -> bool: + """An age derived from a date whose bare year only Python recovered. + + ``age`` comes from ``dob`` and ``t1d_diagnosis_age`` from ``dob`` plus + ``t1d_diagnosis_date`` (``_fix_age``/``_fix_t1d_diagnosis_age``, + clean/patient.py). Where one of those dates is a bare year (see + ``python_reads_bare_year``), R derives from its 1905 reading and Python + from the real one, so the two ages *must* differ -- R's lands around 113 + and its own 0-100 range check then stamps the 999999 sentinel. + + This is the same decided cause one step downstream, not a second one, so + it is keyed on the mechanism: the row must itself carry a bare-year date + mismatch. The derived values' shape alone (R sentinel, Python plausible) + would also match a genuine age disagreement. + """ + return m.row_has_bare_year_date + + +PATIENT_AGE_FROM_BARE_YEAR_CLASSIFIERS: dict[str, Classifier] = { + "python_age_from_bare_year": _is_age_from_bare_year, +} + + +# No human has lived past 123, so a diagnosis age this large is not an age at +# all. Both populations that clear it decode as Excel serials to real dates -- +# 20668 to 1956-08-01 and 42859 to 2017-05-04 -- confirmed in the source +# workbooks themselves (see source_date_in_diagnosis_age). +_MAX_PLAUSIBLE_AGE_YEARS = 130 + + +def _numeric(value: Any) -> float | None: + try: + return float(value) # type: ignore[arg-type] + except TypeError, ValueError: + return None + + +def _is_r_never_derives_diagnosis_age(m: CellMismatch) -> bool: + """Python computes a diagnosis age from the two dates; R never computes one. + + R's ``fix_t1d_diagnosis_age`` (script2_helper_patient_data_fix.R) exists + and is unit-tested against exactly the strings the trackers carry -- "At + birth", "4 months", "5y", "10y10m" -- but its **call site is commented + out** (script2_process_patient_data.R:251, read directly). So R's + ``t1d_diagnosis_age`` is only ever the source column put through + ``as.numeric``: a blank cell stays NA, and a cell written in words becomes + R's 999999 "recorded but invalid" sentinel. Python's + ``_fix_t1d_diagnosis_age`` (clean/patient.py) fills from ``dob`` and + ``t1d_diagnosis_date`` in exactly those two cases and otherwise keeps what + the clinic recorded. + + Python is right, and the source says so rather than the shape of the diff. + Where the source wrote the age in words, Python's derived figure agrees + with the words: 2017 Mandalay's MM_MD010 reads "11yr" against a D.O.B. of + 2004-03-01 and a diagnosis of 2016-01-01, and Python derives 11; MM_MD011 + reads "4mth" against 2013-10-23 and 2014-02-01, and Python derives 0. + Where the cell is blank, both pipelines hold the same two dates and only + Python uses them -- 2024 Sarawak's MY_SW001 (2000-06-30, 2011-01-01 on + both sides) is null in R and 10 in Python. R keeps nothing a clinic + recorded; Python recovers information R discards. + + The bare-year subset of this shape is already ``python_age_from_bare_year`` + (ticket 52), which fires only where the row's own date cells disagree. The + rest of the population has no date mismatch at all, so it needs the + mechanism stated directly. + + **Where the two dates come from matters, though the reason above does + not change** (ticket 63). 683 of this cause's cells are the respelled-ID + patients ticket 58 recovered: 2024 Mahosot's LA_MH056 is null on all three + of ``dob``, ``t1d_diagnosis_date`` and ``t1d_diagnosis_age`` in R's frozen + output, while Python holds 2014-02-22, 2021-06-22 and 7. So these rows are + divergences ticket 58 *created* -- before it, Python had no dates to derive + from either. The cause still states the right reason, because R would + publish nothing here even holding both dates: its ``fix_t1d_diagnosis_age`` + call site is commented out. Recorded rather than re-labelled, and no count + moves. + """ + if m.py_value is None: + return False + if m.r_value is None: + return True + r_numeric = _numeric(m.r_value) + return r_numeric is not None and r_numeric == settings.error_val_numeric + + +def _is_source_date_in_diagnosis_age(m: CellMismatch) -> bool: + """A date typed into the `Age at Diagnosis` column: R carries the serial. + + The template's ``Age at\\nDiagnosis*`` column is formula-derived, so most + cells hold either a number or an Excel error string. Two clinics have a + date sitting there instead, confirmed by reading the source workbooks: + 2023 Chiang Mai's Patient List row for TH_CP005 holds 1956-08-01 (serial + 20668) in a workbook whose ``Date of T1D Diagnosis`` column is empty for + every patient, so its age formula reads ``#NUM!`` throughout; and 2023 + Yangon General's row for MM_YC043_YG holds 2017-05-04 (serial 42859) + against a D.O.B. of 2008-01-01 and a diagnosis date of 2007-06-01 -- + a diagnosis a year before the birth. + + R's readxl reads the column as numeric and carries the raw serial into the + age. Python's cast fails and the cell ends up null, which is the correct + side: 20,668 is not an age, and no arithmetic recovers one here because + the diagnosis date these clinics would need is itself missing or + impossible. Both are source defects for ticket 40, not pipeline + divergences to reconcile. + + The test is that R's value is too large to be an age, rather than that it + decodes to a date -- an age column has no legitimate value in that range, + and R's own 999999 sentinel is excluded because ``r_numeric_error_sentinel`` + owns it. + """ + if m.py_value is not None: + return False + r_numeric = _numeric(m.r_value) + if r_numeric is None or r_numeric == settings.error_val_numeric: + return False + return r_numeric > _MAX_PLAUSIBLE_AGE_YEARS + + +PATIENT_DIAGNOSIS_AGE_CLASSIFIERS: dict[str, Classifier] = { + "r_never_derives_diagnosis_age": _is_r_never_derives_diagnosis_age, + "source_date_in_diagnosis_age": _is_source_date_in_diagnosis_age, +} + +# The three mg/dL readings R's fix_fbg manufactures from text +# (script2_helper_patient_data_fix.R:557-561, citing CDC's "getting tested" +# levels): high/bad/hi -> 200, med/medium -> 170, low/good/okay -> 140. +_R_FBG_CATEGORY_VALUES = frozenset({140.0, 170.0, 200.0}) + + +def _is_r_fbg_text_category_invention(m: CellMismatch) -> bool: + """R converts unreadable text into one of three invented glucose readings. + + ``fix_fbg`` (r-archive/R/script2_helper_patient_data_fix.R:551) runs its + category patterns through ``grepl`` on the whole string with no word + boundary, so any cell merely *containing* the letters wins a number. Two + populations follow, and every one of the 85 affected cells was read back to + its raw source value to confirm which: + + - 44 cells where the source is a category word or a phrase containing one. + 41 of them read ``Lost follow up`` and become **140**, because "fol-low" + contains "low"; the other 3 are a genuine ``low``/``Low``. A patient lost + to follow-up has no glucose reading at all, and R publishes one. + - 41 cells where the source records a meter range or an out-of-range + marker: ``SMBG 50-HI``, ``DSMP 250-HI``, ``129-HI``, ``CBG 57-High``, + ``112/ High``, bare ``HI``. All become **200**, discarding the numeric + endpoint the clinic did write. On a glucometer "HI" means a reading above + the analytical ceiling -- far above 200 -- so R's substitution is not + merely invented but wrong in direction. + + Python is the correct side and diverges here by design: + ``_fix_fbg_column`` (clean/patient.py) implements the same CDC mapping but + anchors each pattern to the full string (``^(low|good|okay)$``), so only a + cell that says exactly "low" is treated as a category and everything else + falls through to the numeric conversion and sentinels. The source cells are + free text in a numeric column -- a defect for ticket 40, not a value to + reconstruct. + + Keyed on R holding exactly one of the three invented constants while Python + holds its numeric sentinel: Python only sentinels a cell it could not read + at all, so a real reading of 200 cannot land in this shape. + """ + r_numeric = _numeric(m.r_value) + py_numeric = _numeric(m.py_value) + return ( + r_numeric in _R_FBG_CATEGORY_VALUES + and py_numeric is not None + and py_numeric == settings.error_val_numeric + ) + + +def _is_r_unit_suffix_not_stripped(m: CellMismatch) -> bool: + """The source writes the unit beside the reading; only Python reads past it. + + The 2018 trackers record updated FBG as ``148 mg/dl (Mar-18)`` -- value, + unit, and the date of the reading in one cell. Python strips the unit and + lifts the parenthetical date out to ``fbg_updated_date`` + (``_fix_fbg_column`` and ``_extract_date_from_measurement``, + clean/patient.py); R's ``fix_fbg`` handles neither, so ``as.numeric`` fails + on the whole string and ``convert_to`` substitutes its sentinel. + + All 28 cells in this shape were read back to their raw source, and every + one is a plain reading carrying its unit (three also carry no date: + ``332 mg/dl``, ``196((Dec-2017)``). Python recovers a real measurement R + throws away, so Python is the correct side. + + Keyed on the reverse of the invention shape above -- R sentinelled, Python + holds a number -- which on this column is only reachable when R's parse + failed and Python's did not. + """ + r_numeric = _numeric(m.r_value) + py_numeric = _numeric(m.py_value) + return ( + r_numeric is not None + and r_numeric == settings.error_val_numeric + and py_numeric is not None + and py_numeric != settings.error_val_numeric + ) + + +PATIENT_FBG_TEXT_CLASSIFIERS: dict[str, Classifier] = { + "r_fbg_text_category_invention": _is_r_fbg_text_category_invention, + "r_unit_suffix_not_stripped": _is_r_unit_suffix_not_stripped, +} + +# A month sheet is named "Jan22", "Sept22" or "Oct'22" -- the trailing two +# digits are the tracker year, which is how the pipeline's own extraction +# derives tracker_year in the first place (get_tracker_year, R; the sheet-name +# year detection documented in docs/CLAUDE.md, Python). +_SHEET_YEAR = re.compile(r"(\d{2})\s*$") + + +def _sheet_year(m: CellMismatch) -> int | None: + """The tracker year the mismatched row sits in, read off its sheet name. + + ``add_row_ordinal`` renames every key column to ``__key_``, so the + lookup matches on the suffix rather than on the bare name -- keying on + ``sheet_name`` alone silently finds nothing and the cause never fires. + """ + if not isinstance(m.key, dict): + return None + sheet = next( + (v for k, v in m.key.items() if str(k).endswith("sheet_name") and isinstance(v, str)), + None, + ) + match = _SHEET_YEAR.search(sheet) if sheet else None + return 2000 + int(match.group(1)) if match else None + + +def _is_python_rejects_beyond_tracker_year(m: CellMismatch) -> bool: + """Python sentinels a date later than its tracker year; R keeps it. + + ``_validate_dates`` (clean/patient.py) replaces any date past 31 December + of ``tracker_year`` with the error sentinel and logs an ``invalid_value`` + error against the patient. R has no equivalent guard -- ``grep`` over + ``r-archive/R`` finds no future-date or tracker-year bound on any date + column -- so R carries the impossible value into its output unchanged. + (The docstring on ``_validate_dates`` claiming this "matches R pipeline + behavior" is wrong, and is corrected as part of this ticket.) + + Python is the correct side, and the source says the value is genuinely + impossible rather than merely surprising: 2022 Vietnam National Children's + Hospital writes every one of its 43 ``Date of T1D Diagnosis`` cells as a + 2023 date (``Patient List!G13`` = 2023-07-16 for VN_VC001, born 2013-03, + recruited 2017-07), so the workbook has patients diagnosed a year after + the tracker was filled in and five years after they were recruited for + having the disease. That is a defect in the workbook -- reported for + [ticket 40] rather than inferred around -- and sentinelling it is the + pipeline recognizing an unusable value, the same verdict + ``buddhist_era_typo`` reached on an unusable year. + + Deliberately narrower than "Python sentinelled and R did not": where the + raw cell holds free text Python's parser could not read at all (a + misspelled month, a date buried in a clinical note), the mechanism is a + parse failure rather than this guard, the verdict is not the same, and + part of that population is still open as its own question. + """ + tracker_year = _sheet_year(m) + r_date = _as_date(m.r_value) + return ( + tracker_year is not None + and _as_date(m.py_value) == SENTINEL_DATE + and r_date is not None + and r_date != SENTINEL_DATE + and r_date.year > tracker_year + ) + + +PATIENT_BEYOND_TRACKER_YEAR_CLASSIFIERS: dict[str, Classifier] = { + "python_rejects_beyond_tracker_year": _is_python_rejects_beyond_tracker_year, +} + + +def _is_r_na_unite_padding(m: CellMismatch) -> bool: + """R renders an absent sub-column as the literal string ``NA`` when uniting. + + The 2023 template splits complication screening across B.P./Kidney/Eye/Foot/ + Lipids sub-columns that all map to one canonical ``complication_screening``; + ``reading_patient_data`` (r-archive/R/script1_read_patient_data.R) merges them + with ``tidyr::unite(sep = ",")``, whose ``na.rm`` defaults to ``FALSE``, so + every empty cell in the group becomes the four characters ``NA`` in R's + output: a patient screened in January reads ``JAN,NA,NA,NA,NA`` and a patient + screened not at all reads ``NA,NA,NA,NA,NA`` rather than being null. + + Python merges the same group in ``ColumnMapper.rename_columns`` but skips + empty cells, so it carries the values alone. Verified against the real source + Excel (2023 Kantha Bopha, Jan'23!AB98-AF98 = B.P./Kidney/Eye/Foot/Lipids; + KH_KB023 has ``JAN`` in Kidney and nothing else): the sub-columns R pads are + genuinely empty in the workbook, so the padding is R's rendering and carries + no information. Python is the correct side. + + Fires only when stripping the ``NA`` tokens from R's value leaves exactly + Python's value, so a real disagreement inside the group stays unclassified. + + Surviving parts are compared trimmed (ticket 50): Python trims each + sub-value before merging -- ticket 46's separately-verified + ``python_trims_merged_subvalue`` -- so where a source cell ends in a space + the two causes stack in one cell and neither explained it alone. Trimming + only ignores leading/trailing space per part, so an internal whitespace + difference (``r_drops_richtext_space``) still falls through to its own + cause. + """ + if m.r_value is None or "NA" not in str(m.r_value): + return False + stripped = [p.strip() for p in str(m.r_value).split(",") if p.strip() not in ("NA", "")] + py_parts = [] if m.py_value is None else str(m.py_value).split(",") + return stripped == [p.strip() for p in py_parts if p.strip() != ""] + + +PATIENT_NA_UNITE_PADDING_CLASSIFIERS: dict[str, Classifier] = { + "r_na_unite_padding": _is_r_na_unite_padding, +} + + +def _spaces_differ_only(m: CellMismatch) -> bool: + if m.r_value is None or m.py_value is None: + return False + r_text, py_text = str(m.r_value), str(m.py_value) + return r_text != py_text and r_text.replace(" ", "") == py_text.replace(" ", "") + + +def _is_r_drops_richtext_space(m: CellMismatch) -> bool: + """R loses a space that Excel stores as a formatting run of its own. + + Where a cell's text carries mixed formatting, xlsx stores it as a + rich-text shared string: a sequence of runs, each with its own font, and + a space between two differently-formatted fragments becomes a run holding + nothing but that space. openpyxl concatenates every run and reproduces the + cell as Excel displays it; readxl drops the whitespace-only run. + + Verified in the source XML of two unrelated workbooks (ticket 46): + + - 2017 Yangon Children's Hospital, ``hba1c_updated``/``fbg_updated_mg``: + ``..8.8.. .. + (20.9.16)``. The same workbook also stores the identical + text as a plain ``8.8 (20.9.16)`` for other patients, + which R reads correctly -- which is why only some cells of the column + diverge. + - 2022 Mahosot, ``observations``: the same three-run shape around the + space in "Unable to contact", the middle run recoloured. + + Python is the correct side: it reproduces the workbook. Fires only when + the two values are identical once every space is removed *and* Python is + the side holding more of them -- the opposite direction is a different + cause (see ``python_trims_merged_subvalue``). + """ + if not _spaces_differ_only(m): + return False + return str(m.py_value).count(" ") > str(m.r_value).count(" ") + + +PATIENT_RICHTEXT_SPACE_CLASSIFIERS: dict[str, Classifier] = { + "r_drops_richtext_space": _is_r_drops_richtext_space, +} + + +def _is_python_trims_merged_subvalue(m: CellMismatch) -> bool: + """Python trims each sub-value before joining a duplicate column group. + + Ticket 31 gave ``ColumnMapper.rename_columns`` a comma-join over the + source columns that share one canonical name; it strips each fragment + first, while R's ``tidyr::unite`` concatenates them exactly as read. So a + source cell carrying a trailing space reaches R's output as + "Normal ,Insulin" and Python's as "Normal,Insulin". + + Python is the correct side -- the space is inside the workbook's cell + padding, not part of the recorded value, and the pipeline trims every + string cell elsewhere. Requires the comma both sides carry, so an + ordinary leading/trailing space (already handled by + ``normalize_whitespace_column``) does not land here. + """ + if not _spaces_differ_only(m): + return False + if "," not in str(m.py_value): + return False + return str(m.r_value).count(" ") > str(m.py_value).count(" ") + + +PATIENT_MERGED_SUBVALUE_TRIM_CLASSIFIERS: dict[str, Classifier] = { + "python_trims_merged_subvalue": _is_python_trims_merged_subvalue, +} + + +def _glucose_float(value: Any) -> float | None: + try: + return float(value) + except TypeError, ValueError: + return None + + +def _is_python_glucose_unit_corrected(m: CellMismatch) -> bool: + """Python resolves a glucose reading recorded under the wrong unit's header; + R carries the number through as written. + + Ticket 42, on A4D's medical advisor's answer of 2026-08-17. Where a whole + column labelled mg/dL holds mmol/L readings (measured: 29 file-columns at 21 + clinics, nine tenths or more of each column below 30 mg/dL), + ``resolve_glucose_units`` (src/a4d/clean/glucose.py) moves the values to the + mmol column and rescales the mg one by 18. It also blanks readings of exactly + 0, which sit below the analytical floor of both units and mean "not + measured", and range validation then rejects readings outside the advisor's + analytical limits. R has no unit resolution at all, so each of those shows as + a divergence. + + This is a **deliberate Python correction, not an R defect and not a + divergence either side could win** -- R was never asked the question. It + fires only on the three shapes the correction provably produces: an exact + 18x ratio in either direction, a null against R's literal 0, and Python's + numeric sentinel against a reading outside the analytical range. R-null + cells are excluded, because that population predates this change and belongs + to the still-open baseline-FBG join question. + """ + if m.column not in GLUCOSE_UNIT_COLUMNS: + return False + r = _glucose_float(m.r_value) + if r is None: + return False + + py = _glucose_float(m.py_value) + if py is None: + return r == 0.0 + + if py == settings.error_val_numeric: + # Judged against the unit the column actually holds, which is not + # always the one its name claims. Ticket 57 found six cells this could + # not reach while the test went by name alone: 2020 Kantha Bopha's + # fbg_baseline_mg is 90.1% sub-30, so the column is read as mmol/L and + # its outliers 47.8 and 53.8 are rejected against the mmol ceiling -- + # while sitting well inside the mg/dL range. Round 10 measured the + # obvious widening ("outside either unit") and backed it out, since it + # would claim every genuinely-rejected mg/dL reading above 45. Ticket + # 44 supplies the missing fact instead: column_unit_swapped comes from + # the run's own glucose_unit_swapped records, so the bound is chosen on + # evidence rather than on the column's name. + # + # A constant fix_fbg manufactured out of text is not a reading, so no + # range has anything to say about it: r_fbg_text_category_invention + # owns those, and without this guard the mmol bound would take five of + # them off it purely because 140 exceeds 45. + if r in _R_FBG_CATEGORY_VALUES: + return False + reads_as_mmol = m.column_unit_swapped or not m.column.endswith("_mg") + low, high = ( + (MMOL_ANALYTICAL_MIN, MMOL_ANALYTICAL_MAX) + if reads_as_mmol + else (MG_ANALYTICAL_MIN, MG_ANALYTICAL_MAX) + ) + return r < low or r > high + + for expected in (r * MMOL_TO_MG_FACTOR, r / MMOL_TO_MG_FACTOR): + if abs(py - expected) <= max(1e-6, abs(expected) * 1e-4): + return True + return False + + +def _is_python_recovers_glucose_r_rejected(m: CellMismatch) -> bool: + """The unit swap moves a reading across that R had already sentinelled. + + Ticket 57, one cell: 2022 Penang MY_PN004 Dec22 writes "-" in the mmol + column and 9 in the mg one. R's converter turns the "-" into its numeric + error value and stops there. Python's ``resolve_glucose_units`` finds that + column 145/145 sub-30 -- so it is mmol throughout -- and moves the 9 into + the mmol column, where it lands inside the advisor's analytical range. + + Python is the correct side: the clinic recorded a reading and R published + an error code over it. Fires only where R holds the numeric error value and + Python holds a reading the analytical limits accept, so it cannot claim a + recovery the range check would itself have rejected. + """ + if m.column not in GLUCOSE_UNIT_COLUMNS: + return False + r, py = _glucose_float(m.r_value), _glucose_float(m.py_value) + if r != settings.error_val_numeric or py is None: + return False + low, high = ( + (MG_ANALYTICAL_MIN, MG_ANALYTICAL_MAX) + if m.column.endswith("_mg") + else (MMOL_ANALYTICAL_MIN, MMOL_ANALYTICAL_MAX) + ) + return low <= py <= high + + +PATIENT_GLUCOSE_UNIT_CLASSIFIERS: dict[str, Classifier] = { + "python_glucose_unit_corrected": _is_python_glucose_unit_corrected, + "python_recovers_glucose_r_rejected": _is_python_recovers_glucose_r_rejected, +} + +_MMOL_GLUCOSE_COLUMNS = frozenset(mmol_col for _, mmol_col in GLUCOSE_COLUMN_PAIRS) + + +def _carries_a_reading(value: Any) -> bool: + """Did this side publish a glucose measurement in this cell at all?""" + reading = _glucose_float(value) + return reading is not None and reading != settings.error_val_numeric + + +def _restates_its_own_mg_cell(mmol_value: Any, mg_value: Any) -> bool: + """Is this side's mmol cell nothing but its own mg cell in mmol/L? + + A side that published nothing here -- null, or its numeric sentinel -- + trivially added no reading of its own. A side that did publish one has to + be the exact 18-fold image of the mg cell it sits beside; anything else is + a measurement the mg cell cannot account for. + """ + if not _carries_a_reading(mmol_value): + return True + mmol = _glucose_float(mmol_value) + if not _carries_a_reading(mg_value) or mmol is None: + return False + mg = _glucose_float(mg_value) + assert mg is not None + expected = mg / MMOL_TO_MG_FACTOR + return abs(mmol - expected) <= max(1e-6, abs(expected) * 1e-4) + + +def _is_mmol_derived_from_mg_sibling(m: CellMismatch) -> bool: + """The mmol cell restates a divergence its mg sibling already carries. + + Neither pipeline reads the mmol column from the workbook independently: + both derive it from the mg cell beside it (``convert_glucose_units``, + clean/patient.py; ``fix_fbg``'s ``fbg/18`` in R). So when the two sides + disagree about the mg cell, they disagree about the mmol cell as an + arithmetic consequence, and the mmol difference is not a second finding. + + Measured over the whole 254-tracker run, all 2,935 cells in this shape sat + beside an mg cell that was itself a mismatch with an already-named cause -- + not one exception -- in three populations: + + - **2,797** where ticket 42's unit swap moved a reading into a column R + never populated (2025 Kantha Bopha II KH_KB097: the source writes 14.4 + under the mg/dL header; R keeps mg=14.4 and no mmol, Python moves it to + mmol and rescales mg to 259.2). + - **110** where R derived mmol from an mg reading Python refused -- either + one ``fix_fbg`` invented out of text (140 from "Lost follow up") or one + beyond the analytical ceiling (2013 mg/dL, which R publishes as 111.8 + mmol/L, above the level the medical advisor called impossible). + - **28** the other way round, where R sentinelled an mg cell carrying its + unit ("148 mg/dl (Mar-18)") and Python read it and derived 8.22. + + Python is the correct side in all three; each is already argued under the + mg cell's own cause, which is why this one names the cascade rather than + re-deciding it. Bounded two ways so it cannot swallow a genuine mmol + divergence: the mg sibling must itself disagree, and each side's mmol value + must be either absent or exactly its own mg cell divided by 18. + """ + if m.column not in _MMOL_GLUCOSE_COLUMNS or m.mg_sibling is None: + return False + r_mg, py_mg = m.mg_sibling + if not _values_differ(r_mg, py_mg): + return False + if not _restates_its_own_mg_cell(m.r_value, r_mg): + return False + if not _restates_its_own_mg_cell(m.py_value, py_mg): + return False + return _carries_a_reading(m.r_value) or _carries_a_reading(m.py_value) + + +PATIENT_GLUCOSE_CASCADE_CLASSIFIERS: dict[str, Classifier] = { + "mmol_derived_from_mg_sibling": _is_mmol_derived_from_mg_sibling, +} + + +_HBA1C_RANGE_COLUMNS = ("hba1c_baseline", "hba1c_updated") + + +def _is_python_rejects_out_of_range_hba1c(m: CellMismatch) -> bool: + """A glucose reading typed into the HbA1c column. + + Ticket 57, eight cells across 2024 Preah Kossamak and two other clinics: + the source holds 240, 299 and 125 where an HbA1c percentage belongs. R has + no range check on this column and publishes the number as an HbA1c; Python + fails it against the bound ``reference_data/validation_rules.yaml`` declares + and stamps the numeric sentinel. + + Python is the correct side -- an HbA1c of 240% is not a measurement, and + publishing it would carry a fasting-glucose reading into the HbA1c series. + The bound is read from the config rather than restated, so the classifier + cannot drift from the rule that produced the rejection. Each cell is also a + source defect for ticket 40: the number itself is real, it is in the wrong + column. + """ + if m.column not in _HBA1C_RANGE_COLUMNS: + return False + ceiling = load_numeric_ranges().get(m.column, {}).get("max") + if ceiling is None: + return False + r, py = _numeric(m.r_value), _numeric(m.py_value) + if r is None or py != settings.error_val_numeric: + return False + return r > ceiling + + +PATIENT_HBA1C_RANGE_CLASSIFIERS: dict[str, Classifier] = { + "python_rejects_out_of_range_hba1c": _is_python_rejects_out_of_range_hba1c, +} + + +def _is_python_declines_unreconstructable_date(m: CellMismatch) -> bool: + """R publishes a date from a token damaged past reading; Python refuses. + + The other face of ``r_parse_order_cannot_read_cell``. Round 9 established by + running R 4.5 that ``parse_dates`` cannot fail quietly: it deletes the fourth + letter of any long word and then walks a fixed order list ending in ``my`` + and ``y``, so *something* comes back for almost any input. Where the source + cell is damaged in a way no reading can be recovered from, that something is + an invention. + + Ticket 57 measured 37 such cells and none is reconstructable: + + - ``25-Ma4-2025`` (18 cells, 2025 Taunggyi) -- "Ma4" is Mar or May and the + workbook says neither; R reads the embedded 4 as the month and publishes + 2025-04-25, an answer neither spelling supports. + - a separator swallowed or a digit group glued (``26/102022``, ``8/1023``, + ``3/10.23``, ``10-Oct-2-24``, ``13-Mar-0202``) -- where the missing + separator goes is a guess, and R's own answer is not stable: its frozen + output reads ``10/1023`` as 2010-10-23 while running its own parser over + that string in isolation returns 2023-10-10. + - ``e.g. xxx (mth-18)`` (2018 Penang DC) -- the tracker template's own + example text left in a patient row, which R turns into 2018-01-01. + + Python is the correct side of all three: declining to publish a date the + source does not determine is right, and every cell here is a finding for + ticket 40 rather than something to infer. The test is the direction alone -- + R on a real date, Python on the sentinel -- so it is wired only to the date + columns whose other causes are already classified ahead of it. + """ + r_date, py_date = _as_date(m.r_value), _as_date(m.py_value) + if r_date is None or py_date != SENTINEL_DATE: + return False + return r_date != SENTINEL_DATE + + +PATIENT_UNRECONSTRUCTABLE_DATE_CLASSIFIERS: dict[str, Classifier] = { + "python_declines_unreconstructable_date": _is_python_declines_unreconstructable_date, +} + + +def _is_wide_format_fragment_truncated(m: CellMismatch) -> bool: + """R's value is a truncated prefix of Python's for a 2017-2019 Mandalay + wide-format ``product_units_released`` cell. + + ``handle_wide_format_cells`` (extract/wide_format.py) splits a + comma-separated "Released To" cell on commas, then each fragment on its + first ``-`` into (name, qty). Verified against the real source Excel + (ticket 24, e.g. 2019_Mandalay Children's Hospital Feb19!K49: + "MM_MD019-2(Error-1),..."): the source cell's free-text notes contain + embedded parentheses and extra hyphens the split logic doesn't fully + anticipate. Python's qty fragment keeps the full remainder after the + first ``-``; R's equivalent step drops content after a second hyphen, + so R's value is consistently a strict prefix of Python's for the same + cell -- a genuine R parsing gap on inherently messy source notes, not a + Python bug, and not something to "fix" toward R's more-truncated answer. + """ + r_str, py_str = str(m.r_value), str(m.py_value) + return r_str != py_str and py_str.startswith(r_str) + + +WIDE_FORMAT_FRAGMENT_CLASSIFIERS: dict[str, Classifier] = { + "wide_format_fragment_truncated": _is_wide_format_fragment_truncated, +} + + +@dataclass(frozen=True) +class FileComparison: + file_name: str + shape: ShapeResult + totals: list[TotalsMismatch] + columns: ColumnsResult + id_overlap: IdOverlapResult | None + categorical_overlap: list[CategoricalOverlap] + row_key_overlap: RowKeyOverlap + cell_mismatches: list[CellMismatch] + + +@dataclass(frozen=True) +class DirectoryComparison: + files: list[FileComparison] + only_in_r: list[str] + only_in_py: list[str] + + +@dataclass(frozen=True) +class DirectorySummary: + """Whole-arm rollup of a DirectoryComparison. + + Each measure is ``(files_affected, total)`` -- the per-file table shows + where a divergence is, this shows how much of it there is overall, which + is otherwise only obtainable by scrolling every year's table. + """ + + files_compared: int + shape_mismatch_files: int + id_divergence: tuple[int, int] + column_divergence: tuple[int, int] + categorical_divergence: tuple[int, int] + totals_divergence: tuple[int, int] + row_key_divergence: tuple[int, int] + cell_divergence: tuple[int, int] + only_in_r: int + only_in_py: int + + +def summarize_directory(comparison: DirectoryComparison) -> DirectorySummary: + """Roll a DirectoryComparison up into per-measure (files affected, total).""" + + def rollup(per_file: list[int]) -> tuple[int, int]: + return sum(1 for n in per_file if n), sum(per_file) + + files = comparison.files + return DirectorySummary( + files_compared=len(files), + shape_mismatch_files=sum(1 for f in files if not f.shape.match), + id_divergence=rollup( + [ + 0 + if f.id_overlap is None + else len(f.id_overlap.only_in_r) + len(f.id_overlap.only_in_py) + for f in files + ] + ), + column_divergence=rollup( + [ + len(f.columns.only_in_r) + + len(f.columns.only_in_py) + + len(f.columns.dtype_mismatches) + for f in files + ] + ), + categorical_divergence=rollup([len(f.categorical_overlap) for f in files]), + totals_divergence=rollup([len(f.totals) for f in files]), + row_key_divergence=rollup( + [f.row_key_overlap.r_unmatched + f.row_key_overlap.py_unmatched for f in files] + ), + cell_divergence=rollup([len(f.cell_mismatches) for f in files]), + only_in_r=len(comparison.only_in_r), + only_in_py=len(comparison.only_in_py), + ) + + +def compare_directory( + r_frames: dict[str, pl.DataFrame], + py_frames: dict[str, pl.DataFrame], + key_cols: list[str], + numeric_cols: list[str] | None = None, + id_col: str | None = None, + categorical_cols: list[str] | None = None, + order_group_cols: list[str] | None = None, + unit_swapped_columns: dict[str, set[str]] | None = None, + tracker_year_col: str | None = None, + static_join_missed_ids_by_frame: dict[str, set[str]] | None = None, +) -> DirectoryComparison: + numeric_cols = numeric_cols or [] + unit_swapped_columns = unit_swapped_columns or {} + static_join_missed_ids_by_frame = static_join_missed_ids_by_frame or {} + categorical_cols = categorical_cols or [] + r_names, py_names = set(r_frames), set(py_frames) + common = sorted(r_names & py_names) + + files = [] + for name in common: + r_df, py_df = r_frames[name], py_frames[name] + files.append( + FileComparison( + file_name=name, + shape=compare_shape(r_df, py_df), + totals=compare_totals(r_df, py_df, numeric_cols), + columns=compare_columns(r_df, py_df), + id_overlap=compare_id_overlap(r_df, py_df, id_col) if id_col else None, + categorical_overlap=compare_categorical_overlap(r_df, py_df, categorical_cols), + row_key_overlap=compare_row_key_overlap(r_df, py_df, key_cols), + cell_mismatches=compare_cells( + r_df, + py_df, + key_cols, + order_group_cols, + unit_swapped_columns.get(name), + tracker_year_col, + static_join_missed_ids_by_frame.get(name), + ), + ) + ) + + return DirectoryComparison( + files=files, + only_in_r=sorted(r_names - py_names), + only_in_py=sorted(py_names - r_names), + ) + + +_R_BLANK_HEADER_COLUMN = re.compile(r"^na\d*(\.(monthly|static|\d+))*$") + + +def classify_column_divergence(kind: str, column: str, r_only: set[str], py_only: set[str]) -> str: + """Name why a column exists on one side of the comparison only. + + Two structural causes account for almost all of the patient raw stage's + divergence, and neither is a content difference: + + - `r_blank_header_artifact`: R's `make.names(header_cols, unique = TRUE)` + (script1_helper_read_patient_data.R) renders a blank header cell as the + *string* "NA.", so the following `is.na(colnames(...))` drop never fires + and every spacer column survives as na, na1, ... na10064. Python drops + blank-header columns in `filter_valid_columns`, so it has no equivalent. + - `name_sanitization_only`: both sides carry the column, but R stores the + sanitized name ("currentinsulinregimen") where Python's raw stage keeps + the literal source header ("Current Insulin Regimen"). + + Checked in that order: R's artifact name would otherwise be masked by any + Python column that happens to sanitize to "na". + """ + if kind == "only in R" and _R_BLANK_HEADER_COLUMN.match(column): + return "r_blank_header_artifact" + + other = py_only if kind == "only in R" else r_only + if sanitize_str(column) in {sanitize_str(c) for c in other}: + return "name_sanitization_only" + + return "unclassified" + + +def build_mismatch_rows( + comparison: DirectoryComparison, + classifiers_by_column: dict[str, dict[str, Classifier]] | None = None, +) -> dict[str, list[dict[str, Any]]]: + classifiers_by_column = classifiers_by_column or {} + + id_overlap_rows = [] + categorical_overlap_rows = [] + row_key_overlap_rows = [] + row_key_unmatched_rows = [] + totals_rows = [] + cell_mismatch_rows = [] + column_divergence_rows = [] + + for file_comparison in comparison.files: + name = file_comparison.file_name + + r_only = set(file_comparison.columns.only_in_r) + py_only = set(file_comparison.columns.only_in_py) + + for column in file_comparison.columns.only_in_r: + column_divergence_rows.append( + { + "file": name, + "column": column, + "kind": "only in R", + "r_dtype": "", + "py_dtype": "", + "cause": classify_column_divergence("only in R", column, r_only, py_only), + } + ) + for column in file_comparison.columns.only_in_py: + column_divergence_rows.append( + { + "file": name, + "column": column, + "kind": "only in Python", + "r_dtype": "", + "py_dtype": "", + "cause": classify_column_divergence("only in Python", column, r_only, py_only), + } + ) + for column, r_dtype, py_dtype in file_comparison.columns.dtype_mismatches: + column_divergence_rows.append( + { + "file": name, + "column": column, + "kind": "dtype mismatch", + "r_dtype": str(r_dtype), + "py_dtype": str(py_dtype), + "cause": "unclassified", + } + ) + + row_key_overlap_rows.append( + { + "file": name, + "matched": file_comparison.row_key_overlap.matched, + "r_unmatched": file_comparison.row_key_overlap.r_unmatched, + "py_unmatched": file_comparison.row_key_overlap.py_unmatched, + } + ) + + # An unmatched row never reaches a cell comparison, so unless its key is + # named here it appears nowhere else in the report (ticket 47). + for side, keys in ( + ("R only", file_comparison.row_key_overlap.r_only_keys), + ("Python only", file_comparison.row_key_overlap.py_only_keys), + ): + for key, surplus in keys: + row_key_unmatched_rows.append( + { + "file": name, + "side": side, + "key": " | ".join(str(part) for part in key), + "rows": surplus, + } + ) + + if file_comparison.id_overlap is not None: + for value in file_comparison.id_overlap.only_in_r: + id_overlap_rows.append({"file": name, "side": "R only", "value": value}) + for value in file_comparison.id_overlap.only_in_py: + id_overlap_rows.append({"file": name, "side": "Python only", "value": value}) + + for overlap in file_comparison.categorical_overlap: + for value in overlap.only_in_r: + categorical_overlap_rows.append( + {"file": name, "column": overlap.column, "side": "R only", "value": value} + ) + for value in overlap.only_in_py: + categorical_overlap_rows.append( + { + "file": name, + "column": overlap.column, + "side": "Python only", + "value": value, + } + ) + + for totals_mismatch in file_comparison.totals: + totals_rows.append( + { + "file": name, + "column": totals_mismatch.column, + "r_total": totals_mismatch.r_total, + "py_total": totals_mismatch.py_total, + "diff": totals_mismatch.py_total - totals_mismatch.r_total, + } + ) + + for mismatch in file_comparison.cell_mismatches: + registry = classifiers_by_column.get(mismatch.column, {}) + cell_mismatch_rows.append( + { + "file": name, + "key": "; ".join(f"{k}={v}" for k, v in mismatch.key.items()), + "column": mismatch.column, + "r_value": mismatch.r_value, + "py_value": mismatch.py_value, + "cause": classify(mismatch, registry), + } + ) + + return { + "column_divergence": column_divergence_rows, + "id_overlap": id_overlap_rows, + "categorical_overlap": categorical_overlap_rows, + "row_key_overlap": row_key_overlap_rows, + "row_key_unmatched": row_key_unmatched_rows, + "totals": totals_rows, + "cell_mismatches": cell_mismatch_rows, + } + + +def snapshot_from_summary(summary: dict[str, list[dict[str, Any]]]) -> dict[str, dict[str, int]]: + """Reduce a ``build_summary_rows`` result to a compact, JSON-serializable + snapshot for run-over-run history (ticket 19). + + Only ``per_column``/``per_cause`` are kept -- the counts that actually + move as triage fixes land -- keyed as plain strings so the snapshot + round-trips through JSON without a custom decoder. + """ + per_column = {row["column"]: row["mismatches"] for row in summary.get("per_column", [])} + per_cause = { + f"{row['column']}|{row['cause']}": row["mismatches"] for row in summary.get("per_cause", []) + } + return {"per_column": per_column, "per_cause": per_cause} + + +@dataclass(frozen=True) +class Delta: + key: str + previous: int + current: int + + +def compute_deltas(previous: dict[str, int], current: dict[str, int]) -> list[Delta]: + """Diff two snapshot count-dicts, keyed by column or "column|cause". + + Only changed keys are returned -- a key with an unchanged count (present + identically in both, or absent from both) is not a fix or a regression + and would just be noise in a delta report. A key missing from one side + is treated as a count of 0 on that side, so a fully-resolved column + (count drops to zero and the row disappears from the new summary) still + shows up as a delta rather than silently vanishing. + """ + keys = previous.keys() | current.keys() + return sorted( + ( + Delta(key=key, previous=previous.get(key, 0), current=current.get(key, 0)) + for key in keys + if previous.get(key, 0) != current.get(key, 0) + ), + key=lambda d: d.key, + ) + + +def build_summary_rows( + comparison: DirectoryComparison, + classifiers_by_column: dict[str, dict[str, Classifier]] | None = None, +) -> dict[str, list[dict[str, Any]]]: + classifiers_by_column = classifiers_by_column or {} + + per_column: dict[str, int] = {} + per_cause: dict[tuple[str, str], int] = {} + for file_comparison in comparison.files: + for mismatch in file_comparison.cell_mismatches: + per_column[mismatch.column] = per_column.get(mismatch.column, 0) + 1 + registry = classifiers_by_column.get(mismatch.column, {}) + cause = classify(mismatch, registry) + per_cause[(mismatch.column, cause)] = per_cause.get((mismatch.column, cause), 0) + 1 + + return { + "per_column": [ + {"column": col, "mismatches": count} + for col, count in sorted(per_column.items(), key=lambda kv: -kv[1]) + ], + "per_cause": [ + {"column": col, "cause": cause, "mismatches": count} + for (col, cause), count in sorted(per_cause.items(), key=lambda kv: -kv[1]) + ], + "only_in_r": [{"file": name} for name in comparison.only_in_r], + "only_in_py": [{"file": name} for name in comparison.only_in_py], + } + + +def _is_python_reads_date_in_clinical_note(m: CellMismatch) -> bool: + """R cannot read a sentence, so it sentinels; Python reads the date it states. + + ``hospitalisation_date``'s header is free text -- "Hospitalisation due to + diabetes emergency or glucose control (Include Date)", read directly from + 2021 Preah Kossamak's ``Feb21`` sheet -- so clinicians write a case note and + put the date inside it. Round 6 measured the column's whole residual as + 100% clinical notes. + + R's ``parse_dates`` hands the string to a fixed order list and gives up, + stamping ERROR_VAL_DATE. Python scans it for an anchored day/month/year + shape (``recover_date_from_text``, clean/date_parser.py) and publishes the + first date the note names. Python is the correct side: the date is legibly + present -- "DKA 23 Oct 2020", "Passed away 28/10/2019 due to DKA" -- and + discarding it loses a hospitalisation the clinic recorded. + + The recogniser refuses rather than guesses, which is what keeps this cause + honest: a cell carrying numbers and no date ("3 month come back meet + Doctor", 48 cells) yields nothing and stays sentinelled on both sides. + """ + r_date, py_date = _as_date(m.r_value), _as_date(m.py_value) + return r_date == SENTINEL_DATE and py_date is not None and py_date != SENTINEL_DATE + + +def _is_note_dates_read_differently(m: CellMismatch) -> bool: + """Both pipelines find a date in the note and pick different ones. + + Four shapes, all traced to their source strings on the real 254-tracker set + and all resolving in Python's favour: + + - **A stay's two endpoints.** "DKA: admitted 6-12 Nov 2020" -> R reads the + range's two numbers as a day and a month (2020-12-06); Python takes the + admission day the note's own month gives it (2020-11-06). + - **A day read as a year.** "26 Jun (ceton urine high)" -> R returns + 2026-01-01; Python resolves 26 June against the tracker's year. + - **A note listing several admissions.** "Dec 2019, Mar 2020 DKA Jan 2021 + DKA" -> R composes 2019-03-20 out of fragments of all three; Python + publishes the first, and logs ``date_multiple_in_cell`` so the discard is + visible. + - **A range with no separators at all.** "25Jul-2Aug2022" (1 cell) is the + one shape neither side reads correctly: R gives 2022-02-25 and Python + 2022-08-02, the discharge day, because the missing separator makes the + opening "25Jul" indistinguishable from a token with an unreadable year. + The source is what needs correcting, and it is reported for that. + """ + r_date, py_date = _as_date(m.r_value), _as_date(m.py_value) + if r_date is None or py_date is None: + return False + return SENTINEL_DATE not in (r_date, py_date) and r_date != py_date + + +def _is_r_invents_january_from_bare_year(m: CellMismatch) -> bool: + """The note names a year and no month; R publishes 1 January, Python refuses. + + "DKA 2019" and "DKA 2020: June, Aug, Nov" (24 cells across two trackers). + R's parse order list ends in ``y``, so a four-digit number anywhere in the + string becomes 1 January of that year -- which the second example shows to + be plainly wrong, since the note says the admissions were in June, August + and November. + + Python refuses: a month cannot be recovered from anything in the cell, and + the whole-cell bare-year convention (``python_reads_bare_year``, ticket 52) + rests on a clinic writing *only* a year, which is not this. The sentinel is + the honest reading -- something was written, and it is not a date. + """ + r_date, py_date = _as_date(m.r_value), _as_date(m.py_value) + if py_date != SENTINEL_DATE or r_date is None or r_date == SENTINEL_DATE: + return False + return r_date.month == 1 and r_date.day == 1 + + +PATIENT_CLINICAL_NOTE_CLASSIFIERS: dict[str, Classifier] = { + "python_reads_date_in_clinical_note": _is_python_reads_date_in_clinical_note, + "note_dates_read_differently": _is_note_dates_read_differently, + "r_invents_january_from_bare_year": _is_r_invents_january_from_bare_year, +} diff --git a/src/a4d/pipeline/__init__.py b/src/a4d/pipeline/__init__.py new file mode 100644 index 0000000..d256ed8 --- /dev/null +++ b/src/a4d/pipeline/__init__.py @@ -0,0 +1,18 @@ +"""Pipeline orchestration for A4D data processing.""" + +from a4d.pipeline.models import PipelineResult, TrackerResult +from a4d.pipeline.patient import ( + discover_tracker_files, + process_patient_tables, + run_patient_pipeline, +) +from a4d.pipeline.tracker import process_tracker_patient + +__all__ = [ + "PipelineResult", + "TrackerResult", + "discover_tracker_files", + "process_patient_tables", + "process_tracker_patient", + "run_patient_pipeline", +] diff --git a/src/a4d/pipeline/models.py b/src/a4d/pipeline/models.py new file mode 100644 index 0000000..c6abf84 --- /dev/null +++ b/src/a4d/pipeline/models.py @@ -0,0 +1,83 @@ +"""Pipeline result models for tracking processing outputs.""" + +from dataclasses import dataclass, field +from pathlib import Path + +from a4d.errors import DataError + + +@dataclass +class TrackerResult: + """Result from processing a single tracker file. + + Attributes: + tracker_file: Original tracker file path + tracker_name: Base name without extension + raw_output: Path to raw parquet file (None if extraction failed) + cleaned_output: Path to cleaned parquet file (None if cleaning failed) + success: Whether processing completed successfully + error: Error message if processing failed + cleaning_errors: Number of data quality errors during cleaning (type conversion, + validation failures, etc.). These are non-fatal - data is cleaned + with error values (999999, "Undefined", etc.) + error_breakdown: Breakdown of errors by type (error_code → count). + Example: {"type_conversion": 10, "invalid_value": 5} + data_errors: Individual data quality error records for aggregation into + the errors table. Empty list if no errors or processing failed. + """ + + tracker_file: Path + tracker_name: str + raw_output: Path | None = None + cleaned_output: Path | None = None + success: bool = True + error: str | None = None + cleaning_errors: int = 0 + error_breakdown: dict[str, int] | None = None + data_errors: list[DataError] = field(default_factory=list) + + +@dataclass +class PipelineResult: + """Result from running the complete patient pipeline. + + Attributes: + tracker_results: Results from processing individual trackers + tables: Dictionary mapping table name to output path + total_trackers: Total number of trackers processed + successful_trackers: Number of successfully processed trackers + failed_trackers: Number of failed trackers + success: Whether entire pipeline completed successfully + """ + + tracker_results: list[TrackerResult] + tables: dict[str, Path] + total_trackers: int + successful_trackers: int + failed_trackers: int + success: bool + + @classmethod + def from_tracker_results( + cls, tracker_results: list[TrackerResult], tables: dict[str, Path] | None = None + ) -> PipelineResult: + """Create PipelineResult from tracker results. + + Args: + tracker_results: List of tracker processing results + tables: Dictionary of created tables (empty if table creation skipped) + + Returns: + PipelineResult with computed statistics + """ + successful = sum(1 for r in tracker_results if r.success) + failed = len(tracker_results) - successful + + return cls( + tracker_results=tracker_results, + tables=tables or {}, + total_trackers=len(tracker_results), + successful_trackers=successful, + failed_trackers=failed, + success=failed == 0, + ) diff --git a/src/a4d/pipeline/patient.py b/src/a4d/pipeline/patient.py new file mode 100644 index 0000000..0a1e225 --- /dev/null +++ b/src/a4d/pipeline/patient.py @@ -0,0 +1,339 @@ +"""Main patient pipeline orchestration.""" + +import os +from collections.abc import Callable +from concurrent.futures import ProcessPoolExecutor, as_completed +from datetime import datetime +from pathlib import Path + +from loguru import logger +from tqdm import tqdm + +from a4d.config import settings +from a4d.logging import setup_logging +from a4d.pipeline.models import PipelineResult, TrackerResult +from a4d.pipeline.tracker import process_tracker_patient +from a4d.tables.errors import create_table_errors +from a4d.tables.logs import create_table_logs +from a4d.tables.patient import ( + create_table_patient_data_annual, + create_table_patient_data_monthly, + create_table_patient_data_static, + read_cleaned_patient_data, +) + + +def _init_worker_logging(output_root: Path) -> None: + """Initialize logging for worker processes (called once per ProcessPoolExecutor worker).""" + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + pid = os.getpid() + setup_logging( + output_root=output_root, + log_name=f"worker_{timestamp}_pid{pid}", + console_level="ERROR", + ) + + +def discover_tracker_files(data_root: Path) -> list[Path]: + """Discover all Excel tracker files in data_root. + + Searches recursively for .xlsx files, excluding temp files (~$*). + + Args: + data_root: Root directory to search + + Returns: + List of tracker file paths + + Example: + >>> tracker_files = discover_tracker_files(Path("/data")) + >>> len(tracker_files) + 42 + """ + tracker_files = [] + for file in data_root.rglob("*.xlsx"): + if not file.name.startswith("~$"): + tracker_files.append(file) + + return sorted(tracker_files) + + +def process_patient_tables(cleaned_dir: Path, output_dir: Path) -> dict[str, Path]: + """Create final patient tables from cleaned parquets. + + Creates three main tables: + - patient_data_static: Latest data per patient + - patient_data_monthly: All monthly records + - patient_data_annual: Latest data per patient per year (2024+) + + Args: + cleaned_dir: Directory containing cleaned parquet files + output_dir: Directory to write final tables + + Returns: + Dictionary mapping table name to output path + + Example: + >>> tables = process_patient_tables( + ... Path("output/patient_data_cleaned"), + ... Path("output/tables") + ... ) + >>> tables.keys() + dict_keys(['static', 'monthly', 'annual']) + """ + logger.info("Creating final patient tables from cleaned data") + + cleaned_files = list(cleaned_dir.glob("*_patient_cleaned.parquet")) + logger.info(f"Found {len(cleaned_files)} cleaned parquet files") + + if not cleaned_files: + logger.warning("No cleaned files found, skipping table creation") + return {} + + patient_data = read_cleaned_patient_data(cleaned_files) + logger.info(f"Loaded combined patient dataframe: {patient_data.shape}") + + tables = {} + + logger.info("Creating static patient table") + static_path = create_table_patient_data_static(patient_data, output_dir) + tables["static"] = static_path + + logger.info("Creating monthly patient table") + monthly_path = create_table_patient_data_monthly(patient_data, output_dir) + tables["monthly"] = monthly_path + + logger.info("Creating annual patient table") + annual_path = create_table_patient_data_annual(patient_data, output_dir) + tables["annual"] = annual_path + + logger.info(f"Created {len(tables)} patient tables") + return tables + + +def run_patient_pipeline( + tracker_files: list[Path] | None = None, + max_workers: int = 1, + output_root: Path | None = None, + skip_tables: bool = False, + clean_output: bool = False, + progress_callback: Callable[[str, bool], None] | None = None, + show_progress: bool = False, + console_log_level: str | None = None, +) -> PipelineResult: + """Run complete patient data pipeline. + + Processing modes: + - Batch mode: If tracker_files is None, discovers all .xlsx in data_root + - Single file mode: If tracker_files provided, processes only those files + + Pipeline steps: + 1. For each tracker (optionally parallel): + - Extract patient data from Excel → raw parquet + - Clean raw data → cleaned parquet + 2. Create final tables from all cleaned parquets (if not skipped) + + Args: + tracker_files: Specific files to process (None = discover all) + max_workers: Number of parallel workers (1 = sequential) + output_root: Output directory (None = use settings.output_root) + skip_tables: If True, only extract + clean, skip table creation + clean_output: If True, wipe patient_data_raw/, patient_data_cleaned/, tables/ before run + progress_callback: Optional callback(tracker_name, success) called after each tracker + show_progress: If True, show tqdm progress bar + console_log_level: Console log level (None=INFO, ERROR=quiet, etc) + + Returns: + PipelineResult with tracker results and table paths + + Example: + >>> # Process all trackers + >>> result = run_patient_pipeline() + >>> result.success + True + >>> result.successful_trackers + 42 + + >>> # Process single file + >>> result = run_patient_pipeline( + ... tracker_files=[Path("/data/2024_Sibu.xlsx")] + ... ) + + >>> # Parallel processing with progress bar (CLI mode) + >>> result = run_patient_pipeline( + ... max_workers=8, + ... show_progress=True, + ... console_log_level="ERROR" + ... ) + """ + import shutil + + # Use settings defaults if not provided + if output_root is None: + output_root = settings.output_root + + # Wipe previous run's outputs so tables reflect only this run. + if clean_output: + for subdir in ("patient_data_raw", "patient_data_cleaned", "tables", "logs"): + target = output_root / subdir + if target.exists(): + shutil.rmtree(target) + logger.info(f"Cleaned output directory: {target}") + + # Setup main pipeline logging + setup_logging( + output_root, + "pipeline_patient", + console_level=console_log_level if console_log_level else "INFO", + ) + logger.info("Starting patient pipeline") + logger.info(f"Output directory: {output_root}") + logger.info(f"Max workers: {max_workers}") + + # Discover or use provided tracker files + if tracker_files is None: + logger.info(f"Discovering tracker files in: {settings.data_root}") + tracker_files = discover_tracker_files(settings.data_root) + else: + tracker_files = [Path(f) for f in tracker_files] + + logger.info(f"Found {len(tracker_files)} tracker files to process") + + if not tracker_files: + logger.warning("No tracker files found") + return PipelineResult.from_tracker_results([], {}) + + # Process trackers + tracker_results: list[TrackerResult] = [] + + if max_workers == 1: + # Sequential processing (easier for debugging) + logger.info("Processing trackers sequentially") + + # Use tqdm if requested + iterator = ( + tqdm(tracker_files, desc="Processing trackers", unit="file") + if show_progress + else tracker_files + ) + + for tracker_file in iterator: + if isinstance(iterator, tqdm): + iterator.set_description(f"Processing {tracker_file.name}") + + result = process_tracker_patient( + tracker_file=tracker_file, + output_root=output_root, + mapper=None, # Each tracker loads mapper if needed + ) + tracker_results.append(result) + + # Call progress callback if provided + if progress_callback: + progress_callback(tracker_file.name, result.success) + + if result.success: + logger.info(f"✓ Successfully processed: {tracker_file.name}") + else: + logger.error(f"✗ Failed to process: {tracker_file.name} - {result.error}") + if show_progress: + tqdm.write(f"✗ {tracker_file.name}: {result.error}") + + else: + # Parallel processing + logger.info(f"Processing trackers in parallel ({max_workers} workers)") + with ProcessPoolExecutor( + max_workers=max_workers, initializer=_init_worker_logging, initargs=(output_root,) + ) as executor: + # Submit all jobs + futures = { + executor.submit( + process_tracker_patient, + tracker_file, + output_root, + None, # Each worker loads synonyms independently + ): tracker_file + for tracker_file in tracker_files + } + + # Collect results as they complete + futures_iterator = as_completed(futures) + if show_progress: + futures_iterator = tqdm( + futures_iterator, total=len(futures), desc="Processing trackers", unit="file" + ) + + for future in futures_iterator: + tracker_file = futures[future] + try: + result = future.result() + tracker_results.append(result) + + # Call progress callback if provided + if progress_callback: + progress_callback(tracker_file.name, result.success) + + if result.success: + logger.info(f"✓ Completed: {tracker_file.name}") + else: + logger.error(f"✗ Failed: {tracker_file.name} - {result.error}") + if show_progress: + tqdm.write(f"✗ {tracker_file.name}: {result.error}") + except Exception as e: + logger.exception(f"Exception processing {tracker_file.name}") + if show_progress: + tqdm.write(f"✗ {tracker_file.name}: Exception - {str(e)}") + tracker_results.append( + TrackerResult( + tracker_file=tracker_file, + tracker_name=tracker_file.stem, + success=False, + error=str(e), + ) + ) + + # Summary + successful = sum(1 for r in tracker_results if r.success) + failed = len(tracker_results) - successful + logger.info(f"Tracker processing complete: {successful} successful, {failed} failed") + + # Create tables + tables: dict[str, Path] = {} + if not skip_tables: + try: + cleaned_dir = output_root / "patient_data_cleaned" + tables_dir = output_root / "tables" + logs_dir = output_root / "logs" + + tables = process_patient_tables(cleaned_dir, tables_dir) + + # Create logs table separately (operational data, not patient data) + if logs_dir.exists(): + logger.info("Creating logs table from pipeline execution logs") + logs_table_path = create_table_logs(logs_dir, tables_dir) + tables["logs"] = logs_table_path + logger.info(f"Logs table created: {logs_table_path}") + + # Aggregate all data quality errors from every tracker into one table + all_data_errors = [e for r in tracker_results for e in r.data_errors] + logger.info(f"Creating errors table ({len(all_data_errors)} total data quality errors)") + errors_table_path = create_table_errors(all_data_errors, tables_dir) + tables["errors"] = errors_table_path + logger.info(f"Errors table created: {errors_table_path}") + + logger.info(f"Created {len(tables)} tables total") + except Exception: + logger.exception("Failed to create tables") + # Don't fail entire pipeline if table creation fails + else: + logger.info("Skipping table creation (skip_tables=True)") + + # Build result + result = PipelineResult.from_tracker_results(tracker_results, tables) + + if result.success: + logger.info("✓ Pipeline completed successfully") + else: + logger.warning(f"✗ Pipeline completed with {failed} failures") + + return result diff --git a/src/a4d/pipeline/product.py b/src/a4d/pipeline/product.py new file mode 100644 index 0000000..3d03e0a --- /dev/null +++ b/src/a4d/pipeline/product.py @@ -0,0 +1,213 @@ +"""Product pipeline orchestration. + +Mirrors ``pipeline/patient.py``: per-tracker extract+clean (optionally +parallel), followed by final-table creation. +""" + +import os +import shutil +from collections.abc import Callable +from concurrent.futures import ProcessPoolExecutor, as_completed +from datetime import datetime +from pathlib import Path + +from loguru import logger +from tqdm import tqdm + +from a4d.config import settings +from a4d.logging import setup_logging +from a4d.pipeline.models import PipelineResult, TrackerResult +from a4d.pipeline.patient import discover_tracker_files +from a4d.pipeline.tracker import process_tracker_product +from a4d.tables.product import create_table_product_data + + +def _init_worker_logging(output_root: Path) -> None: + """Initialize logging for worker processes (called once per ProcessPoolExecutor worker).""" + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + pid = os.getpid() + setup_logging( + output_root=output_root, + log_name=f"worker_product_{timestamp}_pid{pid}", + console_level="ERROR", + ) + + +def process_product_tables(cleaned_dir: Path, output_dir: Path) -> dict[str, Path]: + """Create the final product table from cleaned parquets. + + Thin wrapper around ``tables.product.create_table_product_data``. + Unlike the patient pipeline (static/monthly/annual) the product pipeline + emits a single ``product_data`` table. + """ + logger.info("Creating final product table from cleaned data") + + cleaned_files = list(cleaned_dir.glob("*_product_cleaned.parquet")) + logger.info(f"Found {len(cleaned_files)} cleaned product parquet files") + + if not cleaned_files: + logger.warning("No cleaned product files found, skipping table creation") + return {} + + product_data_path = create_table_product_data(cleaned_files, output_dir) + return {"product_data": product_data_path} + + +def run_product_pipeline( + tracker_files: list[Path] | None = None, + max_workers: int = 1, + output_root: Path | None = None, + skip_tables: bool = False, + clean_output: bool = False, + progress_callback: Callable[[str, bool], None] | None = None, + show_progress: bool = False, + console_log_level: str | None = None, +) -> PipelineResult: + """Run the end-to-end product pipeline. + + Mirrors ``run_patient_pipeline`` argument-for-argument. + """ + if output_root is None: + output_root = settings.output_root + + if clean_output: + for subdir in ("product_data_raw", "product_data_cleaned"): + target = output_root / subdir + if target.exists(): + shutil.rmtree(target) + logger.info(f"Cleaned output directory: {target}") + # Patient tables share output_root/tables; remove only the product table. + product_table = output_root / "tables" / "product_data.parquet" + if product_table.exists(): + product_table.unlink() + logger.info(f"Cleaned product table: {product_table}") + + setup_logging( + output_root, + "pipeline_product", + console_level=console_log_level if console_log_level else "INFO", + ) + logger.info("Starting product pipeline") + logger.info(f"Output directory: {output_root}") + logger.info(f"Max workers: {max_workers}") + + if tracker_files is None: + logger.info(f"Discovering tracker files in: {settings.data_root}") + tracker_files = discover_tracker_files(settings.data_root) + else: + tracker_files = [Path(f) for f in tracker_files] + + logger.info(f"Found {len(tracker_files)} tracker files to process") + + if not tracker_files: + logger.warning("No tracker files found") + return PipelineResult.from_tracker_results([], {}) + + tracker_results: list[TrackerResult] = [] + + if max_workers == 1: + logger.info("Processing trackers sequentially") + + iterator = ( + tqdm(tracker_files, desc="Processing product trackers", unit="file") + if show_progress + else tracker_files + ) + + for tracker_file in iterator: + if isinstance(iterator, tqdm): + iterator.set_description(f"Processing {tracker_file.name}") + + result = process_tracker_product( + tracker_file=tracker_file, + output_root=output_root, + mapper=None, + ) + tracker_results.append(result) + + if progress_callback: + progress_callback(tracker_file.name, result.success) + + if result.success: + logger.info(f"✓ Successfully processed: {tracker_file.name}") + else: + logger.error(f"✗ Failed to process: {tracker_file.name} - {result.error}") + if show_progress: + tqdm.write(f"✗ {tracker_file.name}: {result.error}") + + else: + logger.info(f"Processing trackers in parallel ({max_workers} workers)") + with ProcessPoolExecutor( + max_workers=max_workers, initializer=_init_worker_logging, initargs=(output_root,) + ) as executor: + futures = { + executor.submit( + process_tracker_product, + tracker_file, + output_root, + None, + ): tracker_file + for tracker_file in tracker_files + } + + futures_iterator = as_completed(futures) + if show_progress: + futures_iterator = tqdm( + futures_iterator, + total=len(futures), + desc="Processing product trackers", + unit="file", + ) + + for future in futures_iterator: + tracker_file = futures[future] + try: + result = future.result() + tracker_results.append(result) + + if progress_callback: + progress_callback(tracker_file.name, result.success) + + if result.success: + logger.info(f"✓ Completed: {tracker_file.name}") + else: + logger.error(f"✗ Failed: {tracker_file.name} - {result.error}") + if show_progress: + tqdm.write(f"✗ {tracker_file.name}: {result.error}") + except Exception as e: + logger.exception(f"Exception processing {tracker_file.name}") + if show_progress: + tqdm.write(f"✗ {tracker_file.name}: Exception - {str(e)}") + tracker_results.append( + TrackerResult( + tracker_file=tracker_file, + tracker_name=tracker_file.stem, + success=False, + error=str(e), + ) + ) + + successful = sum(1 for r in tracker_results if r.success) + failed = len(tracker_results) - successful + logger.info(f"Tracker processing complete: {successful} successful, {failed} failed") + + tables: dict[str, Path] = {} + if not skip_tables: + try: + cleaned_dir = output_root / "product_data_cleaned" + tables_dir = output_root / "tables" + tables = process_product_tables(cleaned_dir, tables_dir) + logger.info(f"Created {len(tables)} product tables total") + except Exception: + logger.exception("Failed to create product tables") + else: + logger.info("Skipping product table creation (skip_tables=True)") + + result = PipelineResult.from_tracker_results(tracker_results, tables) + + if result.success: + logger.info("✓ Product pipeline completed successfully") + else: + logger.warning(f"✗ Product pipeline completed with {failed} failures") + + return result diff --git a/src/a4d/pipeline/tracker.py b/src/a4d/pipeline/tracker.py new file mode 100644 index 0000000..945a9dd --- /dev/null +++ b/src/a4d/pipeline/tracker.py @@ -0,0 +1,197 @@ +"""Single tracker processing: extract + clean.""" + +from pathlib import Path + +from loguru import logger + +from a4d.clean.patient import clean_patient_file +from a4d.clean.product import clean_product_file +from a4d.errors import ErrorCollector +from a4d.extract.patient import export_patient_raw, read_all_patient_sheets +from a4d.extract.product import export_product_raw, read_all_product_sheets +from a4d.logging import file_logger +from a4d.pipeline.models import TrackerResult +from a4d.reference.synonyms import ColumnMapper, load_product_mapper + + +def process_tracker_patient( + tracker_file: Path, output_root: Path, mapper: ColumnMapper | None = None +) -> TrackerResult: + """Process single tracker file: extract + clean patient data. + + This function processes one tracker file end-to-end: + 1. Extract patient data from Excel + 2. Export to raw parquet + 3. Clean the raw data + 4. Export to cleaned parquet + + Each step creates a separate log file for debugging. + + Args: + tracker_file: Path to tracker Excel file + output_root: Root output directory (will create subdirs for raw/cleaned) + mapper: ColumnMapper for synonym mapping (loaded if not provided) + + Returns: + TrackerResult with paths to outputs and success status + + Example: + >>> tracker_file = Path("/data/2024_Sibu.xlsx") + >>> output_root = Path("output") + >>> result = process_tracker_patient(tracker_file, output_root) + >>> result.success + True + >>> result.raw_output + Path('output/patient_data_raw/2024_Sibu_patient_raw.parquet') + """ + tracker_name = tracker_file.stem + + try: + # Setup directories + raw_dir = output_root / "patient_data_raw" + cleaned_dir = output_root / "patient_data_cleaned" + raw_dir.mkdir(parents=True, exist_ok=True) + cleaned_dir.mkdir(parents=True, exist_ok=True) + + # Expected output paths + raw_output = raw_dir / f"{tracker_name}_patient_raw.parquet" + cleaned_output = cleaned_dir / f"{tracker_name}_patient_cleaned.parquet" + + # Log context for this tracker + with file_logger(f"{tracker_name}_patient", output_root): + logger.info(f"Processing tracker: {tracker_file.name}") + + # STEP 1: Extract + logger.info("Step 1: Extracting patient data from Excel") + error_collector = ErrorCollector() + + df_raw = read_all_patient_sheets( + tracker_file=tracker_file, mapper=mapper, error_collector=error_collector + ) + logger.info(f"Extracted {len(df_raw)} rows") + + # Export raw parquet + raw_output = export_patient_raw( + df=df_raw, tracker_file=tracker_file, output_dir=raw_dir + ) + logger.info(f"Raw parquet saved: {raw_output}") + + # STEP 2: Clean + logger.info("Step 2: Cleaning patient data") + + clean_patient_file( + raw_parquet_path=raw_output, + output_parquet_path=cleaned_output, + error_collector=error_collector, + ) + + error_count = len(error_collector) + error_breakdown = error_collector.get_error_summary() + logger.info(f"Cleaned parquet saved: {cleaned_output}") + logger.info(f"Total data quality errors: {error_count}") + if error_breakdown: + logger.info(f"Error breakdown: {error_breakdown}") + + return TrackerResult( + tracker_file=tracker_file, + tracker_name=tracker_name, + raw_output=raw_output, + cleaned_output=cleaned_output, + success=True, + error=None, + cleaning_errors=error_count, + error_breakdown=error_breakdown if error_breakdown else None, + data_errors=error_collector.errors.copy(), + ) + + except Exception as e: + logger.bind(error_code="critical_abort").exception( + f"Failed to process tracker: {tracker_file.name}" + ) + return TrackerResult( + tracker_file=tracker_file, + tracker_name=tracker_name, + raw_output=None, + cleaned_output=None, + success=False, + error=str(e), + ) + + +def process_tracker_product( + tracker_file: Path, output_root: Path, mapper: ColumnMapper | None = None +) -> TrackerResult: + """Process single tracker file: extract + clean product data. + + Mirrors ``process_tracker_patient``. Non-fatal cleaning errors keep + ``success=True`` and surface via ``error_breakdown``; only unhandled + exceptions set ``success=False``. + """ + tracker_name = tracker_file.stem + + try: + raw_dir = output_root / "product_data_raw" + cleaned_dir = output_root / "product_data_cleaned" + raw_dir.mkdir(parents=True, exist_ok=True) + cleaned_dir.mkdir(parents=True, exist_ok=True) + + cleaned_output = cleaned_dir / f"{tracker_name}_product_cleaned.parquet" + + with file_logger(f"{tracker_name}_product", output_root): + logger.info(f"Processing tracker: {tracker_file.name}") + + logger.info("Step 1: Extracting product data from Excel") + error_collector = ErrorCollector() + + mapper = mapper or load_product_mapper() + + df_raw = read_all_product_sheets( + tracker_file=tracker_file, + mapper=mapper, + error_collector=error_collector, + ) + logger.info(f"Extracted {len(df_raw)} rows") + + raw_output = export_product_raw( + df=df_raw, tracker_file=tracker_file, output_dir=raw_dir + ) + logger.info(f"Raw parquet saved: {raw_output}") + + logger.info("Step 2: Cleaning product data") + clean_product_file( + raw_parquet_path=raw_output, + output_parquet_path=cleaned_output, + error_collector=error_collector, + ) + + error_count = len(error_collector) + error_breakdown = error_collector.get_error_summary() + logger.info(f"Cleaned parquet saved: {cleaned_output}") + logger.info(f"Total data quality errors: {error_count}") + if error_breakdown: + logger.info(f"Error breakdown: {error_breakdown}") + + return TrackerResult( + tracker_file=tracker_file, + tracker_name=tracker_name, + raw_output=raw_output, + cleaned_output=cleaned_output, + success=True, + error=None, + cleaning_errors=error_count, + error_breakdown=error_breakdown if error_breakdown else None, + data_errors=error_collector.errors.copy(), + ) + + except Exception as e: + logger.bind(error_code="critical_abort").exception( + f"Failed to process tracker: {tracker_file.name}" + ) + return TrackerResult( + tracker_file=tracker_file, + tracker_name=tracker_name, + raw_output=None, + cleaned_output=None, + success=False, + error=str(e), + ) diff --git a/src/a4d/reference/__init__.py b/src/a4d/reference/__init__.py new file mode 100644 index 0000000..7662305 --- /dev/null +++ b/src/a4d/reference/__init__.py @@ -0,0 +1,43 @@ +"""Reference data loaders and validators. + +This package contains modules for loading and working with reference data +from the shared reference_data/ directory. +""" + +# Loaders (internal utilities) +from a4d.reference.loaders import ( + find_reference_data_dir, + get_reference_data_path, + load_yaml, +) + +# Provinces (validation) +from a4d.reference.provinces import ( + get_country_for_province, + is_valid_province, + load_allowed_provinces, + load_provinces_by_country, +) + +# Synonyms (column mapping) +from a4d.reference.synonyms import ( + ColumnMapper, + load_patient_mapper, + load_product_mapper, +) + +__all__ = [ + # Loaders + "find_reference_data_dir", + "get_reference_data_path", + "load_yaml", + # Synonyms + "ColumnMapper", + "load_patient_mapper", + "load_product_mapper", + # Provinces + "get_country_for_province", + "is_valid_province", + "load_allowed_provinces", + "load_provinces_by_country", +] diff --git a/src/a4d/reference/loaders.py b/src/a4d/reference/loaders.py new file mode 100644 index 0000000..50d3cbe --- /dev/null +++ b/src/a4d/reference/loaders.py @@ -0,0 +1,91 @@ +"""Utilities for loading reference data files. + +This module provides common utilities for loading YAML and other reference +data files under reference_data/. +""" + +import os +from pathlib import Path +from typing import Any + +import yaml +from loguru import logger + + +def find_reference_data_dir() -> Path: + """Find reference_data directory. + + Checks A4D_REFERENCE_DATA env var first (used in Docker/Cloud Run where + the directory is at /app/reference_data). Falls back to walking up from + this file to find the repo root for local development. + + Returns: + Path to reference_data directory + + Raises: + FileNotFoundError: If reference_data directory not found + """ + # Explicit override for Docker/Cloud Run (set A4D_REFERENCE_DATA=/app/reference_data) + if env_path := os.environ.get("A4D_REFERENCE_DATA"): + path = Path(env_path) + if path.exists(): + return path + raise FileNotFoundError(f"reference_data directory not found at {path}") + + # Local dev: navigate from src/a4d/reference/loaders.py up to repo root + # loaders.py -> reference -> a4d -> src -> repo root + repo_root = Path(__file__).parents[3] + reference_data_dir = repo_root / "reference_data" + + if not reference_data_dir.exists(): + raise FileNotFoundError(f"reference_data directory not found at {reference_data_dir}") + + return reference_data_dir + + +def load_yaml( + yaml_path: Path, + relative_to_reference_data: bool = False, +) -> Any: + """Load and parse a YAML file. + + Args: + yaml_path: Path to the YAML file + relative_to_reference_data: If True, yaml_path is relative to + reference_data directory + + Returns: + Parsed YAML content + + Raises: + FileNotFoundError: If the YAML file doesn't exist + yaml.YAMLError: If the YAML file is malformed + """ + if relative_to_reference_data: + reference_data_dir = find_reference_data_dir() + yaml_path = reference_data_dir / yaml_path + + if not yaml_path.exists(): + raise FileNotFoundError(f"YAML file not found: {yaml_path}") + + logger.debug(f"Loading YAML file: {yaml_path}") + + with open(yaml_path) as f: + return yaml.safe_load(f) + + +def get_reference_data_path(*parts: str) -> Path: + """Get path to a file in reference_data directory. + + Args: + *parts: Path components relative to reference_data directory + + Returns: + Absolute path to the file + + Example: + >>> path = get_reference_data_path("synonyms", "synonyms_patient.yaml") + >>> # Returns: /path/to/repo/reference_data/synonyms/synonyms_patient.yaml + """ + reference_data_dir = find_reference_data_dir() + return reference_data_dir.joinpath(*parts) diff --git a/src/a4d/reference/products.py b/src/a4d/reference/products.py new file mode 100644 index 0000000..884d3ae --- /dev/null +++ b/src/a4d/reference/products.py @@ -0,0 +1,77 @@ +"""Product reference-data loaders, backing cleaning steps 2.19 and 2.20. + +Loads the known-products list and product category mapping from the +``Stock_Summary`` sheet of ``master_tracker_variables.xlsx`` in the +shared ``reference_data/`` directory. +""" + +from functools import lru_cache + +import openpyxl +import polars as pl + +from a4d.reference.loaders import get_reference_data_path + + +@lru_cache(maxsize=1) +def _read_stock_summary() -> pl.DataFrame: + """Read the Stock_Summary sheet as a two-column DataFrame. + + Returns a frame with columns ``product`` (lowercased values) and + ``product_category`` (original casing). Null rows and duplicates + are dropped. + """ + xlsx_path = get_reference_data_path("master_tracker_variables.xlsx") + wb = openpyxl.load_workbook(xlsx_path, data_only=True, read_only=True) + try: + ws = wb["Stock_Summary"] + rows = list(ws.iter_rows(min_row=2, values_only=True)) + finally: + wb.close() + + products: list[str] = [] + categories: list[str | None] = [] + for row in rows: + if not row: + continue + name = row[0] + if name is None: + continue + name_str = str(name).strip() + if not name_str: + continue + products.append(name_str.lower()) + cat = row[1] if len(row) > 1 else None + categories.append(str(cat).strip() if cat is not None else None) + + return pl.DataFrame({"product": products, "product_category": categories}).unique( + subset=["product"], keep="first", maintain_order=True + ) + + +def load_known_products() -> list[str]: + """Load the lowercased list of known product names. + + Backs step 2.19 (flagging product names absent from the reference) + (``report_unknown_products``). Reads the ``Stock_Summary`` sheet of + ``reference_data/master_tracker_variables.xlsx`` and returns each + product name lowercased. + + Returns: + List of lowercased product names. + """ + return _read_stock_summary()["product"].to_list() + + +def load_product_categories() -> pl.DataFrame: + """Load the product-to-category mapping. + + Backs step 2.20 (joining each product's category) + (``add_product_categories``). Reads the ``Stock_Summary`` sheet of + ``reference_data/master_tracker_variables.xlsx`` and returns a + two-column DataFrame. + + Returns: + DataFrame with columns ``product`` and ``product_category``. + """ + return _read_stock_summary() diff --git a/src/a4d/reference/provinces.py b/src/a4d/reference/provinces.py new file mode 100644 index 0000000..2fa1694 --- /dev/null +++ b/src/a4d/reference/provinces.py @@ -0,0 +1,166 @@ +"""Province validation for patient data. + +This module loads allowed provinces from the reference_data YAML file +and provides utilities for validation. +""" + +from functools import lru_cache + +from loguru import logger + +from a4d.reference.loaders import get_reference_data_path, load_yaml + + +@lru_cache +def load_allowed_provinces() -> list[str]: + """Load all allowed provinces from YAML file (lowercased for case-insensitive matching). + + Provinces are organized by country in the YAML file. This function + flattens them into a single list and lowercases them for validation. + + The result is cached for performance since provinces don't change + during runtime. + + Returns: + List of all allowed province names (lowercased) across all countries + + Example: + >>> provinces = load_allowed_provinces() + >>> "bangkok" in provinces + True + >>> "BANGKOK" in provinces + False # List is lowercased, use is_valid_province() for validation + """ + path = get_reference_data_path("provinces", "allowed_provinces.yaml") + provinces_by_country: dict[str, list[str]] = load_yaml(path) + + # Flatten all provinces into single list and lowercase for matching + all_provinces = [] + for _, provinces in provinces_by_country.items(): + all_provinces.extend(p.lower() for p in provinces) + + logger.info(f"Loaded {len(all_provinces)} provinces from {len(provinces_by_country)} countries") + + return all_provinces + + +@lru_cache +def load_provinces_by_country() -> dict[str, list[str]]: + """Load provinces organized by country (lowercased for case-insensitive matching). + + Returns: + Dict mapping country names to lists of their provinces (lowercased) + + Example: + >>> provinces = load_provinces_by_country() + >>> "bangkok" in provinces["THAILAND"] + True + >>> len(provinces["VIETNAM"]) + 63 + """ + path = get_reference_data_path("provinces", "allowed_provinces.yaml") + provinces_by_country_raw: dict[str, list[str]] = load_yaml(path) + + # Lowercase all province names for case-insensitive matching + provinces_by_country = { + country: [p.lower() for p in provinces] + for country, provinces in provinces_by_country_raw.items() + } + + logger.info(f"Loaded provinces for {len(provinces_by_country)} countries") + + return provinces_by_country + + +@lru_cache +def load_canonical_provinces() -> list[str]: + """Load all allowed provinces with canonical casing (for validation). + + Unlike load_allowed_provinces() which lowercases for matching, + this returns the original province names from the YAML with proper + casing and accents to use as canonical values in validation. + + Returns: + List of all allowed province names (original casing) across all countries + + Example: + >>> provinces = load_canonical_provinces() + >>> "Takéo" in provinces + True + >>> "Bangkok" in provinces + True + """ + path = get_reference_data_path("provinces", "allowed_provinces.yaml") + provinces_by_country: dict[str, list[str]] = load_yaml(path) + + # Flatten all provinces into single list WITHOUT lowercasing + all_provinces = [] + for _, provinces in provinces_by_country.items(): + all_provinces.extend(provinces) + + logger.info( + f"Loaded {len(all_provinces)} canonical province names " + f"from {len(provinces_by_country)} countries" + ) + + return all_provinces + + +def is_valid_province(province: str | None) -> bool: + """Check if a province name is valid (case-insensitive). + + Args: + province: Province name to validate (case-insensitive, None allowed) + + Returns: + True if province is None or in the allowed list, False otherwise + + Example: + >>> is_valid_province("Bangkok") + True + >>> is_valid_province("BANGKOK") + True + >>> is_valid_province("bangkok") + True + >>> is_valid_province(None) + True + >>> is_valid_province("Invalid Province") + False + """ + if province is None: + return True + + allowed = load_allowed_provinces() + return province.lower() in allowed + + +def get_country_for_province(province: str) -> str | None: + """Get the country for a given province (case-insensitive). + + Args: + province: Province name (case-insensitive) + + Returns: + Country name if province is found, None otherwise + + Example: + >>> get_country_for_province("Bangkok") + 'THAILAND' + >>> get_country_for_province("bangkok") + 'THAILAND' + >>> get_country_for_province("BANGKOK") + 'THAILAND' + """ + provinces_by_country = load_provinces_by_country() + province_lower = province.lower() + + for country, provinces in provinces_by_country.items(): + if province_lower in provinces: + return country + + return None + + +if __name__ == "__main__": + for c, p in load_provinces_by_country().items(): + print(f"{c}: {p}") diff --git a/src/a4d/reference/synonyms.py b/src/a4d/reference/synonyms.py new file mode 100644 index 0000000..658f0b4 --- /dev/null +++ b/src/a4d/reference/synonyms.py @@ -0,0 +1,350 @@ +"""Column name mapper for standardizing tracker file columns. + +This module handles the mapping of various column name variants (synonyms) +to standardized column names used throughout the pipeline. +""" + +import re +from functools import lru_cache +from pathlib import Path + +import polars as pl +from loguru import logger + +from a4d.reference.loaders import get_reference_data_path, load_yaml + + +def sanitize_str(text: str) -> str: + """Sanitize a string for column name matching. + + Converts to lowercase, removes all spaces and special characters, + keeping only alphanumeric characters, so a header's punctuation, casing and + spacing cannot decide whether a column is recognised. + + Args: + text: String to sanitize + + Returns: + Sanitized string with only lowercase alphanumeric characters + + Examples: + >>> sanitize_str("Patient ID*") + 'patientid' + >>> sanitize_str("Age* On Reporting") + 'ageonreporting' + >>> sanitize_str("Date 2022") + 'date2022' + >>> sanitize_str("My Awesome 1st Column!!") + 'myawesome1stcolumn' + """ + # Convert to lowercase + text = text.lower() + # Remove spaces + text = text.replace(" ", "") + # Remove all non-alphanumeric characters + text = re.sub(r"[^a-z0-9]", "", text) + return text + + +class ColumnMapper: + """Maps synonym column names to standardized names. + + Loads column synonyms from YAML files and provides methods to rename + DataFrame columns to their standardized names. + + Example YAML structure: + age: + - Age + - Age* + - age on reporting + - Age (Years) + patient_id: + - ID + - Patient ID + - Patient ID* + + Attributes: + yaml_path: Path to the synonym YAML file + synonyms: Dict mapping standard names to lists of synonyms + _lookup: Reverse lookup dict mapping SANITIZED synonyms to standard names + + Note: + Synonym matching is case-insensitive and ignores special characters. + Both sides of the lookup are sanitized, because a synonym file written + by hand and a header typed into Excel will not agree on punctuation. + Historically this used sanitize_str() for both + column names and synonym keys before matching. + """ + + def __init__(self, yaml_path: Path): + """Initialize the mapper by loading synonyms from YAML. + + Args: + yaml_path: Path to the synonym YAML file + + Raises: + FileNotFoundError: If the YAML file doesn't exist + yaml.YAMLError: If the YAML file is malformed + """ + self.yaml_path = yaml_path + self.synonyms: dict[str, list[str]] = load_yaml(yaml_path) + + # Build reverse lookup: sanitized_synonym -> standard_name + # Sanitize both the column name and the synonym keys, so neither side's + # punctuation or casing decides the match. + self._lookup: dict[str, str] = self._build_lookup() + + logger.info( + f"Loaded {len(self.synonyms)} standard columns with " + f"{len(self._lookup)} total synonyms from {yaml_path.name}" + ) + + def _build_lookup(self) -> dict[str, str]: + """Build reverse lookup dictionary from SANITIZED synonyms to standard names. + + Sanitizes all synonym keys before adding them to the lookup. + + Returns: + Dict mapping each SANITIZED synonym to its standard column name + + Example: + >>> # YAML has: patient_id: ["Patient ID", "Patient ID*", "ID"] + >>> # Lookup will have: {"patientid": "patient_id", "id": "patient_id"} + """ + lookup = {} + for standard_name, synonym_list in self.synonyms.items(): + # Handle empty lists (columns with no synonyms) + if not synonym_list: + continue + + if isinstance(synonym_list, str): + synonym_list = [synonym_list] + + for synonym in synonym_list: + # Sanitize the synonym key before adding to lookup + sanitized_key = sanitize_str(synonym) + + if sanitized_key in lookup: + logger.bind(error_code="invalid_tracker").warning( + f"Duplicate sanitized synonym '{sanitized_key}' " + f"(from '{synonym}') found for both " + f"'{lookup[sanitized_key]}' and '{standard_name}'. " + f"Using '{standard_name}'." + ) + lookup[sanitized_key] = standard_name + + return lookup + + def get_standard_name(self, column: str) -> str: + """Get the standard name for a column. + + Sanitizes the input column name before lookup. + + Args: + column: Column name (may be a synonym, with special characters/spaces) + + Returns: + Standard column name, or original if no mapping exists + + Example: + >>> mapper.get_standard_name("Patient ID*") + 'patient_id' # "Patient ID*" → "patientid" → "patient_id" + >>> mapper.get_standard_name("Age* On Reporting") + 'age' # "Age* On Reporting" → "ageonreporting" → "age" + """ + # Sanitize input column name before lookup + sanitized_col = sanitize_str(column) + return self._lookup.get(sanitized_col, column) + + def is_known_column(self, column: str) -> bool: + """Check if column name maps to a known standard name. + + Used for validating forward-filled headers during Excel extraction. + Returns True if the column is either a known synonym or a standard name. + + Args: + column: Column name to check + + Returns: + True if column maps to a known standard name + + Example: + >>> mapper.is_known_column("Current Patient Observations Category") + True # Maps to observations_category + >>> mapper.is_known_column("Level of Support Status") + False # No such column in synonyms + """ + sanitized = sanitize_str(column) + return sanitized in self._lookup or column in self.synonyms + + def rename_columns( + self, + df: pl.DataFrame, + strict: bool = False, + ) -> pl.DataFrame: + """Rename DataFrame columns using synonym mappings. + + Args: + df: Input DataFrame with potentially non-standard column names + strict: If True, raise error if unmapped columns exist + If False, keep unmapped columns as-is + + Returns: + DataFrame with standardized column names + + Raises: + ValueError: If strict=True and unmapped columns exist + """ + # Build rename mapping for columns that need renaming + rename_map = {} + unmapped_columns = [] + + for col in df.columns: + standard_name = self.get_standard_name(col) + + if standard_name == col and col not in self.synonyms: + # Column is not in lookup and not a standard name + unmapped_columns.append(col) + elif standard_name != col: + # Column needs to be renamed + rename_map[col] = standard_name + + # Log unmapped columns + if unmapped_columns: + if strict: + raise ValueError( + f"Unmapped columns found: {unmapped_columns}. " + "These columns do not appear in the synonym file." + ) + else: + logger.bind(error_code="missing_column").warning( + f"Keeping {len(unmapped_columns)} unmapped columns as-is: {unmapped_columns}" + ) + + # Several source columns can map to one canonical name: the 2023 template + # splits complication screening into B.P./Kidney/Eye/Foot/Lipids sub-columns, + # each carrying an independent value. Merge them -- as + # merge_duplicate_columns_data() already merges repeated raw headers -- + # rather than keeping the first, which silently discarded + # 2,489 recorded values across 27 trackers. + sources_by_target: dict[str, list[str]] = {} + for source_col, target_col in rename_map.items(): + sources_by_target.setdefault(target_col, []).append(source_col) + + merge_groups = {t: cols for t, cols in sources_by_target.items() if len(cols) > 1} + + if merge_groups: + logger.bind(error_code="duplicate_source_columns").warning( + f"Merging source columns that share a target name: {merge_groups}. " + "Values are comma-joined in column order; empty cells are skipped." + ) + + for source_cols in merge_groups.values(): + parts = [ + pl.when(pl.col(col).cast(pl.String, strict=False).str.strip_chars() != "") + .then(pl.col(col).cast(pl.String, strict=False).str.strip_chars()) + .otherwise(None) + for col in source_cols + ] + merged = pl.concat_str(parts, separator=",", ignore_nulls=True) + # concat_str yields "" when every part is null; an absent value is null. + merged = pl.when(merged != "").then(merged).otherwise(None) + df = df.with_columns(merged.alias(source_cols[0])).drop(source_cols[1:]) + for col in source_cols[1:]: + del rename_map[col] + + # Log successful mappings + if rename_map: + logger.debug(f"Renaming {len(rename_map)} columns: {list(rename_map.items())}") + + return df.rename(rename_map) if rename_map else df + + def get_expected_columns(self) -> set[str]: + """Get set of all standard column names. + + Returns: + Set of standard column names defined in the synonym file + """ + return set(self.synonyms) + + def get_missing_columns(self, df: pl.DataFrame) -> set[str]: + """Get standard columns that are missing from the DataFrame. + + Args: + df: DataFrame to check + + Returns: + Set of standard column names not present in the DataFrame + """ + current_columns = set(df.columns) + expected_columns = self.get_expected_columns() + return expected_columns - current_columns + + def validate_required_columns( + self, + df: pl.DataFrame, + required: list[str], + ) -> None: + """Validate that required columns are present after renaming. + + Args: + df: DataFrame to validate + required: List of required standard column names + + Raises: + ValueError: If any required columns are missing + """ + missing = set(required) - set(df.columns) + if missing: + raise ValueError(f"Required columns missing after renaming: {missing}") + + +@lru_cache(maxsize=1) +def load_patient_mapper() -> ColumnMapper: + """Load the patient data column mapper. + + Cached so callers on the per-tracker hot path don't re-read the YAML. + Cache is per-process; ProcessPoolExecutor workers get their own. + + Returns: + ColumnMapper for patient data + + Example: + >>> mapper = load_patient_mapper() + >>> df = mapper.rename_columns(raw_df) + """ + path = get_reference_data_path("synonyms", "synonyms_patient.yaml") + return ColumnMapper(path) + + +@lru_cache(maxsize=1) +def load_product_mapper() -> ColumnMapper: + """Load the product data column mapper. + + Returns: + ColumnMapper for product data + + Example: + >>> mapper = load_product_mapper() + >>> df = mapper.rename_columns(raw_df) + """ + path = get_reference_data_path("synonyms", "synonyms_product.yaml") + return ColumnMapper(path) + + +if __name__ == "__main__": + # Example usage + patient_mapper = load_patient_mapper() + product_mapper = load_product_mapper() + + # Example DataFrame + df = pl.DataFrame( + { + "Age": [25, 30], + "Patient ID": [1, 2], + "Product Name": ["A", "B"], + } + ) + + renamed_df = patient_mapper.rename_columns(df) + print(renamed_df) diff --git a/src/a4d/state/__init__.py b/src/a4d/state/__init__.py new file mode 100644 index 0000000..2a934fc --- /dev/null +++ b/src/a4d/state/__init__.py @@ -0,0 +1,15 @@ +"""Incremental processing: skip trackers whose MD5 + completion state match a +previous run's manifest. +""" + +from a4d.state.filter import FilterSummary, filter_unchanged_trackers +from a4d.state.manifest import Manifest, ManifestEntry +from a4d.state.source import load_previous_manifest + +__all__ = [ + "FilterSummary", + "Manifest", + "ManifestEntry", + "filter_unchanged_trackers", + "load_previous_manifest", +] diff --git a/src/a4d/state/filter.py b/src/a4d/state/filter.py new file mode 100644 index 0000000..ae0167e --- /dev/null +++ b/src/a4d/state/filter.py @@ -0,0 +1,83 @@ +"""Filter discovered tracker files against the previous run's manifest. + +The pipeline currently re-processes every ``.xlsx`` under ``data_root`` on every +run. With the ``--incremental`` flag, the CLI calls +:func:`filter_unchanged_trackers` between discovery and the worker loop to drop +trackers whose bytes haven't changed AND whose previous run completed all four +output stages. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from loguru import logger + +from a4d.state.manifest import Manifest +from a4d.tables.metadata import md5_file + + +@dataclass(frozen=True) +class FilterSummary: + queued: int + skipped: int + new: int + changed: int + previously_incomplete: int + + +def filter_unchanged_trackers( + tracker_files: list[Path], + manifest: Manifest, +) -> tuple[list[Path], FilterSummary]: + """Return trackers needing reprocessing, plus a counts summary. + + A tracker is **queued** (not skipped) when any of: + + * ``(clinic_code, file_name)`` is absent from the manifest (new tracker), + * current MD5 differs from the manifest MD5 (changed tracker), or + * manifest entry has ``complete=False`` (previous run didn't finish all + four output stages — re-run to fill the gaps). + + ``clinic_code`` is the parent folder name and ``file_name`` is the path + stem, matching ``tables/metadata.py``'s schema. MD5 is computed via the + same chunked helper used by the metadata producer, so hashes are + bit-comparable. + """ + queued: list[Path] = [] + new = changed = incomplete = 0 + + for path in tracker_files: + clinic_code = path.parent.name + file_name = path.stem + entry = manifest.get(clinic_code, file_name) + + if entry is None: + queued.append(path) + new += 1 + continue + if not entry.complete: + queued.append(path) + incomplete += 1 + continue + if md5_file(path) != entry.md5: + queued.append(path) + changed += 1 + continue + # Unchanged + complete: skip. + + summary = FilterSummary( + queued=len(queued), + skipped=len(tracker_files) - len(queued), + new=new, + changed=changed, + previously_incomplete=incomplete, + ) + logger.info( + f"Incremental filter: queued {summary.queued} " + f"(new={summary.new}, changed={summary.changed}, " + f"incomplete={summary.previously_incomplete}); " + f"skipped {summary.skipped} unchanged" + ) + return queued, summary diff --git a/src/a4d/state/manifest.py b/src/a4d/state/manifest.py new file mode 100644 index 0000000..1e6f06c --- /dev/null +++ b/src/a4d/state/manifest.py @@ -0,0 +1,37 @@ +"""Previous-run manifest used by incremental processing. + +A ``Manifest`` is an in-memory snapshot of ``tracker_metadata.parquet``: one +entry per (clinic_code, file_name) pair, capturing the previous run's MD5 and +whether the run completed (all four per-tracker output presence flags were +True). The filter step compares this snapshot against the current on-disk +trackers to decide which to re-process. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +ManifestKey = tuple[str, str] +"""(clinic_code, file_name) — file_name is the path stem, matching +``tracker_metadata.parquet``'s ``file_name`` column.""" + + +@dataclass(frozen=True) +class ManifestEntry: + md5: str + complete: bool + + +@dataclass(frozen=True) +class Manifest: + entries: dict[ManifestKey, ManifestEntry] = field(default_factory=dict) + + @classmethod + def empty(cls) -> Manifest: + return cls(entries={}) + + def get(self, clinic_code: str, file_name: str) -> ManifestEntry | None: + return self.entries.get((clinic_code, file_name)) + + def __len__(self) -> int: + return len(self.entries) diff --git a/src/a4d/state/source.py b/src/a4d/state/source.py new file mode 100644 index 0000000..895bf12 --- /dev/null +++ b/src/a4d/state/source.py @@ -0,0 +1,70 @@ +"""Load the previous run's manifest from BigQuery or a local parquet. + +Source precedence: + +1. BigQuery (``select_tracker_metadata``) — authoritative when running in + Cloud Run with credentials available. +2. Local ``output_root/tables/tracker_metadata.parquet`` — fallback for + developers without ``gcloud auth`` configured, or when BQ is unreachable. +3. Empty manifest — first-ever run, every tracker queues. +""" + +from __future__ import annotations + +from pathlib import Path + +import polars as pl +from loguru import logger + +from a4d.state.manifest import Manifest, ManifestEntry + + +def _build_manifest(df: pl.DataFrame) -> Manifest: + """Convert a (file_name, clinic_code, md5, complete) frame into a Manifest.""" + entries: dict[tuple[str, str], ManifestEntry] = {} + for row in df.iter_rows(named=True): + key = (row["clinic_code"], row["file_name"]) + entries[key] = ManifestEntry(md5=row["md5"], complete=bool(row["complete"])) + return Manifest(entries=entries) + + +def load_previous_manifest( + output_root: Path, + *, + prefer_bigquery: bool = True, +) -> Manifest: + """Load the previous run's tracker manifest. + + Args: + output_root: Pipeline output root; the local fallback reads + ``output_root/tables/tracker_metadata.parquet`` from here. + prefer_bigquery: If True, try BigQuery first. Set to False to skip the + BQ lookup entirely (useful in tests or when running offline). + + Returns: + A populated ``Manifest`` on success, or ``Manifest.empty()`` when no + prior state is available. Never raises — every error path falls + through with a logged warning. + """ + if prefer_bigquery: + # Imported here so the BigQuery client isn't constructed at module + # import time. Keeps tests that patch the source module fast and + # avoids touching gcloud creds when prefer_bigquery=False. + from a4d.gcp.bigquery import select_tracker_metadata + + df = select_tracker_metadata() + if df is not None and df.height > 0: + logger.info(f"Loaded manifest from BigQuery ({df.height} rows)") + return _build_manifest(df) + + local_parquet = output_root / "tables" / "tracker_metadata.parquet" + if local_parquet.exists(): + try: + df = pl.read_parquet(local_parquet) + logger.info(f"Loaded manifest from local parquet ({df.height} rows): {local_parquet}") + return _build_manifest(df) + except Exception as e: + logger.warning(f"Failed to read local manifest {local_parquet}: {e}") + + logger.info("No previous manifest available; treating every tracker as new") + return Manifest.empty() diff --git a/src/a4d/tables/__init__.py b/src/a4d/tables/__init__.py new file mode 100644 index 0000000..af9d4f9 --- /dev/null +++ b/src/a4d/tables/__init__.py @@ -0,0 +1 @@ +"""Table creation module for final output tables.""" diff --git a/src/a4d/tables/clinic.py b/src/a4d/tables/clinic.py new file mode 100644 index 0000000..a8af78e --- /dev/null +++ b/src/a4d/tables/clinic.py @@ -0,0 +1,78 @@ +"""Create clinic static data table from reference data. + +Reads clinic_data.xlsx, fills down hierarchical columns, exports as parquet. +""" + +import warnings +from pathlib import Path + +import polars as pl +from loguru import logger + +from a4d.reference.loaders import find_reference_data_dir + +# polars' own calamine+pyarrow "eager sheet load" path in read_excel() calls +# from_arrow() on an ArrowStreamExportable without disambiguating DataFrame +# vs Series, and warns about its own internal call — nothing in our +# read_excel() arguments can influence it. Upstream issue, not ours to fix; +# narrowly suppress rather than let it flood every run's console output. +warnings.filterwarnings( + "ignore", + message=r"from_arrow\(\) will return a Series", + category=FutureWarning, +) + +# Text columns filled downward. The source sheet merges these cells, so only +# the first row of each country/province block carries a value. +_FILL_COLUMNS = [ + "country", + "clinic_province", + "clinic_name", + "clinic_status", + "clinic_id", + "country_code", + "clinic_code", + "patient_id_example", +] + + +def create_table_clinic_static(output_dir: Path) -> Path: + """Create clinic static data table from reference data. + + Reads clinic_data.xlsx from reference_data/, fills hierarchical columns + downward, and writes parquet. + + Args: + output_dir: Directory to write the parquet file + + Returns: + Path to created clinic_data_static.parquet + """ + reference_dir = find_reference_data_dir() + clinic_file = reference_dir / "clinic_data.xlsx" + + if not clinic_file.exists(): + raise FileNotFoundError(f"Clinic data file not found: {clinic_file}") + + logger.info(f"Reading clinic data from: {clinic_file}") + + df = pl.read_excel(clinic_file, sheet_id=1) + + # Drop the sheet's unnamed index column + unnamed_cols = [c for c in df.columns if c.startswith("__UNNAMED")] + if unnamed_cols: + df = df.drop(unnamed_cols) + + # Fill nulls downward for the merged hierarchical columns + fill_cols = [c for c in _FILL_COLUMNS if c in df.columns] + if fill_cols: + df = df.with_columns([pl.col(c).forward_fill() for c in fill_cols]) + + logger.info(f"Clinic static data: {df.shape[0]} rows, {df.shape[1]} columns") + + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / "clinic_data_static.parquet" + df.write_parquet(output_file) + + logger.info(f"Clinic static table saved: {output_file}") + return output_file diff --git a/src/a4d/tables/errors.py b/src/a4d/tables/errors.py new file mode 100644 index 0000000..108cf37 --- /dev/null +++ b/src/a4d/tables/errors.py @@ -0,0 +1,70 @@ +"""Create errors table from data quality errors collected during pipeline processing. + +Aggregates all DataError records from tracker results into a single parquet file +for BigQuery upload and dashboard analysis. +""" + +from pathlib import Path + +import polars as pl +from loguru import logger + +from a4d.errors import DataError + + +def create_table_errors(data_errors: list[DataError], output_dir: Path) -> Path: + """Create errors table from all data quality errors across all trackers. + + Args: + data_errors: All DataError records collected during pipeline processing + output_dir: Directory to write the errors table parquet + + Returns: + Path to created errors table parquet file + + Example: + >>> errors_path = create_table_errors(all_errors, Path("output/tables")) + >>> errors_path + Path('output/tables/table_errors.parquet') + """ + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / "table_errors.parquet" + + if not data_errors: + logger.info("No data quality errors to write, creating empty errors table") + pl.DataFrame( + schema={ + "file_name": pl.Utf8, + "patient_id": pl.Utf8, + "column": pl.Utf8, + "original_value": pl.Utf8, + "error_message": pl.Utf8, + "error_code": pl.Categorical, + "script": pl.Categorical, + "function_name": pl.Categorical, + "timestamp": pl.Datetime, + } + ).write_parquet(output_file) + return output_file + + records = [e.model_dump() for e in data_errors] + df = ( + pl.DataFrame(records) + .with_columns( + pl.col("error_code").cast(pl.Categorical), + pl.col("script").cast(pl.Categorical), + pl.col("function_name").cast(pl.Categorical), + ) + .sort("timestamp") + ) + + df.write_parquet(output_file) + + logger.info(f"Errors table saved: {output_file} ({len(df):,} records)") + + breakdown = ( + df.group_by("error_code").agg(pl.len().alias("count")).sort("count", descending=True) + ) + logger.info(f"Error breakdown: {breakdown.to_dict(as_series=False)}") + + return output_file diff --git a/src/a4d/tables/logs.py b/src/a4d/tables/logs.py new file mode 100644 index 0000000..e5de419 --- /dev/null +++ b/src/a4d/tables/logs.py @@ -0,0 +1,237 @@ +"""Create logs table from pipeline execution logs. + +This module reads all JSON-formatted log files created by the pipeline +and creates a structured table for BigQuery upload and dashboard analysis. + +Log files are created by loguru with serialize=True, producing JSON lines format. +Each line contains structured data about pipeline execution: timestamps, levels, +messages, source locations, exceptions, and custom context fields. +""" + +import json +from pathlib import Path + +import polars as pl +from loguru import logger + + +def parse_log_file(log_file: Path) -> pl.DataFrame: + """Parse a single JSON lines log file into a DataFrame. + + Args: + log_file: Path to .log file (JSON lines format from loguru) + + Returns: + DataFrame with parsed log records, or empty DataFrame if file is invalid + + Example: + >>> df = parse_log_file(Path("output/logs/2024_Penang_patient.log")) + >>> df.columns + ['timestamp', 'level', 'message', 'log_file', ...] + """ + records = [] + + try: + with open(log_file) as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + + try: + log_entry = json.loads(line) + record_data = log_entry.get("record", {}) + + # Extract timestamp + time_data = record_data.get("time", {}) + timestamp = time_data.get("timestamp") + + # Extract level + level_data = record_data.get("level", {}) + level = level_data.get("name", "UNKNOWN") + + # Extract message + message = record_data.get("message", "") + + # Extract source location + file_data = record_data.get("file", {}) + source_file = file_data.get("name", "") + source_path = file_data.get("path", "") + + function = record_data.get("function", "") + line = record_data.get("line", 0) + module = record_data.get("module", "") + + # Extract context fields (file_name, tracker_year, tracker_month, error_code) + extra = record_data.get("extra", {}) + file_name = extra.get("file_name") + tracker_year = extra.get("tracker_year") + tracker_month = extra.get("tracker_month") + error_code = extra.get("error_code") + + # Extract process info (useful for debugging parallel processing) + process_data = record_data.get("process", {}) + process_name = process_data.get("name", "") + + # Extract exception info if present + exception = record_data.get("exception") + has_exception = exception is not None + exception_type = None + exception_value = None + + if has_exception and exception: + exception_type = exception.get("type") + exception_value = exception.get("value") + + # Create record + records.append( + { + "timestamp": timestamp, + "level": level, + "message": message, + "error_code": error_code, + "log_file": log_file.name, + "file_name": file_name, + "tracker_year": tracker_year, + "tracker_month": tracker_month, + "source_file": source_file, + "source_path": source_path, + "function": function, + "line": line, + "module": module, + "process_name": process_name, + "has_exception": has_exception, + "exception_type": exception_type, + "exception_value": exception_value, + } + ) + + except json.JSONDecodeError as e: + logger.warning(f"Failed to parse JSON in {log_file.name}:{line_num}: {e}") + continue + except Exception as e: + logger.warning(f"Error processing line {line_num} in {log_file.name}: {e}") + continue + + except Exception as e: + logger.error(f"Failed to read log file {log_file.name}: {e}") + return pl.DataFrame() + + if not records: + return pl.DataFrame() + + # Create DataFrame with proper types. schema_overrides pins the dtype of + # nullable columns so a log file whose first ~100 rows happen to have + # error_code=None / no exception / no file_name doesn't infer pl.Null + # and fail when later rows append actual strings or when concat'd with + # log files where these columns *are* populated. + df = pl.DataFrame( + records, + schema_overrides={ + "error_code": pl.Utf8, + "file_name": pl.Utf8, + "tracker_year": pl.Int32, + "tracker_month": pl.Int32, + "exception_type": pl.Utf8, + "exception_value": pl.Utf8, + }, + ) + + # Cast categorical columns for efficiency + df = df.with_columns( + [ + pl.col("level").cast(pl.Categorical), + pl.col("log_file").cast(pl.Categorical), + pl.col("source_file").cast(pl.Categorical), + pl.col("function").cast(pl.Categorical), + pl.col("module").cast(pl.Categorical), + pl.col("process_name").cast(pl.Categorical), + ] + ) + + return df + + +def create_table_logs(logs_dir: Path, output_dir: Path) -> Path: + """Create logs table from pipeline log files. + + Reads all .log files from the logs directory, parses JSON lines, + and creates a structured table for BigQuery upload. + + Args: + logs_dir: Directory containing .log files (e.g., output/logs/) + output_dir: Directory to write the logs table parquet + + Returns: + Path to created logs table parquet file + + Example: + >>> logs_path = create_table_logs( + ... Path("output/logs"), + ... Path("output/tables") + ... ) + >>> logs_path + Path('output/tables/table_logs.parquet') + """ + logger.info(f"Creating logs table from: {logs_dir}") + + # Find all .log files (exclude .zip compressed files) + log_files = sorted(logs_dir.glob("*.log")) + logger.info(f"Found {len(log_files)} log files to process") + + if not log_files: + logger.warning("No log files found, creating empty logs table") + # Create empty DataFrame with correct schema + empty_df = pl.DataFrame( + schema={ + "timestamp": pl.Datetime, + "level": pl.Categorical, + "message": pl.Utf8, + "error_code": pl.Utf8, + "log_file": pl.Categorical, + "file_name": pl.Utf8, + "tracker_year": pl.Int32, + "tracker_month": pl.Int32, + "source_file": pl.Categorical, + "source_path": pl.Utf8, + "function": pl.Categorical, + "line": pl.Int32, + "module": pl.Categorical, + "process_name": pl.Categorical, + "has_exception": pl.Boolean, + "exception_type": pl.Utf8, + "exception_value": pl.Utf8, + } + ) + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / "table_logs.parquet" + empty_df.write_parquet(output_file) + return output_file + + # Parse all log files + all_logs = [] + for log_file in log_files: + logger.debug(f"Parsing: {log_file.name}") + df = parse_log_file(log_file) + if len(df) > 0: + all_logs.append(df) + + logs_table = pl.concat(all_logs, how="vertical") + + # Sort by timestamp for chronological analysis + logs_table = logs_table.sort("timestamp") + + logger.info(f"Created logs table with {len(logs_table)} records") + logger.info(f"Date range: {logs_table['timestamp'].min()} to {logs_table['timestamp'].max()}") + + # Log summary by level + level_counts = logs_table.group_by("level").agg(pl.len()).sort("level") + logger.info(f"Log level distribution: {level_counts.to_dict(as_series=False)}") + + # Write to parquet + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / "table_logs.parquet" + logs_table.write_parquet(output_file) + + logger.info(f"Logs table saved: {output_file}") + logger.info(f"Table size: {output_file.stat().st_size / 1024 / 1024:.2f} MB") + + return output_file diff --git a/src/a4d/tables/metadata.py b/src/a4d/tables/metadata.py new file mode 100644 index 0000000..593b87a --- /dev/null +++ b/src/a4d/tables/metadata.py @@ -0,0 +1,106 @@ +"""Tracker metadata table generator. + +For each ``.xlsx`` tracker under ``data_root``, emits a row with an MD5 hash +and presence flags for the +four per-tracker output subdirs (``patient_data_{raw,cleaned}`` and +``product_data_{raw,cleaned}``). + +The MD5 helper :func:`md5_file` is also imported by ``a4d.state.filter`` to +hash current trackers when filtering against the previous run's manifest. +""" + +from __future__ import annotations + +import hashlib +from datetime import UTC, datetime +from pathlib import Path + +import polars as pl +from loguru import logger + +# Column order is part of the published table's contract; BigQuery consumers +# may depend on it. +_SUBDIRS: tuple[str, ...] = ( + "patient_data_cleaned", + "patient_data_raw", + "product_data_cleaned", + "product_data_raw", +) + +_SCHEMA: dict[str, type[pl.DataType] | pl.DataType] = { + "file_name": pl.String, + "clinic_code": pl.String, + "md5": pl.String, + "patient_data_cleaned": pl.Boolean, + "patient_data_raw": pl.Boolean, + "product_data_cleaned": pl.Boolean, + "product_data_raw": pl.Boolean, + "complete": pl.Boolean, + "timestamp": pl.Datetime("us"), +} + + +def md5_file(path: Path, chunk_size: int = 65536) -> str: + """Stream-hash a file with MD5. ``usedforsecurity=False`` satisfies FIPS.""" + h = hashlib.md5(usedforsecurity=False) + with path.open("rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + h.update(chunk) + return h.hexdigest() + + +def create_table_tracker_metadata( + data_root: Path, + output_root: Path, +) -> Path: + """Emit ``tracker_metadata.parquet`` to ``output_root/tables/``. + + Args: + data_root: Directory holding tracker ``.xlsx`` files. Scanned recursively + so clinic-folder layouts are supported. + output_root: Pipeline output root. Per-tracker output presence is looked + up in its four subdirs (``_SUBDIRS``); the metadata parquet is + written to ``output_root/tables/``. + + Returns: + Path to the written parquet file. + """ + tracker_files = sorted(data_root.rglob("*.xlsx")) + logger.info(f"Building tracker metadata for {len(tracker_files)} tracker(s)") + + # Index each subdir once so per-tracker lookups are O(1) prefix checks. + subdir_names: dict[str, list[str]] = {} + for subdir in _SUBDIRS: + target = output_root / subdir + subdir_names[subdir] = ( + [p.name for p in target.iterdir() if p.is_file()] if target.exists() else [] + ) + + now = datetime.now(tz=UTC).replace(tzinfo=None) + rows: list[dict] = [] + for tracker_path in tracker_files: + # Stripped (ticket 36) so this table's file_name joins the cleaned + # outputs' own file_name, which both arms now trim. One real tracker + # is named "... - final .xlsx", trailing space included. + file_name = tracker_path.stem.strip() + row = { + "file_name": file_name, + "clinic_code": tracker_path.parent.name, + "md5": md5_file(tracker_path), + } + for subdir in _SUBDIRS: + # Output parquet names still carry the untrimmed stem, so the + # trimmed name remains a prefix of them. + row[subdir] = any(name.startswith(file_name) for name in subdir_names[subdir]) + row["complete"] = all(row[s] for s in _SUBDIRS) + row["timestamp"] = now + rows.append(row) + + df = pl.DataFrame(rows, schema=_SCHEMA) + + output_dir = output_root / "tables" + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / "tracker_metadata.parquet" + df.write_parquet(output_file) + logger.info(f"Tracker metadata saved: {output_file} ({df.height} rows)") + return output_file diff --git a/src/a4d/tables/patient.py b/src/a4d/tables/patient.py new file mode 100644 index 0000000..3697967 --- /dev/null +++ b/src/a4d/tables/patient.py @@ -0,0 +1,207 @@ +"""Create final patient data tables from cleaned data.""" + +from pathlib import Path + +import polars as pl +from loguru import logger + + +def read_cleaned_patient_data(cleaned_files: list[Path]) -> pl.DataFrame: + """Read and combine all cleaned patient data files. + + Args: + cleaned_files: List of paths to cleaned parquet files + + Returns: + Combined DataFrame with all cleaned patient data + """ + if not cleaned_files: + raise ValueError("No cleaned files provided") + + dfs = [pl.read_parquet(file) for file in cleaned_files] + return pl.concat(dfs, how="vertical") + + +def create_table_patient_data_static(patient_data: pl.DataFrame, output_dir: Path) -> Path: + """Create static patient data table. + + Selects static columns (data that doesn't change monthly) from the already-loaded + cleaned patient dataframe. Groups by patient_id and takes the latest available + data (latest year and month). + + Args: + patient_data: Combined cleaned patient dataframe (from read_cleaned_patient_data) + output_dir: Directory to save output parquet file + + Returns: + Path to created parquet file + """ + static_columns = [ + "clinic_id", + "dob", + "fbg_baseline_mg", + "fbg_baseline_mmol", + "file_name", + "hba1c_baseline", + "hba1c_baseline_exceeds", + "lost_date", + "name", + "patient_consent", + "patient_id", + "province", + "recruitment_date", + "sex", + "status_out", + "t1d_diagnosis_age", + "t1d_diagnosis_date", + "t1d_diagnosis_with_dka", + "tracker_date", + "tracker_month", + "tracker_year", + ] + + static_data = ( + patient_data.select(static_columns) + .sort(["patient_id", "tracker_year", "tracker_month"]) + .group_by("patient_id") + .last() + .sort(["tracker_year", "tracker_month", "patient_id"]) + ) + + logger.info(f"Static patient data dimensions: {static_data.shape}") + + output_file = output_dir / "patient_data_static.parquet" + output_dir.mkdir(parents=True, exist_ok=True) + static_data.write_parquet(output_file) + + return output_file + + +def create_table_patient_data_monthly(patient_data: pl.DataFrame, output_dir: Path) -> Path: + """Create monthly patient data table. + + Selects dynamic monthly columns from the already-loaded cleaned patient + dataframe. Keeps all monthly records. + + Args: + patient_data: Combined cleaned patient dataframe (from read_cleaned_patient_data) + output_dir: Directory to save output parquet file + + Returns: + Path to created parquet file + """ + monthly_columns = [ + "age", + "bmi", + "bmi_date", + "clinic_id", + "fbg_updated_date", + "fbg_updated_mg", + "fbg_updated_mmol", + "file_name", + "hba1c_updated", + "hba1c_updated_exceeds", + "hba1c_updated_date", + "height", + "hospitalisation_cause", + "hospitalisation_date", + "insulin_injections", + "insulin_regimen", + "insulin_total_units", + "insulin_type", + "insulin_subtype", + "last_clinic_visit_date", + "last_remote_followup_date", + "observations", + "observations_category", + "patient_id", + "sheet_name", + "status", + "support_level", + "testing_frequency", + "tracker_date", + "tracker_month", + "tracker_year", + "weight", + ] + + monthly_data = patient_data.select(monthly_columns).sort( + ["tracker_year", "tracker_month", "patient_id"] + ) + + logger.info(f"Monthly patient data dimensions: {monthly_data.shape}") + + output_file = output_dir / "patient_data_monthly.parquet" + output_dir.mkdir(parents=True, exist_ok=True) + monthly_data.write_parquet(output_file) + + return output_file + + +def create_table_patient_data_annual(patient_data: pl.DataFrame, output_dir: Path) -> Path: + """Create annual patient data table. + + Selects annual columns (data collected once per year) from the already-loaded + cleaned patient dataframe. Groups by patient_id and tracker_year, taking the + latest month for each year. Only includes data from 2024 onwards. + + Args: + patient_data: Combined cleaned patient dataframe (from read_cleaned_patient_data) + output_dir: Directory to save output parquet file + + Returns: + Path to created parquet file + """ + annual_columns = [ + "patient_id", + "status", + "edu_occ", + "edu_occ_updated", + "blood_pressure_updated", + "blood_pressure_sys_mmhg", + "blood_pressure_dias_mmhg", + "complication_screening_kidney_test_date", + "complication_screening_kidney_test_value", + "complication_screening_eye_exam_date", + "complication_screening_eye_exam_value", + "complication_screening_foot_exam_date", + "complication_screening_foot_exam_value", + "complication_screening_lipid_profile_date", + "complication_screening_lipid_profile_triglycerides_value", + "complication_screening_lipid_profile_cholesterol_value", + "complication_screening_lipid_profile_ldl_mg_value", + "complication_screening_lipid_profile_ldl_mmol_value", + "complication_screening_lipid_profile_hdl_mg_value", + "complication_screening_lipid_profile_hdl_mmol_value", + "complication_screening_thyroid_test_date", + "complication_screening_thyroid_test_ft4_ng_value", + "complication_screening_thyroid_test_ft4_pmol_value", + "complication_screening_thyroid_test_tsh_value", + "complication_screening_remarks", + "dm_complication_eye", + "dm_complication_kidney", + "dm_complication_others", + "dm_complication_remarks", + "family_history", + "other_issues", + "tracker_date", + "tracker_month", + "tracker_year", + ] + + annual_data = ( + patient_data.select(annual_columns) + .filter(pl.col("tracker_year") >= 2024) + .sort(["patient_id", "tracker_year", "tracker_month"]) + .group_by(["patient_id", "tracker_year"]) + .last() + .sort(["tracker_year", "tracker_month", "patient_id"]) + ) + + logger.info(f"Annual patient data dimensions: {annual_data.shape}") + + output_file = output_dir / "patient_data_annual.parquet" + output_dir.mkdir(parents=True, exist_ok=True) + annual_data.write_parquet(output_file) + + return output_file diff --git a/src/a4d/tables/product.py b/src/a4d/tables/product.py new file mode 100644 index 0000000..343ebb5 --- /dev/null +++ b/src/a4d/tables/product.py @@ -0,0 +1,210 @@ +"""Product table creation. + +Merges all cleaned product parquets into a single ``product_data`` table with +the canonical 20-column schema. +""" + +from pathlib import Path + +import polars as pl +from loguru import logger + +from a4d.clean.converters import safe_convert_column +from a4d.clean.schema_product import apply_schema, get_product_data_schema +from a4d.clean.validators import fix_patient_id +from a4d.errors import ErrorCollector + + +def read_cleaned_product_data(cleaned_files: list[Path]) -> pl.DataFrame: + """Step 3.1 — read and diagonally concatenate cleaned product parquets. + + Uses ``pl.concat(..., how="diagonal")`` so trackers with missing columns + are filled with nulls rather than dropped. + + Args: + cleaned_files: List of cleaned product parquet paths. + + Returns: + Single DataFrame with the union of all tracker columns. + """ + if not cleaned_files: + raise ValueError("No cleaned product files provided") + + dfs = [pl.read_parquet(file) for file in cleaned_files] + return pl.concat(dfs, how="diagonal") + + +def create_table_product_data(cleaned_files: list[Path], output_dir: Path) -> Path: + """Build the final ``product_data`` table from cleaned tracker parquets. + + 1. Concatenate every cleaned product parquet. + 2. Preserve ``product_released_to`` as ``orig_product_released_to`` and + normalise ``product_released_to`` via ``fix_patient_id``. + 3. Apply the 20-column product schema with type enforcement; values that + fail conversion are replaced by the error sentinels from ``settings``. + + Args: + cleaned_files: Cleaned product parquet files from Script 2. + output_dir: Directory where ``product_data.parquet`` is written. + + Returns: + Path to the written ``product_data.parquet`` file. + """ + df = read_cleaned_product_data(cleaned_files) + + df = df.with_columns(pl.col("product_released_to").alias("orig_product_released_to")) + + error_collector = ErrorCollector() + df = fix_patient_id( + df=df, + error_collector=error_collector, + patient_id_col="product_released_to", + ) + + # Adds any missing schema columns as typed nulls so the cast loop below + # has every target column available; safe_convert_column preserves order. + df = apply_schema(df) + + schema = get_product_data_schema() + for col, dtype in schema.items(): + if dtype in (pl.Int32, pl.Int64, pl.Float32, pl.Float64, pl.Date): + df = safe_convert_column( + df=df, + column=col, + target_type=dtype, + error_collector=error_collector, + patient_id_col="product_released_to", + ) + + # Surface any errors collected by fix_patient_id / safe_convert_column at + # the table-aggregation stage. These run outside any per-tracker file_logger, + # so without this loop the errors disappear silently. Full per-row detail + # goes to DEBUG only — with thousands of truncated/invalid IDs across a + # real run, logging each at WARNING flooded the console (same issue as + # link_product_patient's per-pair loop). The console gets one aggregate + # line per (column, error_message) group instead. + for err in error_collector.errors: + logger.bind( + error_code=err.error_code, + script="script3", + function_name=err.function_name, + file_name=err.file_name, + column=err.column, + ).debug( + f"Table-stage error: {err.error_message} " + f"(file={err.file_name}, column={err.column}, value={err.original_value})" + ) + if error_collector.errors: + group_counts: dict[tuple[str, str], int] = {} + for err in error_collector.errors: + key = (err.column, err.error_message) + group_counts[key] = group_counts.get(key, 0) + 1 + for (column, message), count in sorted(group_counts.items(), key=lambda kv: -kv[1]): + logger.warning(f"Table-stage errors: {count}x '{message}' (column={column})") + logger.info(f"Table-stage errors: {len(error_collector.errors)} total") + + logger.info(f"Product data table dimensions: {df.shape}") + + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / "product_data.parquet" + df.write_parquet(output_path) + + return output_path + + +def link_product_patient( + product_df: pl.DataFrame, + patient_table_path: Path, +) -> int: + """Validate product_released_to against a per-file patient table. + + LEFT-joins product rows onto the patient table on + ``(file_name, product_released_to ↔ patient_id)`` and logs each + ``(file_name, product_released_to)`` pair that has no patient match. + Logging-only — does not modify either table and never raises. + + ``patient_table_path`` must have one row per ``(file_name, patient_id)`` + pair actually present in each tracker file — i.e. ``patient_data_monthly``. + **Not** + ``patient_data_static``: that table collapses each patient down to a + single latest record/file, so joining against it only covers each + patient's most recent tracker file and misreports every product row tied + to an earlier file as unmatched (verified against real data: an 88% + mismatch rate against `static` dropped to 1.4% against `monthly`). + + Rows where ``product_released_to`` is null or equals the + ``error_val_character`` sentinel ("Undefined") are filtered out + before joining. Those are not real patient IDs and would flood the + log with non-issues. + + Args: + product_df: Product table (post-``fix_patient_id``). + patient_table_path: Path to a per-file patient table, normally + ``patient_data_monthly.parquet``. + + Returns: + Total count of mismatched product rows for telemetry/test use. + """ + from a4d.config import settings + + if not patient_table_path.exists(): + logger.warning( + f"Patient table not available at {patient_table_path}; " + "skipping product-patient link validation." + ) + return 0 + + # .unique() guards against join fan-out if the patient table has more + # than one row per (file_name, patient_id) — e.g. patient_data_monthly + # has one row per month, so the same patient/file pair repeats. + patient_keys = ( + pl.read_parquet(patient_table_path) + .select(["file_name", "patient_id"]) + .unique() + .with_columns(pl.lit(True).alias("_patient_matched")) + ) + + sentinel = settings.error_val_character + candidates = product_df.filter( + pl.col("product_released_to").is_not_null() & (pl.col("product_released_to") != sentinel) + ) + + joined = candidates.join( + patient_keys, + left_on=["file_name", "product_released_to"], + right_on=["file_name", "patient_id"], + how="left", + ) + mismatches = joined.filter(pl.col("_patient_matched").is_null()) + + mismatch_groups = ( + mismatches.group_by(["file_name", "product_released_to"]) + .agg(pl.len().alias("count")) + .sort("count", descending=True) + ) + + total_mismatched_rows = mismatches.height + distinct_pairs = mismatch_groups.height + total_examined = candidates.height + + # Full per-pair detail goes to the file/BigQuery logs only (DEBUG) — with + # hundreds of distinct mismatched pairs across a real run, logging each at + # WARNING flooded the console. The aggregate line below is the console- + # visible signal; drill into per-pair detail via the logs table. + for row in mismatch_groups.iter_rows(named=True): + logger.debug( + f"Unmatched product_released_to: file_name='{row['file_name']}' " + f"patient_id='{row['product_released_to']}' count={row['count']}" + ) + + summary = ( + f"Product-patient link validation: {total_mismatched_rows} mismatched rows, " + f"{distinct_pairs} distinct (file × id) pairs, " + f"{total_examined} candidate product rows examined." + ) + if total_mismatched_rows > 0: + logger.warning(summary) + else: + logger.info(summary) + + return total_mismatched_rows diff --git a/src/a4d/utils/__init__.py b/src/a4d/utils/__init__.py new file mode 100644 index 0000000..12455b7 --- /dev/null +++ b/src/a4d/utils/__init__.py @@ -0,0 +1,3 @@ +"""Utility modules.""" + +__all__ = [] diff --git a/src/a4d/validate/__init__.py b/src/a4d/validate/__init__.py new file mode 100644 index 0000000..c731706 --- /dev/null +++ b/src/a4d/validate/__init__.py @@ -0,0 +1,6 @@ +"""Post-pipeline validation that compares cleaned output against raw source. + +Used by ``scripts/validate_source_vs_output.py``. The validator does not import +or modify either golden cleaning module (clean/patient.py, clean/product.py); +it replicates only the small slices of logic it needs. +""" diff --git a/src/a4d/validate/common.py b/src/a4d/validate/common.py new file mode 100644 index 0000000..7367510 --- /dev/null +++ b/src/a4d/validate/common.py @@ -0,0 +1,116 @@ +"""Shared helpers for the source-vs-output validators.""" + +from __future__ import annotations + +from typing import Any + +import polars as pl + +from a4d.clean.converters import safe_convert_column +from a4d.config import settings +from a4d.errors import ErrorCode, ErrorCollector + +# Mirrors clean/patient.py:241-256. Hyphen->underscore happens first, then we +# extract the leading "LETTERS_NON-UNDERSCORE-CHARS" group. Single-token IDs +# (no underscore) pass through unchanged. +_NORMALIZE_REGEX = r"^([A-Z]+_[^_]+)" + + +def normalize_patient_id(col: pl.Expr) -> pl.Expr: + """Replicate the patient_id normalization in clean/patient.py:241-256. + + Returns an expression — caller wraps it in ``with_columns``. + """ + hyphen_to_underscore = col.str.replace_all("-", "_") + return ( + pl.when(hyphen_to_underscore.str.contains("_")) + .then(hyphen_to_underscore.str.extract(_NORMALIZE_REGEX, 1)) + .otherwise(hyphen_to_underscore) + ) + + +def safe_parse_series( + raw_col: pl.Series, + target_type: pl.DataType | type[pl.DataType], +) -> pl.Series: + """Re-parse a raw string Series to ``target_type``, discarding parse errors. + + ``safe_convert_column`` writes to its caller's ErrorCollector with code + ``type_conversion``. We pass a sacrificial collector here so those entries + don't pollute the validator's findings collector. + """ + name = raw_col.name + sacrificial = ErrorCollector() + df = pl.DataFrame({name: raw_col}) + parsed = safe_convert_column( + df=df, + column=name, + target_type=target_type, + error_collector=sacrificial, + # safe_convert_column reads file_name/patient_id for error attribution; we + # don't care about the entries it generates, but the columns must exist. + file_name_col="__unused_file__", + patient_id_col="__unused_pid__", + ) + series = parsed[name] + # safe_convert_column writes settings.error_val_numeric / error_val_character / + # error_val_date for parse failures; map those sentinels back to null so the + # validator can cleanly distinguish "parsed" from "failed to parse". + if series.dtype.is_numeric(): + return series.replace({settings.error_val_numeric: None}) + if series.dtype in (pl.Utf8, pl.String): + return series.replace({settings.error_val_character: None}) + if series.dtype == pl.Date: + sentinel_date = pl.Series("_s", [settings.error_val_date]).str.to_date()[0] + return series.replace({sentinel_date: None}) + return series + + +def emit_finding( + collector: ErrorCollector, + *, + file_name: str, + patient_id: str, + column: str, + original_value: Any, + error_message: str, + error_code: ErrorCode, + function_name: str, +) -> None: + """Thin wrapper around ``ErrorCollector.add_error`` enforcing the schema. + + The collector accepts only the ErrorCode literal types — validator codes + that don't fit (MISSING_ROW, VALUE_SHIFT, ...) are encoded in + ``error_message`` and the underlying ``error_code`` is set to the closest + existing literal. Caller must pass one of: ``"missing_value"`` for + missing/phantom rows, ``"invalid_value"`` for shifts/range violations, + ``"type_conversion"`` for parse-driven nulls. + """ + collector.add_error( + file_name=file_name, + patient_id=patient_id, + column=column, + original_value="" if original_value is None else str(original_value), + error_message=error_message, + error_code=error_code, + function_name=function_name, + script="validate", + ) + + +def is_close( + a: float | None, b: float | None, *, abs_tol: float = 1e-6, rel_tol: float = 1e-4 +) -> bool: + """Null-aware float comparison with abs and relative tolerance. + + Both null -> equal. Either-side null -> not equal. + """ + if a is None and b is None: + return True + if a is None or b is None: + return False + diff = abs(a - b) + if diff <= abs_tol: + return True + scale = max(abs(a), abs(b)) + return diff <= rel_tol * scale diff --git a/src/a4d/validate/source_vs_output_patient.py b/src/a4d/validate/source_vs_output_patient.py new file mode 100644 index 0000000..fe94a30 --- /dev/null +++ b/src/a4d/validate/source_vs_output_patient.py @@ -0,0 +1,292 @@ +"""Source-vs-output validation for the patient pipeline. + +See ``C:/Users/furin/.claude/plans/output-vs-source-validation-swirling-nygaard.md`` +for the design rationale and the catalog of checks. +""" + +from __future__ import annotations + +from pathlib import Path + +import polars as pl +from loguru import logger + +from a4d.clean.validators import load_numeric_ranges +from a4d.errors import ErrorCollector +from a4d.validate.common import ( + emit_finding, + is_close, + normalize_patient_id, + safe_parse_series, +) + +# Columns whose values the cleaner re-derives or transforms. Excluded from +# VALUE_SHIFT comparisons. +DERIVED_COLUMNS = frozenset({"age", "bmi", "t1d_diagnosis_age"}) + +# Columns the cleaner converts (height: cm->m; FBG: mmol<->mg). Excluded from +# VALUE_SHIFT in v1; promoted to a follow-up. +UNIT_CONVERTED_COLUMNS = frozenset({"height", "fbg_updated_mg", "fbg_updated_mmol"}) + +# Join key for row-level checks. Mirrors patient_data_monthly's natural grain. +ROW_KEY = ["clinic_id", "patient_id", "tracker_year", "tracker_month"] + + +def _load_raw(run_dir: Path) -> pl.DataFrame | None: + """Concatenate all per-tracker raw patient parquets in the run directory.""" + raw_dir = run_dir / "patient_data_raw" + if not raw_dir.exists(): + return None + files = sorted(raw_dir.glob("*_patient_raw.parquet")) + if not files: + return None + frames = [pl.read_parquet(f) for f in files] + return pl.concat(frames, how="diagonal") + + +def _load_cleaned_monthly(run_dir: Path) -> pl.DataFrame | None: + path = run_dir / "tables" / "patient_data_monthly.parquet" + if not path.exists(): + return None + return pl.read_parquet(path) + + +def _normalize_join_keys(df: pl.DataFrame) -> pl.DataFrame: + """Apply patient_id normalization and cast year/month for clean joins.""" + out = df.with_columns(normalize_patient_id(pl.col("patient_id")).alias("patient_id")) + return out.with_columns( + [ + pl.col("tracker_year").cast(pl.Int64, strict=False), + pl.col("tracker_month").cast(pl.Int64, strict=False), + ] + ) + + +def check_missing_patients( + raw: pl.DataFrame, cleaned: pl.DataFrame, collector: ErrorCollector +) -> None: + """MISSING_ROW + PHANTOM_ROW: anti-joins on the normalized row key.""" + raw_keys = _normalize_join_keys(raw).select(ROW_KEY).unique() + cleaned_keys = _normalize_join_keys(cleaned).select(ROW_KEY).unique() + + missing = raw_keys.join(cleaned_keys, on=ROW_KEY, how="anti") + for row in missing.iter_rows(named=True): + emit_finding( + collector, + file_name="", + patient_id=row["patient_id"] or "unknown", + column="__row__", + original_value=f"{row['clinic_id']}|{row['tracker_year']}|{row['tracker_month']}", + error_message=( + f"MISSING_ROW: raw row for {row['patient_id']} " + f"({row['tracker_year']}-{row['tracker_month']:02d}) " + f"not found in cleaned patient_data_monthly" + ), + error_code="missing_value", + function_name="check_missing_patients", + ) + + phantom = cleaned_keys.join(raw_keys, on=ROW_KEY, how="anti") + for row in phantom.iter_rows(named=True): + emit_finding( + collector, + file_name="", + patient_id=row["patient_id"] or "unknown", + column="__row__", + original_value=f"{row['clinic_id']}|{row['tracker_year']}|{row['tracker_month']}", + error_message=( + f"PHANTOM_ROW: cleaned row for {row['patient_id']} " + f"({row['tracker_year']}-{row['tracker_month']:02d}) " + f"has no matching raw source" + ), + error_code="missing_value", + function_name="check_missing_patients", + ) + + +def _join_for_cell_checks(raw: pl.DataFrame, cleaned: pl.DataFrame) -> pl.DataFrame: + """Inner join raw and cleaned on the normalized row key. + + Suffixes raw columns with ``_raw`` and cleaned columns with ``_clean``; + the join key columns are deduplicated under their original names. + """ + raw_n = _normalize_join_keys(raw) + cleaned_n = _normalize_join_keys(cleaned) + + raw_renamed = raw_n.rename({c: f"{c}_raw" for c in raw_n.columns if c not in ROW_KEY}) + cleaned_renamed = cleaned_n.rename( + {c: f"{c}_clean" for c in cleaned_n.columns if c not in ROW_KEY} + ) + return raw_renamed.join(cleaned_renamed, on=ROW_KEY, how="inner") + + +def check_unexpected_nulls( + joined: pl.DataFrame, collector: ErrorCollector, file_name_col: str = "file_name_raw" +) -> None: + """UNEXPECTED_NULL: raw cell non-empty, cleaned cell null. + + Each finding carries a ``was_parseable`` indicator in the message so + reviewers can filter to surprising cases (raw value parsed cleanly but the + cleaned cell is still null). + """ + cleaned_cols = [c for c in joined.columns if c.endswith("_clean")] + for clean_col in cleaned_cols: + base = clean_col[: -len("_clean")] + raw_col = f"{base}_raw" + if raw_col not in joined.columns: + continue + + target_dtype = joined.schema[clean_col] + # null in cleaned, non-empty (after strip) in raw + raw_str = pl.col(raw_col).cast(pl.Utf8).str.strip_chars() + suspect = joined.filter( + pl.col(clean_col).is_null() & raw_str.is_not_null() & (raw_str.str.len_chars() > 0) + ) + if suspect.is_empty(): + continue + + # Determine parseability for this column once via the sacrificial parser. + if target_dtype.is_numeric() or target_dtype == pl.Date or target_dtype == pl.Boolean: + parsed = safe_parse_series(suspect[raw_col].cast(pl.Utf8), target_dtype) + else: + parsed = suspect[raw_col].cast(pl.Utf8) + + for row, parsed_val in zip(suspect.iter_rows(named=True), parsed.to_list(), strict=True): + was_parseable = parsed_val is not None + emit_finding( + collector, + file_name=row.get(file_name_col) or "", + patient_id=row.get("patient_id") or "unknown", + column=base, + original_value=row[raw_col], + error_message=( + f"UNEXPECTED_NULL: raw={row[raw_col]!r} cleaned=null " + f"was_parseable={was_parseable}" + ), + error_code="missing_value", + function_name="check_unexpected_nulls", + ) + + +def check_value_shifts( + joined: pl.DataFrame, collector: ErrorCollector, file_name_col: str = "file_name_raw" +) -> None: + """VALUE_SHIFT: numeric, non-derived, non-unit-converted columns only. + + For each eligible column, re-parse raw via the sacrificial converter and + compare to cleaned with abs/rel tolerance. + """ + for clean_col in [c for c in joined.columns if c.endswith("_clean")]: + base = clean_col[: -len("_clean")] + if base in DERIVED_COLUMNS or base in UNIT_CONVERTED_COLUMNS: + continue + raw_col = f"{base}_raw" + if raw_col not in joined.columns: + continue + clean_dtype = joined.schema[clean_col] + if not clean_dtype.is_numeric(): + continue + + # Subset to rows where both sides have content; let the sacrificial + # parser handle the empty/whitespace -> null normalization. + subset = joined.select([raw_col, clean_col, "patient_id", file_name_col]) + parsed_raw = safe_parse_series(subset[raw_col].cast(pl.Utf8), clean_dtype) + + cleaned_vals = subset[clean_col].to_list() + raw_vals = parsed_raw.to_list() + pids = subset["patient_id"].to_list() + files = subset[file_name_col].to_list() + + for raw_v, clean_v, pid, fname in zip(raw_vals, cleaned_vals, pids, files, strict=True): + # Both null is fine; one-sided null is captured by other checks. + if raw_v is None or clean_v is None: + continue + if is_close(float(raw_v), float(clean_v)): + continue + emit_finding( + collector, + file_name=fname or "", + patient_id=pid or "unknown", + column=base, + original_value=raw_v, + error_message=f"VALUE_SHIFT: raw={raw_v} cleaned={clean_v}", + error_code="invalid_value", + function_name="check_value_shifts", + ) + + +def _apply_height_auto_conversion(s: pl.Series) -> pl.Series: + """Mirror clean/patient.py:530-535: divide values >2.3 by 100.""" + return pl.Series( + s.name, + [None if v is None else (v / 100.0 if v > 2.3 else v) for v in s.to_list()], + dtype=pl.Float64, + ) + + +def check_out_of_range( + raw: pl.DataFrame, collector: ErrorCollector, file_name_col: str = "file_name" +) -> None: + """OUT_OF_RANGE_RAW: raw values outside the YAML numeric_ranges. + + Runs against the raw frame because the cleaner replaces out-of-range + values with the sentinel before they reach the cleaned table. ``height`` + has the cm->m auto-conversion applied first to mirror the cleaner. + """ + ranges = load_numeric_ranges() + raw_n = _normalize_join_keys(raw) + for column, bounds in ranges.items(): + if column not in raw_n.columns: + continue + parsed = safe_parse_series(raw_n[column].cast(pl.Utf8), pl.Float64) + if column == "height": + parsed = _apply_height_auto_conversion(parsed) + + min_v = float(bounds["min"]) + max_v = float(bounds["max"]) + + pids = raw_n["patient_id"].to_list() + files = ( + raw_n[file_name_col].to_list() + if file_name_col in raw_n.columns + else [""] * raw_n.height + ) + for v, pid, fname in zip(parsed.to_list(), pids, files, strict=True): + if v is None: + continue + if v < min_v or v > max_v: + emit_finding( + collector, + file_name=fname or "", + patient_id=pid or "unknown", + column=column, + original_value=v, + error_message=(f"OUT_OF_RANGE_RAW: value {v} outside [{min_v}, {max_v}]"), + error_code="invalid_value", + function_name="check_out_of_range", + ) + + +def validate_patient_run(run_dir: Path) -> ErrorCollector | None: + """Run all four patient checks against a pipeline run directory. + + Returns ``None`` if the run does not contain patient pipeline outputs + (e.g. V5 is product-only). + """ + raw = _load_raw(run_dir) + cleaned = _load_cleaned_monthly(run_dir) + if raw is None or cleaned is None: + logger.info(f"Patient pipeline outputs not found in {run_dir}; skipping.") + return None + + collector = ErrorCollector() + logger.info(f"Patient validation: raw={raw.shape}, cleaned_monthly={cleaned.shape}") + + check_missing_patients(raw, cleaned, collector) + joined = _join_for_cell_checks(raw, cleaned) + check_unexpected_nulls(joined, collector) + check_value_shifts(joined, collector) + check_out_of_range(raw, collector) + + logger.info(f"Patient validation: {len(collector)} findings") + return collector diff --git a/src/a4d/validate/source_vs_output_product.py b/src/a4d/validate/source_vs_output_product.py new file mode 100644 index 0000000..5076a80 --- /dev/null +++ b/src/a4d/validate/source_vs_output_product.py @@ -0,0 +1,196 @@ +"""Source-vs-output validation for the product pipeline. + +Group-granularity checks only in v1: per-cell checks would require reproducing +cleaning steps 2.0-2.5 to reconstruct the per-(sheet, product) row index, which +we deliberately avoid (see the plan at +``C:/Users/furin/.claude/plans/output-vs-source-validation-swirling-nygaard.md``). +""" + +from __future__ import annotations + +from pathlib import Path + +import polars as pl +from loguru import logger + +from a4d.errors import ErrorCollector +from a4d.validate.common import emit_finding + +GROUP_KEY = ["file_name", "product_sheet_name", "product"] + +# Threshold for COLUMN_NULL_RATE_DELTA: emit a finding if cleaned null rate +# exceeds raw null rate by more than this fraction (5 percentage points). +NULL_RATE_DELTA_THRESHOLD = 0.05 + + +def _explode_multi_product_cells(df: pl.DataFrame) -> pl.DataFrame: + """Mirror clean/product.py:169-242 (the row-changing part only). + + Splits ``product`` on '; ' and ' and ', then ``.explode()``. We deliberately + skip the parenthetical-units extraction because it doesn't change row count + or the join keys. + """ + if "product" not in df.columns: + return df + df = df.with_columns( + pl.col("product").cast(pl.Utf8).str.replace_all(" and ", "; ").alias("product") + ) + return df.with_columns(pl.col("product").str.split("; ")).explode("product") + + +def _load_raw(run_dir: Path) -> pl.DataFrame | None: + raw_dir = run_dir / "product_data_raw" + if not raw_dir.exists(): + return None + files = sorted(raw_dir.glob("*_product_raw.parquet")) + if not files: + return None + frames = [pl.read_parquet(f) for f in files] + return pl.concat(frames, how="diagonal") + + +def _load_cleaned(run_dir: Path) -> pl.DataFrame | None: + path = run_dir / "tables" / "product_data.parquet" + if not path.exists(): + return None + return pl.read_parquet(path) + + +def check_missing_groups( + raw_exploded: pl.DataFrame, cleaned: pl.DataFrame, collector: ErrorCollector +) -> None: + """MISSING_GROUP + PHANTOM_GROUP: anti-joins on (file, sheet, product).""" + raw_keys = raw_exploded.filter(pl.col("product").is_not_null()).select(GROUP_KEY).unique() + cleaned_keys = cleaned.filter(pl.col("product").is_not_null()).select(GROUP_KEY).unique() + + missing = raw_keys.join(cleaned_keys, on=GROUP_KEY, how="anti") + for row in missing.iter_rows(named=True): + emit_finding( + collector, + file_name=row["file_name"] or "", + patient_id="", + column="__group__", + original_value=f"{row['product_sheet_name']}|{row['product']}", + error_message=( + f"MISSING_GROUP: raw (sheet={row['product_sheet_name']}, " + f"product={row['product']!r}) absent in cleaned product_data" + ), + error_code="missing_value", + function_name="check_missing_groups", + ) + + phantom = cleaned_keys.join(raw_keys, on=GROUP_KEY, how="anti") + for row in phantom.iter_rows(named=True): + emit_finding( + collector, + file_name=row["file_name"] or "", + patient_id="", + column="__group__", + original_value=f"{row['product_sheet_name']}|{row['product']}", + error_message=( + f"PHANTOM_GROUP: cleaned (sheet={row['product_sheet_name']}, " + f"product={row['product']!r}) has no matching raw source" + ), + error_code="missing_value", + function_name="check_missing_groups", + ) + + +def check_row_count_delta( + raw_exploded: pl.DataFrame, cleaned: pl.DataFrame, collector: ErrorCollector +) -> None: + """ROW_COUNT_DELTA: per-group row counts raw vs cleaned. + + Reported as a magnitude with no judgment about right/wrong — the cleaner + legitimately drops uninformative rows. Reviewers triage. + """ + raw_counts = ( + raw_exploded.filter(pl.col("product").is_not_null()) + .group_by(GROUP_KEY) + .len() + .rename({"len": "raw_count"}) + ) + cleaned_counts = ( + cleaned.filter(pl.col("product").is_not_null()) + .group_by(GROUP_KEY) + .len() + .rename({"len": "cleaned_count"}) + ) + joined = raw_counts.join(cleaned_counts, on=GROUP_KEY, how="inner") + deltas = joined.filter(pl.col("raw_count") != pl.col("cleaned_count")) + + for row in deltas.iter_rows(named=True): + delta = row["cleaned_count"] - row["raw_count"] + emit_finding( + collector, + file_name=row["file_name"] or "", + patient_id="", + column="__group__", + original_value=f"raw={row['raw_count']}|cleaned={row['cleaned_count']}", + error_message=( + f"ROW_COUNT_DELTA: (sheet={row['product_sheet_name']}, " + f"product={row['product']!r}) raw={row['raw_count']} " + f"cleaned={row['cleaned_count']} delta={delta:+d}" + ), + error_code="invalid_value", + function_name="check_row_count_delta", + ) + + +def check_column_null_rate_delta( + raw: pl.DataFrame, cleaned: pl.DataFrame, collector: ErrorCollector +) -> None: + """COLUMN_NULL_RATE_DELTA: per-column null rate raw vs cleaned across the run. + + Emit one finding per column where cleaned null rate exceeds raw null rate + by >NULL_RATE_DELTA_THRESHOLD. Coarse, but surfaces silent column-wide losses. + """ + raw_total = raw.height + cleaned_total = cleaned.height + if raw_total == 0 or cleaned_total == 0: + return + + common = [c for c in cleaned.columns if c in raw.columns] + for col in common: + raw_null_rate = raw[col].null_count() / raw_total + cleaned_null_rate = cleaned[col].null_count() / cleaned_total + delta = cleaned_null_rate - raw_null_rate + if delta > NULL_RATE_DELTA_THRESHOLD: + emit_finding( + collector, + file_name="", + patient_id="", + column=col, + original_value=f"raw_null_rate={raw_null_rate:.3f}", + error_message=( + f"COLUMN_NULL_RATE_DELTA: column={col!r} " + f"raw_null_rate={raw_null_rate:.3f} " + f"cleaned_null_rate={cleaned_null_rate:.3f} " + f"delta={delta:+.3f}" + ), + error_code="invalid_value", + function_name="check_column_null_rate_delta", + ) + + +def validate_product_run(run_dir: Path) -> ErrorCollector | None: + """Run the four product checks. Returns None if no product output exists.""" + raw = _load_raw(run_dir) + cleaned = _load_cleaned(run_dir) + if raw is None or cleaned is None: + logger.info(f"Product pipeline outputs not found in {run_dir}; skipping.") + return None + + collector = ErrorCollector() + raw_exploded = _explode_multi_product_cells(raw) + logger.info( + f"Product validation: raw={raw.shape} (exploded={raw_exploded.shape}), " + f"cleaned={cleaned.shape}" + ) + + check_missing_groups(raw_exploded, cleaned, collector) + check_row_count_delta(raw_exploded, cleaned, collector) + check_column_null_rate_delta(raw, cleaned, collector) + + logger.info(f"Product validation: {len(collector)} findings") + return collector diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..650bbd2 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,16 @@ +"""Test-session setup that must run before any test module imports a4d.cli. + +GitHub Actions sets GITHUB_ACTIONS=1 for every job, and Typer reads that at +import time (typer.rich_utils.FORCE_TERMINAL) to force colored --help +rendering. That splits multi-character options like "--file" into separate +ANSI-styled spans (e.g. "-" then "-file"), breaking plain `"--file" in +result.output` assertions in CLI tests even though the same tests pass +locally where GITHUB_ACTIONS is unset. Typer's own escape hatch, +_TYPER_FORCE_DISABLE_TERMINAL, disables that forced styling; it must be set +before typer.rich_utils is first imported, hence here rather than in a +fixture. +""" + +import os + +os.environ.setdefault("_TYPER_FORCE_DISABLE_TERMINAL", "1") diff --git a/tests/test_clean/__init__.py b/tests/test_clean/__init__.py new file mode 100644 index 0000000..167c8d2 --- /dev/null +++ b/tests/test_clean/__init__.py @@ -0,0 +1 @@ +"""Tests for data cleaning modules.""" diff --git a/tests/test_clean/test_converters.py b/tests/test_clean/test_converters.py new file mode 100644 index 0000000..00bfea8 --- /dev/null +++ b/tests/test_clean/test_converters.py @@ -0,0 +1,987 @@ +"""Tests for type conversion with error tracking.""" + +from datetime import date + +import polars as pl +import pytest + +from a4d.clean.converters import ( + correct_decimal_sign, + cut_numeric_value, + normalize_excel_formula_errors, + parse_date_column, + safe_convert_column, + safe_convert_multiple_columns, +) +from a4d.clean.date_parser import ( + parse_date_flexible, + recover_date_from_text, + rescue_date_typos, +) +from a4d.config import settings +from a4d.errors import ErrorCollector + + +def test_safe_convert_column_success(): + """Test successful conversion without errors.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 3, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003"], + "age": ["25", "30", "18"], + } + ) + + collector = ErrorCollector() + + result = safe_convert_column( + df=df, + column="age", + target_type=pl.Int32, + error_collector=collector, + ) + + assert result.schema["age"] == pl.Int32 + assert result["age"].to_list() == [25, 30, 18] + assert len(collector) == 0 # No errors + + +def test_safe_convert_column_with_failures(): + """Test conversion with some failures.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 4, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003", "XX_YY004"], + "age": ["25", "invalid", "30", "abc"], + } + ) + + collector = ErrorCollector() + + result = safe_convert_column( + df=df, + column="age", + target_type=pl.Int32, + error_collector=collector, + ) + + assert result.schema["age"] == pl.Int32 + assert result["age"].to_list() == [ + 25, + int(settings.error_val_numeric), + 30, + int(settings.error_val_numeric), + ] + assert len(collector) == 2 # Two failures + + # Check error details + errors_df = collector.to_dataframe() + assert errors_df.filter(pl.col("patient_id") == "XX_YY002")["original_value"][0] == "invalid" + assert errors_df.filter(pl.col("patient_id") == "XX_YY004")["original_value"][0] == "abc" + assert all(errors_df["error_code"] == "type_conversion") + + +def test_safe_convert_column_preserves_nulls(): + """Test that existing nulls are preserved.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 3, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003"], + "age": ["25", None, "30"], + } + ) + + collector = ErrorCollector() + + result = safe_convert_column( + df=df, + column="age", + target_type=pl.Int32, + error_collector=collector, + ) + + assert result["age"].to_list() == [25, None, 30] + assert len(collector) == 0 # Nulls are not errors + + +def test_normalize_excel_formula_errors_nulls_and_logs(): + """Excel formula-error strings become null and are logged (ticket 27). + + Extraction preserves the source cell's literal error text; cleaning is + where it becomes null, so the raw layer stays a faithful capture. + """ + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 3, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003"], + "t1d_diagnosis_age": ["9", "#NUM!", "#DIV/0!"], + } + ) + + collector = ErrorCollector() + + result = normalize_excel_formula_errors(df, collector) + + assert result["t1d_diagnosis_age"].to_list() == ["9", None, None] + assert len(collector) == 2 + codes = {e.error_code for e in collector.errors} + assert codes == {"source_formula_error"} + originals = {e.original_value for e in collector.errors} + assert originals == {"#NUM!", "#DIV/0!"} + + +def test_normalize_excel_formula_errors_leaves_clean_data_untouched(): + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 2, + "patient_id": ["XX_YY001", "XX_YY002"], + "bmi": ["17.5", None], + } + ) + + collector = ErrorCollector() + + result = normalize_excel_formula_errors(df, collector) + + assert result["bmi"].to_list() == ["17.5", None] + assert len(collector) == 0 + + +def test_normalize_excel_formula_errors_skips_non_string_columns(): + df = pl.DataFrame( + {"product_table_year": [2024.0, 2024.0]}, + schema={"product_table_year": pl.Float64}, + ) + + collector = ErrorCollector() + + result = normalize_excel_formula_errors(df, collector) + + assert result.equals(df) + assert len(collector) == 0 + + +def test_normalize_excel_formula_errors_uses_custom_id_column(): + """The product arm identifies rows by `product`, not `patient_id`.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"], + "product": ["Insulin"], + "product_balance": ["#REF!"], + } + ) + + collector = ErrorCollector() + + result = normalize_excel_formula_errors(df, collector, patient_id_col="product") + + assert result["product_balance"].to_list() == [None] + assert len(collector) == 1 + assert collector.errors[0].patient_id == "Insulin" + + +def test_correct_decimal_sign(): + """Test decimal sign correction.""" + df = pl.DataFrame( + { + "weight": ["70,5", "80,2", "65.5"], + } + ) + + result = correct_decimal_sign(df, "weight") + + assert result["weight"].to_list() == ["70.5", "80.2", "65.5"] + + +def test_cut_numeric_value(): + """Test cutting out-of-range values.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 5, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003", "XX_YY004", "XX_YY005"], + "age": [15, -5, 20, 30, 18], + } + ) + + collector = ErrorCollector() + + result = cut_numeric_value( + df=df, + column="age", + min_val=0, + max_val=25, + error_collector=collector, + ) + + assert result["age"].to_list() == [ + 15, + settings.error_val_numeric, # -5 replaced + 20, + settings.error_val_numeric, # 30 replaced + 18, + ] + assert len(collector) == 2 # Two values out of range + + +def test_safe_convert_multiple_columns(): + """Test batch conversion of multiple columns.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 2, + "patient_id": ["XX_YY001", "XX_YY002"], + "age": ["25", "30"], + "height": ["1.75", "1.80"], + "weight": ["70", "80"], + } + ) + + collector = ErrorCollector() + + result = safe_convert_multiple_columns( + df=df, + columns=["age", "height", "weight"], + target_type=pl.Float64, + error_collector=collector, + ) + + assert result.schema["age"] == pl.Float64 + assert result.schema["height"] == pl.Float64 + assert result.schema["weight"] == pl.Float64 + assert len(collector) == 0 + + +def test_safe_convert_column_missing_column(): + """Test that missing columns are handled gracefully.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"], + "patient_id": ["XX_YY001"], + } + ) + + collector = ErrorCollector() + + # Should not raise error + result = safe_convert_column( + df=df, + column="nonexistent", + target_type=pl.Int32, + error_collector=collector, + ) + + assert result.equals(df) + assert len(collector) == 0 + + +def test_safe_convert_column_float64(): + """Test conversion to Float64 with decimal values.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 3, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003"], + "weight": ["70.5", "not_a_number", "85.2"], + } + ) + + collector = ErrorCollector() + + result = safe_convert_column( + df=df, + column="weight", + target_type=pl.Float64, + error_collector=collector, + ) + + assert result.schema["weight"] == pl.Float64 + assert result["weight"][0] == 70.5 + assert result["weight"][1] == settings.error_val_numeric + assert result["weight"][2] == 85.2 + assert len(collector) == 1 + + +def test_safe_convert_column_custom_error_value(): + """Test using a custom error value.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 2, + "patient_id": ["XX_YY001", "XX_YY002"], + "age": ["25", "invalid"], + } + ) + + collector = ErrorCollector() + + result = safe_convert_column( + df=df, + column="age", + target_type=pl.Int32, + error_collector=collector, + error_value=-1, + ) + + assert result["age"].to_list() == [25, -1] + assert len(collector) == 1 + + +def test_safe_convert_column_string_type(): + """Test conversion to string type (always succeeds).""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 2, + "patient_id": ["XX_YY001", "XX_YY002"], + "value": [123, 456], + } + ) + + collector = ErrorCollector() + + result = safe_convert_column( + df=df, + column="value", + target_type=pl.Utf8, + error_collector=collector, + ) + + assert result.schema["value"] == pl.Utf8 + assert result["value"].to_list() == ["123", "456"] + assert len(collector) == 0 + + +def test_correct_decimal_sign_missing_column(): + """Test decimal sign correction with missing column.""" + df = pl.DataFrame({"other": ["value"]}) + + result = correct_decimal_sign(df, "nonexistent") + + assert result.equals(df) + + +def test_cut_numeric_value_missing_column(): + """Test cutting with missing column.""" + df = pl.DataFrame({"other": [1, 2, 3]}) + + collector = ErrorCollector() + + result = cut_numeric_value( + df=df, + column="nonexistent", + min_val=0, + max_val=10, + error_collector=collector, + ) + + assert result.equals(df) + assert len(collector) == 0 + + +def test_cut_numeric_value_with_nulls(): + """Test that nulls are preserved when cutting values.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 4, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003", "XX_YY004"], + "age": [15, None, 30, 20], + } + ) + + collector = ErrorCollector() + + result = cut_numeric_value( + df=df, + column="age", + min_val=0, + max_val=25, + error_collector=collector, + ) + + assert result["age"].to_list() == [15, None, settings.error_val_numeric, 20] + assert len(collector) == 1 # Only 30 is out of range + + +def test_cut_numeric_value_ignores_existing_errors(): + """Test that existing error values are not re-logged.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 3, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003"], + "age": [15.0, settings.error_val_numeric, 30.0], + } + ) + + collector = ErrorCollector() + + result = cut_numeric_value( + df=df, + column="age", + min_val=0, + max_val=25, + error_collector=collector, + ) + + # Only 30 should be logged, not the existing error value + assert result["age"].to_list() == [15, settings.error_val_numeric, settings.error_val_numeric] + assert len(collector) == 1 + + +def test_rescue_date_typos_known_patterns(): + assert rescue_date_typos("23-Mach-20") == ("23-MAR-20", True) + assert rescue_date_typos("15-N0v-2021") == ("15-NOV-2021", True) + assert rescue_date_typos("10-0ct-2024") == ("10-OCT-2024", True) + assert rescue_date_typos("01-N0vember-2021") == ("01-NOVEMBER-2021", True) + + +def test_rescue_date_typos_passthrough(): + assert rescue_date_typos("15-Mar-2024") == ("15-Mar-2024", False) + # Word-boundary protects unrelated substrings. + assert rescue_date_typos("CON0CTOR") == ("CON0CTOR", False) + + +def test_rescue_date_typos_malay_month_names(): + assert rescue_date_typos("04-Mac-2026") == ("04-MAR-2026", True) + assert rescue_date_typos("5-Mei-2023") == ("5-MAY-2023", True) + assert rescue_date_typos("4-Okt-2023") == ("4-OCT-2023", True) + assert rescue_date_typos("1-Ogos-2024") == ("1-AUG-2024", True) + assert rescue_date_typos("2-Dis-2024") == ("2-DEC-2024", True) + + +def test_rescue_date_typos_thai_month_names(): + assert rescue_date_typos("3 เมย 2026") == ("3 APR 2026", True) + assert rescue_date_typos("9 มค 2026") == ("9 JAN 2026", True) + assert rescue_date_typos("10-ม.ค.-2025") == ("10-JAN-2025", True) + + +def test_rescue_date_typos_dropped_and_transposed_letters(): + assert rescue_date_typos("9-Dce-20") == ("9-DEC-20", True) + assert rescue_date_typos("6-ug-2025") == ("6-AUG-2025", True) + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + # A trailing separator followed by a space (ticket 56). + ("26-05- 2007", date(2007, 5, 26)), + # A stray underscore or equals sign typed instead of the separator. + ("19-Jan_2023", date(2023, 1, 19)), + ("7_May-21", date(2021, 5, 7)), + ("02-Apr=-2026", date(2026, 4, 2)), + # A doubled or mixed separator run. + ("23/05//2025", date(2025, 5, 23)), + ("15-05-/2026", date(2026, 5, 15)), + ("16-July-/2025", date(2025, 7, 16)), + # A zero-width space pasted in from another application. + ("11​ Mar 2026", date(2026, 3, 11)), + ], +) +def test_parse_date_flexible_recovers_separator_damage(source, expected): + assert parse_date_flexible(source) == expected + + +@pytest.mark.parametrize( + "source", + [ + # Glued digit runs: where the missing separator goes is a guess, and + # any reading invents information the cell does not carry. + "26/102022", + "10/1023", + # A date with a stray digit group nobody can resolve. A year *was* + # written here, so the tracker year must not be substituted for it the + # way it is for a note that names none (ticket 39) -- including when the + # stray group is joined by a damaged separator ("=", "_"). + "10-Oct-2-24", + "12-Jun=2-26", + ], +) +def test_parse_date_flexible_still_rejects_ambiguous_damage(source): + assert parse_date_flexible(source) == date(9999, 9, 9) + + +def test_parse_date_flexible_keeps_trailing_free_text_behaviour(): + # Separator normalization must not run the free-text clause into the date. + assert parse_date_flexible("16-Nov-2019 due to DKA") == date(2019, 11, 16) + + +def test_parse_date_column_rescues_typo_and_logs(): + df = pl.DataFrame( + { + "file_name": ["t.xlsx", "t.xlsx"], + "patient_id": ["P1", "P2"], + "entry_date": ["23-Mach-20", "15-Mar-2024"], + } + ) + collector = ErrorCollector() + + result = parse_date_column(df, "entry_date", collector) + + parsed = result["entry_date"].to_list() + assert parsed[0] == date(2020, 3, 23) + assert parsed[1] == date(2024, 3, 15) + assert len(collector) == 1 + err = collector.errors[0] + assert err.error_code == "typo_rescued" + assert err.column == "entry_date" + assert err.original_value == "23-Mach-20" + assert err.patient_id == "P1" + + +def test_parse_date_column_logs_unparseable_dates(): + """Pin parse_date_column's observability for genuinely unparseable cells. + + A cell that cannot be read must be recoverable from the log rather than + silently sentinelled, so the type_conversion entry carries the original + value. Abbreviated forms like "Mar 18" parse successfully and must not + appear here — the log covers unreadable cells with strictly + better signal-to-noise. See cleaning_divergences.md §10. + """ + df = pl.DataFrame( + { + "file_name": ["test.xlsx", "test.xlsx", "test.xlsx"], + "patient_id": ["P1", "P2", "P3"], + "entry_date": ["2024-03-15", "garbage_value_xyz", "2024-04-20"], + }, + schema={ + "file_name": pl.String, + "patient_id": pl.String, + "entry_date": pl.String, + }, + ) + collector = ErrorCollector() + + result = parse_date_column(df=df, column="entry_date", error_collector=collector) + + parsed = result["entry_date"].to_list() + assert parsed[0] == date(2024, 3, 15) + assert parsed[1] == date(9999, 9, 9) # error_val_date sentinel + assert parsed[2] == date(2024, 4, 20) + + assert len(collector) == 1 + err = collector.errors[0] + assert err.error_code == "type_conversion" + assert err.function_name == "parse_date_column" + assert err.column == "entry_date" + assert err.original_value == "garbage_value_xyz" + assert err.patient_id == "P2" + + +def test_parse_date_flexible_month_with_four_digit_year_is_first_of_month(): + """ "Jun 2006" is a real recorded diagnosis date in 2017-era trackers. + + Without an explicit branch dateutil fills the missing day from *today*, + which makes the pipeline's output depend on the day it runs. The first of + the month is the only defensible reading: it is the earliest the recorded + month can mean. + """ + assert parse_date_flexible("Jun 2006") == date(2006, 6, 1) + assert parse_date_flexible("March 2011") == date(2011, 3, 1) + + +def test_parse_date_flexible_month_with_two_digit_year_still_first_of_month(): + """The pre-existing two-digit branch keeps its behaviour.""" + assert parse_date_flexible("Mar-18") == date(2018, 3, 1) + + +def test_parse_date_flexible_handles_full_month_names(): + """Full month names were truncated to nonsense ("March" -> "Marh") and + sentinelled; these are the docstring's own examples, which never worked. + """ + assert parse_date_flexible("January-20") == date(2020, 1, 1) + assert parse_date_flexible("March 2011") == date(2011, 3, 1) + assert parse_date_flexible("07 March 2015") == date(2015, 3, 7) + assert parse_date_flexible("Sept-19") == date(2019, 9, 1) + + +def test_parse_date_flexible_handles_a_month_name_run_into_its_year(): + """A rich-text run can leave a month name glued to its year -- "July2014" + (ticket 50, 2017 Yangon Children's Feb17!I66). + + The month-name truncation required a word boundary after the name, which a + following digit does not provide, so the month-year branch never saw it and + dateutil filled the day from *today* -- the same run-date dependence ticket + 37 removed from the spaced form. + """ + assert parse_date_flexible("July2014") == date(2014, 7, 1) + assert parse_date_flexible("Jan2012") == date(2012, 1, 1) + assert parse_date_flexible("May2015") == date(2015, 5, 1) + + +def test_parse_date_flexible_reads_a_buddhist_era_serial_as_the_date_it_encodes(): + """A Thai Buddhist-Era year typed into a Gregorian date cell produces an + Excel serial far above a plausible Gregorian one (ticket 50, 2022 Hat Yai + Patient List!G25 holds 2560-01-01, serial 241062). + + Below the raised ceiling it fell through to dateutil, which read the digits + positionally as 24/10/62 -- a plausible-looking date that is not what the + cell holds. The cleaned stage's own future-date guard still sentinels it. + """ + assert parse_date_flexible("241062") == date(2560, 1, 1) + assert parse_date_flexible("243498") == date(2566, 9, 2) + + +def test_parse_date_flexible_reads_a_bare_year_as_the_first_of_that_year(): + """A bare four-digit year typed into a date cell is a year, not an Excel + serial (ticket 52, Sarawak General Hospital Patient List!G10 holds 2011). + + As a serial it lands in 1905, which no tracker records, and the derived + t1d_diagnosis_age went negative. The same clinic wrote the same patients' + diagnoses as real 2011-01-01 dates in its 2024 workbook, so the first of + January is the clinic's own convention for a year without a day. + """ + assert parse_date_flexible("2011") == date(2011, 1, 1) + assert parse_date_flexible("1994") == date(1994, 1, 1) + assert parse_date_flexible("2017.0") == date(2017, 1, 1) + + +def test_parse_date_flexible_still_rejects_a_number_too_large_to_be_a_date(): + """The numeric error sentinel and other large counts must not become dates.""" + assert parse_date_flexible("999999") == date(9999, 9, 9) + assert parse_date_flexible("1141523") == date(9999, 9, 9) + + +def test_parse_date_flexible_recovers_a_date_followed_by_free_text(): + """Hospitalisation cells carry a clause after the date. The prefix + fallback must reach the date without a bare month name completing itself + from today's date. + """ + assert parse_date_flexible("16-Nov-2019 due to DKA") == date(2019, 11, 16) + assert parse_date_flexible("Jan-2020 due to poor glycaemic control") == date(2020, 1, 1) + assert parse_date_flexible("May 2019, Dec 2019 DKA") == date(2019, 5, 1) + + +def test_parse_date_flexible_never_completes_a_missing_day_from_today(): + """A cell that records only a month and a year must resolve to the first of + that month, whatever day the pipeline happens to run on (ticket 53). + + The month-year branch only ever recognised an alphabetic month, so the + numeric and comma-separated spellings fell through to dateutil, which fills + an absent day from ``datetime.now()``. On the real 248-tracker set that put + 42 production cells on the run date's own day-of-month -- "10/2019" read as + the 19th of October because the comparison ran on the 19th -- and made the + cleaned output differ from one run to the next with no input change. + """ + for text in ("10/2019", "06/2020", "4/2018", "10-2019"): + parsed = parse_date_flexible(text) + assert parsed is not None, text + assert parsed.day == 1, text + assert parse_date_flexible("10/2019") == date(2019, 10, 1) + assert parse_date_flexible("Mar, 2017") == date(2017, 3, 1) + assert parse_date_flexible("August,2015") == date(2015, 8, 1) + assert parse_date_flexible("June ,2016") == date(2016, 6, 1) + + +def test_parse_date_flexible_rejects_a_month_with_no_year_at_all(): + """A bare "Sep" carries no year, so dateutil supplies the current one and the + result moves every January. There is no year to recover, so the cell is + unparseable rather than a date (ticket 53). + """ + assert parse_date_flexible("Sep") == date(9999, 9, 9) + assert parse_date_flexible("Jan") == date(9999, 9, 9) + + +def test_parse_date_flexible_still_sentinels_genuine_garbage(): + assert parse_date_flexible("garbage_value_xyz") == date(9999, 9, 9) + assert parse_date_flexible("NA") is None + + +def test_parse_date_flexible_treats_every_numeric_missing_marker_as_missing(): + """A date column's missing markers are the same ones the numeric path + already normalizes (ticket 38). "-" and "N/A" reaching the sentinel meant + the cleaned output claimed "a date was recorded but is invalid" for a cell + that plainly recorded nothing. + """ + for marker in ("-", ".", "N/A", "n/a", "NULL", "None", " "): + assert parse_date_flexible(marker) is None, marker + + +def test_parse_date_flexible_treats_written_absence_as_missing(): + """Clinicians write absence in words, not only as "NA" (ticket 38: + "Nil" 510 rows, "Unknown" 481, "?" 130 across the real 254-tracker set). + """ + for marker in ("Nil", "nil", "Nill", "Unknown", "unknwon", "uncertain", "?"): + assert parse_date_flexible(marker) is None, marker + + +def test_parse_date_flexible_treats_template_placeholder_text_as_missing(): + """The tracker template's own instruction text leaks into data rows + ("Insert Date", "NA or Hospitalisation Date") -- it is a blank cell that + was never filled in, not a date that failed to parse. + """ + for marker in ("Insert Date", "Insert Date or NA", "NA or Hospitalisation Date"): + assert parse_date_flexible(marker) is None, marker + + +def test_parse_date_flexible_still_sentinels_a_recorded_but_unusable_value(): + """The widened missing set must not swallow the case the sentinel exists + for: something was written, and it is not a date and not an absence. + """ + assert parse_date_flexible("garbage_value_xyz") == date(9999, 9, 9) + assert parse_date_flexible("She stay in Hospital") == date(9999, 9, 9) + + +def test_parse_date_flexible_rejects_a_year_with_a_digit_missing(): + """A source year typed short is unusable, not a date in antiquity. + + 2024 Vietnam National Children's writes `1/16/224` and `5/16/223`, 2023 + Yangon General writes `13-Mar-0202`, and dateutil reads each literally, so + `0224-01-16` reached production output (ticket 55). No clinical date this + dataset records predates 1900, and no + arithmetic recovers the missing digit, so the cell is unreadable. + """ + assert parse_date_flexible("1/16/224") == date(9999, 9, 9) + assert parse_date_flexible("13-Mar-0202") == date(9999, 9, 9) + assert parse_date_flexible("31 oct 222") == date(9999, 9, 9) + assert parse_date_flexible("1-Oct-205") == date(9999, 9, 9) + + +def test_parse_date_flexible_keeps_the_oldest_dates_the_data_really_holds(): + assert parse_date_flexible("1/1/1950") == date(1950, 1, 1) + assert parse_date_flexible("1994") == date(1994, 1, 1) + + +# --- Dates buried inside a clinical note (ticket 39) ------------------------- +# +# The strings below are all real values of the `hospitalisation_date` column, +# taken from the 254-tracker set (7,847 non-null cells, 745 distinct strings). +# The column's header is free text -- "Hospitalisation due to diabetes emergency +# or glucose control (Include Date)" -- so clinicians write a case note and put +# the date somewhere inside it. + + +def test_recovers_a_date_written_in_the_middle_of_a_note(): + """The largest shape in the column: 302 cells, one full date inside prose. + + Recovery used to depend on *where* the date sat -- the prefix walk only + reaches a date at the front of the string -- so "16-Nov-2019 due to DKA" + was published and "DKA 16-Nov-2019" was sentinelled. + """ + assert parse_date_flexible("DKA 23 Oct 2020") == date(2020, 10, 23) + assert parse_date_flexible("Passed away 28/10/2019 due to DKA") == date(2019, 10, 28) + assert parse_date_flexible("DKA 30-Nov-2020 (Meikhtila hospital)") == date(2020, 11, 30) + assert parse_date_flexible("7/04/2021: DKA") == date(2021, 4, 7) + assert parse_date_flexible("22nd Feb.2019(DKA)") == date(2019, 2, 22) + assert parse_date_flexible("DKA 13 Nov2020") == date(2020, 11, 13) + assert parse_date_flexible("DKA 4 Sept 2020") == date(2020, 9, 4) + + +def test_recovers_a_month_and_year_written_inside_a_note(): + """45 cells name a month and a year but no day. Resolved to the 1st, the + same convention the whole-cell month-year branch already uses. + """ + assert parse_date_flexible("DKA - Feb-2020") == date(2020, 2, 1) + assert parse_date_flexible("DKA; Aug 2018") == date(2018, 8, 1) + assert parse_date_flexible("April-19: high HbA1c") == date(2019, 4, 1) + assert parse_date_flexible("May'21: Rx at Emergency room Hyperglycemia") == date(2021, 5, 1) + assert parse_date_flexible("Hypoglycemic seizure and DKA ; Apr 2019") == date(2019, 4, 1) + + +def test_takes_the_first_date_when_a_note_records_several(): + """A single date column cannot hold three admissions, so the first + mentioned is published and the discard is recorded. This is already what + the prefix walk does for the cells that happen to start with a date. + """ + text = "DKA Apr-2020 (Pokaku Hospital); DKA May-2021 (Mingalar Hospital)" + assert parse_date_flexible(text) == date(2020, 4, 1) + assert parse_date_flexible("Dec 2019, Mar 2020 DKA Jan 2021 DKA") == date(2019, 12, 1) + + +def test_takes_the_admission_day_when_the_note_records_a_stay(): + """A range is one admission written with both endpoints. The header asks + for the hospitalisation date, so the first endpoint is the answer. + """ + assert parse_date_flexible("22/7/2020-26/7/2020") == date(2020, 7, 22) + assert parse_date_flexible("1-5/6/2020") == date(2020, 6, 1) + assert parse_date_flexible("Admit 20-29/12/2020 :DKA") == date(2020, 12, 20) + assert parse_date_flexible("11-15 /01/2019") == date(2019, 1, 11) + assert parse_date_flexible("DKA: admitted 6-12 Nov 2020") == date(2020, 11, 6) + assert parse_date_flexible("23-27/9/2020 for controled blood sugar") == date(2020, 9, 23) + + +def test_fills_an_absent_year_from_the_tracker_year(): + """31 cells name a day and a month and no year at all ("26 Jun (ceton urine + high)"). Nothing in the cell supplies the year, so the tracker's own year + is used -- a patient seen in a tracker's year is the assumption this rests + on, recorded on the map rather than derivable from the data. + """ + assert parse_date_flexible("26 Jun (ceton urine high)", tracker_year=2020) == date(2020, 6, 26) + assert parse_date_flexible( + "admitted to Pokaku Hosp due to DKA 7-15 Apr", tracker_year=2020 + ) == date(2020, 4, 7) + + +def test_refuses_a_yearless_date_when_no_tracker_year_is_known(): + """The year is invented from context or not at all; there is no fallback + to today, which is the failure this parser has already been bitten by + three times (tickets 50, 53, 56). + """ + assert parse_date_flexible("26 Jun (ceton urine high)") == date(9999, 9, 9) + + +def test_refuses_a_note_whose_only_date_component_is_a_year(): + """ "DKA 2019" names no month, and a month cannot be recovered from + anything. The whole-cell bare-year branch stays as it is -- a cell holding + only "2019" is a clinic's own convention for a year with no day (ticket 52) + -- but a stray four-digit number inside prose is not that. + """ + assert parse_date_flexible("DKA 2019") == date(9999, 9, 9) + + +def test_refuses_a_note_that_contains_numbers_but_no_date(): + """48 cells carry digits and no date. This is the case that rules out + dateutil's fuzzy mode, which returns a date built from today for every one + of them rather than declining. + """ + assert parse_date_flexible("3 month come back meet Doctor") == date(9999, 9, 9) + assert parse_date_flexible("check 2 or 3 time per/day") == date(9999, 9, 9) + assert parse_date_flexible("on stamlor 5mg") == date(9999, 9, 9) + assert parse_date_flexible( + "Frequent hypoglycemia - admitted local hospital 3 times/year" + ) == date(9999, 9, 9) + + +def test_refuses_a_buddhist_era_year_written_into_a_note(): + """ "18-19/11/2567" is a BE year in a Gregorian field. Reading it as 2567 + fails the plausible-year guard; converting it would be a separate decision + about the source workbook, so the cell is sentinelled and reported. + """ + assert parse_date_flexible("18-19/11/2567") == date(9999, 9, 9) + + +def test_the_recogniser_reports_what_it_did(): + """converters logs a distinct code per decision, so recovery stays + auditable: what was read, what was discarded, what was filled in. + """ + recovered = recover_date_from_text("DKA 23 Oct 2020", tracker_year=2020) + assert recovered.value == date(2020, 10, 23) + assert recovered.tokens_found == 1 + assert recovered.year_inferred is False + + several = recover_date_from_text("Dec 2019, Mar 2020 DKA Jan 2021 DKA", tracker_year=2021) + assert several.value == date(2019, 12, 1) + assert several.tokens_found > 1 + + stay = recover_date_from_text("22/7/2020-26/7/2020", tracker_year=2021) + assert stay.tokens_found > 1 + + inferred = recover_date_from_text("26 Jun (ceton urine high)", tracker_year=2020) + assert inferred.value == date(2020, 6, 26) + assert inferred.year_inferred is True + + nothing = recover_date_from_text("on stamlor 5mg", tracker_year=2020) + assert nothing.value is None + assert nothing.tokens_found == 0 + + +def test_recovering_from_text_leaves_every_other_reading_untouched(): + """The recogniser fires only after the whole string, the separator repair + and the prefix walk have all failed, so no value that already parsed may + change reading -- including the ones earlier rounds fixed. + """ + assert parse_date_flexible("28/8/2017") == date(2017, 8, 28) + assert parse_date_flexible("Mar-18") == date(2018, 3, 1) + assert parse_date_flexible("10/2019") == date(2019, 10, 1) + assert parse_date_flexible("16-Nov-2019 due to DKA") == date(2019, 11, 16) + assert parse_date_flexible("26-05- 2007") == date(2007, 5, 26) + assert parse_date_flexible("1994") == date(1994, 1, 1) + assert parse_date_flexible("Nil") is None + assert parse_date_flexible("She stay in Hospital") == date(9999, 9, 9) + assert parse_date_flexible("1/16/224") == date(9999, 9, 9) + + +def test_parse_date_column_logs_a_date_read_out_of_a_note(): + """Recovery has to stay auditable: the entry carries the note it was read + from, so a reviewer can judge the extraction without re-running anything. + """ + df = pl.DataFrame( + { + "file_name": ["t.xlsx"], + "patient_id": ["P1"], + "tracker_year": [2020], + "hospitalisation_date": ["DKA 23 Oct 2020"], + } + ) + collector = ErrorCollector() + + result = parse_date_column(df, "hospitalisation_date", collector) + + assert result["hospitalisation_date"].to_list() == [date(2020, 10, 23)] + codes = [e.error_code for e in collector.errors] + assert codes == ["date_recovered_from_text"] + assert collector.errors[0].original_value == "DKA 23 Oct 2020" + + +def test_parse_date_column_reports_a_note_holding_several_dates(): + """A single date column cannot represent three admissions. The extra dates + are discarded, and the cell is reported so the workbook can be corrected -- + the repair belongs in the tracker, not in the parser. + """ + df = pl.DataFrame( + { + "file_name": ["t.xlsx"], + "patient_id": ["P1"], + "tracker_year": [2021], + "hospitalisation_date": ["Dec 2019, Mar 2020 DKA Jan 2021 DKA"], + } + ) + collector = ErrorCollector() + + result = parse_date_column(df, "hospitalisation_date", collector) + + assert result["hospitalisation_date"].to_list() == [date(2019, 12, 1)] + assert "date_multiple_in_cell" in [e.error_code for e in collector.errors] + + +def test_parse_date_column_reports_a_year_taken_from_the_tracker(): + """The one component published that the cell does not state, so it is + logged under its own code rather than folded into recovery. + """ + df = pl.DataFrame( + { + "file_name": ["t.xlsx"], + "patient_id": ["P1"], + "tracker_year": [2020], + "hospitalisation_date": ["26 Jun (ceton urine high)"], + } + ) + collector = ErrorCollector() + + result = parse_date_column(df, "hospitalisation_date", collector) + + assert result["hospitalisation_date"].to_list() == [date(2020, 6, 26)] + assert "date_year_inferred" in [e.error_code for e in collector.errors] + + +def test_parse_date_column_takes_the_year_from_each_row_not_the_column(): + """The same note appears in trackers from different years -- 2020 VNCH and + 2021 VNCH both carry "26 Jun (ceton urine high)" for the same patient -- so + the year has to be resolved per row, not once per distinct string. + """ + df = pl.DataFrame( + { + "file_name": ["a.xlsx", "b.xlsx"], + "patient_id": ["P1", "P1"], + "tracker_year": [2020, 2021], + "hospitalisation_date": ["26 Jun (ceton urine high)"] * 2, + } + ) + collector = ErrorCollector() + + result = parse_date_column(df, "hospitalisation_date", collector) + + assert result["hospitalisation_date"].to_list() == [date(2020, 6, 26), date(2021, 6, 26)] + + +def test_parse_date_column_says_nothing_when_no_note_is_involved(): + """The codes must stay rare enough to read: an ordinary date column logs + none of them. + """ + df = pl.DataFrame( + { + "file_name": ["t.xlsx", "t.xlsx"], + "patient_id": ["P1", "P2"], + "tracker_year": [2024, 2024], + "entry_date": ["15-Mar-2024", "2024-04-20"], + } + ) + collector = ErrorCollector() + + parse_date_column(df, "entry_date", collector) + + assert collector.errors == [] diff --git a/tests/test_clean/test_glucose.py b/tests/test_clean/test_glucose.py new file mode 100644 index 0000000..849ba65 --- /dev/null +++ b/tests/test_clean/test_glucose.py @@ -0,0 +1,124 @@ +"""Tests for glucose unit resolution (ticket 42).""" + +import polars as pl + +from a4d.clean.glucose import ( + MMOL_TO_MG_FACTOR, + column_is_recorded_in_mmol, + resolve_glucose_units, +) +from a4d.errors import ErrorCollector + + +def _frame(mg: list, mmol: list | None = None, column: str = "fbg_updated_mg") -> pl.DataFrame: + mmol_column = column.replace("_mg", "_mmol") + return pl.DataFrame( + { + "file_name": ["t.xlsx"] * len(mg), + "patient_id": [f"KH_KB{i:03d}" for i in range(len(mg))], + column: pl.Series(mg, dtype=pl.Float64), + mmol_column: pl.Series( + mmol if mmol is not None else [None] * len(mg), dtype=pl.Float64 + ), + } + ) + + +class TestColumnIsRecordedInMmol: + def test_true_when_nearly_every_reading_lands_in_mmol_territory(self): + values = pl.Series([8.0, 9.4, 12.1, 7.2, 15.0, 6.6, 19.3, 11.0, 5.5, 22.0]) + + assert column_is_recorded_in_mmol(values) is True + + def test_false_when_the_column_reads_as_ordinary_mg(self): + values = pl.Series([120.0, 190.0, 233.0, 88.0, 310.0, 145.0, 99.0, 275.0, 180.0, 210.0]) + + assert column_is_recorded_in_mmol(values) is False + + def test_false_when_only_some_readings_are_low(self): + """The 50-90% band is per-patient mixing, not a mislabelled column.""" + values = pl.Series([8.0, 9.4, 12.1, 7.2, 15.0, 190.0, 233.0, 145.0, 180.0, 210.0]) + + assert column_is_recorded_in_mmol(values) is False + + def test_false_when_too_few_readings_to_judge(self): + values = pl.Series([8.0, 9.4, 12.1]) + + assert column_is_recorded_in_mmol(values) is False + + def test_nulls_are_not_counted_as_readings(self): + values = pl.Series([8.0, 9.4, 12.1, None, None, None, None, None, None, None]) + + assert column_is_recorded_in_mmol(values) is False + + +class TestResolveGlucoseUnits: + def test_a_wholly_mmol_column_is_moved_to_its_mmol_sibling(self): + df = _frame([8.0, 9.4, 12.1, 7.2, 15.0, 6.6, 19.3, 11.0, 5.5, 22.0]) + collector = ErrorCollector() + + out = resolve_glucose_units(df, collector) + + assert out["fbg_updated_mmol"].to_list()[0] == 8.0 + assert out["fbg_updated_mg"].to_list()[0] == 8.0 * MMOL_TO_MG_FACTOR + + def test_a_swapped_column_is_reported_once_not_once_per_row(self): + """The defect is the column's label, so the source fix is file-level.""" + df = _frame([8.0, 9.4, 12.1, 7.2, 15.0, 6.6, 19.3, 11.0, 5.5, 22.0]) + collector = ErrorCollector() + + resolve_glucose_units(df, collector) + + swapped = [e for e in collector.errors if e.error_code == "glucose_unit_swapped"] + assert len(swapped) == 1 + assert swapped[0].column == "fbg_updated_mg" + + def test_an_existing_mmol_reading_is_not_overwritten(self): + df = _frame( + [8.0, 9.4, 12.1, 7.2, 15.0, 6.6, 19.3, 11.0, 5.5, 22.0], + mmol=[4.4] + [None] * 9, + ) + collector = ErrorCollector() + + out = resolve_glucose_units(df, collector) + + assert out["fbg_updated_mmol"].to_list()[0] == 4.4 + + def test_a_stray_low_reading_in_a_clean_column_is_flagged_not_converted(self): + """A genuine severe hypo is indistinguishable from a mis-entered unit, + so a scattered value keeps its recorded number.""" + df = _frame([120.0, 190.0, 233.0, 88.0, 310.0, 145.0, 99.0, 275.0, 180.0, 12.0]) + collector = ErrorCollector() + + out = resolve_glucose_units(df, collector) + + assert out["fbg_updated_mg"].to_list()[-1] == 12.0 + assert out["fbg_updated_mmol"].to_list()[-1] is None + suspect = [e for e in collector.errors if e.error_code == "glucose_unit_suspect"] + assert len(suspect) == 1 + assert suspect[0].patient_id == "KH_KB009" + + def test_a_swapped_column_raises_no_row_level_suspicion(self): + df = _frame([8.0, 9.4, 12.1, 7.2, 15.0, 6.6, 19.3, 11.0, 5.5, 22.0]) + collector = ErrorCollector() + + resolve_glucose_units(df, collector) + + assert not [e for e in collector.errors if e.error_code == "glucose_unit_suspect"] + + def test_zero_is_not_a_reading(self): + """Zero sits below the analytical floor of both units, so it records + 'not measured' rather than profound hypoglycaemia.""" + df = _frame([120.0, 0.0, 233.0], mmol=[None, 0.0, None]) + collector = ErrorCollector() + + out = resolve_glucose_units(df, collector) + + assert out["fbg_updated_mg"].to_list() == [120.0, None, 233.0] + assert out["fbg_updated_mmol"].to_list() == [None, None, None] + + def test_missing_columns_are_tolerated(self): + df = pl.DataFrame({"file_name": ["t.xlsx"], "patient_id": ["KH_KB001"]}) + collector = ErrorCollector() + + assert resolve_glucose_units(df, collector).equals(df) diff --git a/tests/test_clean/test_patient.py b/tests/test_clean/test_patient.py new file mode 100644 index 0000000..7628145 --- /dev/null +++ b/tests/test_clean/test_patient.py @@ -0,0 +1,786 @@ +"""Unit tests for patient cleaning functions.""" + +from datetime import date + +import polars as pl +import pytest + +from a4d.clean.patient import ( + _apply_preprocessing, + _apply_range_validation, + _apply_type_conversions, + _convert_buddhist_era_dates, + _derive_insulin_fields, + _extract_date_from_measurement, + _fix_age_from_dob, + _fix_t1d_diagnosis_age, + _validate_dates, + clean_patient_data, +) +from a4d.config import settings +from a4d.errors import ErrorCollector + + +class TestPatientIdNormalization: + """Tests for patient_id normalization (transfer clinic suffix removal).""" + + def test_normalize_transfer_patient_id(self): + """Should normalize patient_id by removing transfer clinic suffix.""" + df = pl.DataFrame( + { + "patient_id": ["MY_SM003_SB", "TH_BK001_PT", "LA_VT002_VP"], + "name": ["Patient A", "Patient B", "Patient C"], + } + ) + + result = _apply_preprocessing(df) + + assert result["patient_id"].to_list() == ["MY_SM003", "TH_BK001", "LA_VT002"] + + def test_preserve_normal_patient_id(self): + """Should preserve patient_id without transfer suffix.""" + df = pl.DataFrame( + { + "patient_id": ["MY_SB001", "TH_ST003", "LA_LFH042"], + "name": ["Patient A", "Patient B", "Patient C"], + } + ) + + result = _apply_preprocessing(df) + + # Should remain unchanged + assert result["patient_id"].to_list() == ["MY_SB001", "TH_ST003", "LA_LFH042"] + + def test_mixed_patient_ids(self): + """Should handle mix of normal and transfer patient IDs.""" + df = pl.DataFrame( + { + "patient_id": [ + "MY_SB001", # Normal + "MY_SM003_SB", # Transfer + "TH_ST003", # Normal + "TH_BK001_PT", # Transfer + ], + "name": ["A", "B", "C", "D"], + } + ) + + result = _apply_preprocessing(df) + + assert result["patient_id"].to_list() == [ + "MY_SB001", + "MY_SM003", # Normalized + "TH_ST003", + "TH_BK001", # Normalized + ] + + def test_multiple_underscores_keeps_only_first_two_parts(self): + """Should keep only first two underscore-separated parts.""" + df = pl.DataFrame( + { + "patient_id": ["MY_SM003_SB_EXTRA"], # Three underscores + "name": ["Patient A"], + } + ) + + result = _apply_preprocessing(df) + + # Should extract only MY_SM003 + assert result["patient_id"][0] == "MY_SM003" + + def test_patient_id_without_underscores(self): + """Should preserve patient_id without underscores.""" + df = pl.DataFrame( + { + "patient_id": ["MYID001", "NOMATCH"], + "name": ["Patient A", "Patient B"], + } + ) + + result = _apply_preprocessing(df) + + # Pattern won't match, should keep original + assert result["patient_id"].to_list() == ["MYID001", "NOMATCH"] + + def test_null_patient_id_preserved(self): + """Should preserve null patient_ids.""" + df = pl.DataFrame( + { + "patient_id": [None, "MY_SB001", None], + "name": ["A", "B", "C"], + } + ) + + result = _apply_preprocessing(df) + + assert result["patient_id"][0] is None + assert result["patient_id"][1] == "MY_SB001" + assert result["patient_id"][2] is None + + +class TestHbA1cPreprocessing: + """Tests for HbA1c exceeds marker handling.""" + + def test_hba1c_baseline_exceeds_marker(self): + """Should extract > or < markers and remove them from value.""" + df = pl.DataFrame( + { + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003"], + "hba1c_baseline": [">14", "<5.5", "7.2"], + } + ) + + result = _apply_preprocessing(df) + + assert result["hba1c_baseline_exceeds"].to_list() == [True, True, False] + assert result["hba1c_baseline"].to_list() == ["14", "5.5", "7.2"] + + def test_hba1c_updated_exceeds_marker(self): + """Should extract > or < markers from updated HbA1c.""" + df = pl.DataFrame( + { + "patient_id": ["XX_YY001"], + "hba1c_updated": [">12.5"], + } + ) + + result = _apply_preprocessing(df) + + assert result["hba1c_updated_exceeds"][0] is True + assert result["hba1c_updated"][0] == "12.5" + + +class TestFbgPreprocessing: + """Tests for FBG (fasting blood glucose) text value handling.""" + + def test_fbg_qualitative_to_numeric(self): + """Should convert qualitative FBG values to numeric.""" + df = pl.DataFrame( + { + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003", "XX_YY004"], + "fbg_updated_mg": ["high", "medium", "low", "150"], + } + ) + + result = _apply_preprocessing(df) + + # high→200, medium→170, low→140 + assert result["fbg_updated_mg"].to_list() == ["200", "170", "140", "150"] + + def test_fbg_removes_dka_marker(self): + """Should attempt to remove (DKA) marker from FBG values.""" + df = pl.DataFrame( + { + "patient_id": ["XX_YY001"], + "fbg_updated_mg": ["350 (DKA)"], + } + ) + + result = _apply_preprocessing(df) + + # Note: Current implementation lowercases first, then tries to remove literal "(DKA)" + # which doesn't match lowercase "(dka)", so it's not actually removed + # This is a known issue but matches current behavior + assert result["fbg_updated_mg"][0] == "350 (dka)" + + +class TestYesNoHyphenReplacement: + """Tests for replacing '-' with 'N' in insulin-related Y/N columns.""" + + def test_replace_hyphen_in_insulin_columns(self): + """Should replace '-' with 'N' in analog insulin columns (2024+ trackers).""" + df = pl.DataFrame( + { + "patient_id": ["XX_YY001"], + "analog_insulin_long_acting": ["-"], + "analog_insulin_rapid_acting": ["-"], + } + ) + + result = _apply_preprocessing(df) + + assert result["analog_insulin_long_acting"][0] == "N" + assert result["analog_insulin_rapid_acting"][0] == "N" + + def test_preserve_hyphen_in_other_columns(self): + """Should NOT replace '-' in non-insulin Y/N columns.""" + df = pl.DataFrame( + { + "patient_id": ["XX_YY001"], + "clinic_visit": ["-"], + "active": ["-"], + } + ) + + result = _apply_preprocessing(df) + + # These columns are not in the insulin list, so '-' is preserved + assert result["clinic_visit"][0] == "-" + assert result["active"][0] == "-" + + +class TestFixAgeFromDob: + """Tests for age calculation from DOB.""" + + def test_calculates_age_from_dob(self): + """Should calculate age from DOB and tracker date.""" + df = pl.DataFrame( + { + "patient_id": ["P001"], + "age": [None], + "dob": [date(2010, 6, 15)], + "tracker_year": [2025], + "tracker_month": [1], + } + ) + collector = ErrorCollector() + + result = _fix_age_from_dob(df, collector) + + # 2025 - 2010 = 15, but Jan < June so 15 - 1 = 14 + assert result["age"][0] == 14 + + def test_birthday_already_passed(self): + """Should not subtract 1 if birthday already passed in tracker year.""" + df = pl.DataFrame( + { + "patient_id": ["P001"], + "age": [None], + "dob": [date(2010, 3, 15)], + "tracker_year": [2025], + "tracker_month": [6], + } + ) + collector = ErrorCollector() + + result = _fix_age_from_dob(df, collector) + + # 2025 - 2010 = 15, June > March so no adjustment + assert result["age"][0] == 15 + + def test_missing_dob_keeps_null(self): + """Should keep null age if DOB is missing.""" + df = pl.DataFrame( + { + "patient_id": ["P001"], + "age": [None], + "dob": pl.Series([None], dtype=pl.Date), + "tracker_year": [2025], + "tracker_month": [1], + } + ) + collector = ErrorCollector() + + result = _fix_age_from_dob(df, collector) + + assert result["age"][0] is None + + def test_error_date_dob_keeps_null(self): + """Should keep null age if DOB is error date.""" + error_date = date.fromisoformat(settings.error_val_date) + df = pl.DataFrame( + { + "patient_id": ["P001"], + "age": [None], + "dob": [error_date], + "tracker_year": [2025], + "tracker_month": [1], + } + ) + collector = ErrorCollector() + + result = _fix_age_from_dob(df, collector) + + assert result["age"][0] is None + + def test_corrects_wrong_excel_age(self): + """Should replace wrong Excel age with calculated age.""" + df = pl.DataFrame( + { + "patient_id": ["P001"], + "age": [99.0], # Wrong value from Excel + "dob": [date(2010, 6, 15)], + "tracker_year": [2025], + "tracker_month": [8], + } + ) + collector = ErrorCollector() + + result = _fix_age_from_dob(df, collector) + + # Should be corrected to 15 + assert result["age"][0] == 15 + + +class TestFixT1dDiagnosisAge: + """Tests for t1d_diagnosis_age calculation from DOB and diagnosis date.""" + + def test_calculates_diagnosis_age(self): + """Should calculate age at diagnosis from DOB and diagnosis date.""" + df = pl.DataFrame( + { + "patient_id": ["P001"], + "dob": [date(2005, 8, 20)], + "t1d_diagnosis_date": [date(2020, 3, 15)], + "t1d_diagnosis_age": [None], + } + ) + + result = _fix_t1d_diagnosis_age(df) + + # 2020 - 2005 = 15, but March < August so 15 - 1 = 14 + assert result["t1d_diagnosis_age"][0] == 14 + + def test_birthday_passed_before_diagnosis(self): + """Should not subtract 1 if birthday passed before diagnosis.""" + df = pl.DataFrame( + { + "patient_id": ["P001"], + "dob": [date(2005, 3, 20)], + "t1d_diagnosis_date": [date(2020, 8, 15)], + "t1d_diagnosis_age": [None], + } + ) + + result = _fix_t1d_diagnosis_age(df) + + # 2020 - 2005 = 15, August > March so no adjustment + assert result["t1d_diagnosis_age"][0] == 15 + + def test_diagnosis_before_birth_returns_null(self): + """A diagnosis date earlier than the birth date is a source + contradiction, so the derived age is meaningless (ticket 52). + + 2023 Likas Women & Children's records MY_LW004 as born 2016-08-17 and + diagnosed 2015-05-29; the workbook's own age formula resolves to + #NUM!. Deriving anyway emitted -2 into BigQuery. + """ + df = pl.DataFrame( + { + "patient_id": ["P001"], + "dob": [date(2016, 8, 17)], + "t1d_diagnosis_date": [date(2015, 5, 29)], + "t1d_diagnosis_age": [None], + } + ) + + result = _fix_t1d_diagnosis_age(df) + + assert result["t1d_diagnosis_age"][0] is None + + def test_missing_dob_returns_null(self): + """Should return null if DOB is missing.""" + df = pl.DataFrame( + { + "patient_id": ["P001"], + "dob": pl.Series([None], dtype=pl.Date), + "t1d_diagnosis_date": [date(2020, 3, 15)], + "t1d_diagnosis_age": [None], + } + ) + + result = _fix_t1d_diagnosis_age(df) + + assert result["t1d_diagnosis_age"][0] is None + + def test_missing_diagnosis_date_returns_null(self): + """Should return null if diagnosis date is missing.""" + df = pl.DataFrame( + { + "patient_id": ["P001"], + "dob": [date(2005, 8, 20)], + "t1d_diagnosis_date": pl.Series([None], dtype=pl.Date), + "t1d_diagnosis_age": [None], + } + ) + + result = _fix_t1d_diagnosis_age(df) + + assert result["t1d_diagnosis_age"][0] is None + + def test_error_date_dob_returns_null(self): + """Should return null if DOB is error date.""" + error_date = date.fromisoformat(settings.error_val_date) + df = pl.DataFrame( + { + "patient_id": ["P001"], + "dob": [error_date], + "t1d_diagnosis_date": [date(2020, 3, 15)], + "t1d_diagnosis_age": [None], + } + ) + + result = _fix_t1d_diagnosis_age(df) + + assert result["t1d_diagnosis_age"][0] is None + + def test_error_date_diagnosis_returns_null(self): + """Should return null if diagnosis date is error date.""" + error_date = date.fromisoformat(settings.error_val_date) + df = pl.DataFrame( + { + "patient_id": ["P001"], + "dob": [date(2005, 8, 20)], + "t1d_diagnosis_date": [error_date], + "t1d_diagnosis_age": [None], + } + ) + + result = _fix_t1d_diagnosis_age(df) + + assert result["t1d_diagnosis_age"][0] is None + + def test_replaces_excel_error_value(self): + """Should replace Excel error (#NUM!) that became 999999 with calculated value.""" + df = pl.DataFrame( + { + "patient_id": ["P001"], + "dob": [date(2005, 8, 20)], + "t1d_diagnosis_date": [date(2020, 3, 15)], + "t1d_diagnosis_age": [999999], # Error value from Excel + } + ) + + result = _fix_t1d_diagnosis_age(df) + + # Should be calculated as 14 + assert result["t1d_diagnosis_age"][0] == 14 + + def test_keeps_recorded_age_when_dates_missing(self): + """A real recorded diagnosis age must survive even if the dates + needed to recompute it don't parse: a value the clinic typed is not + discarded to null just because the arithmetic cannot run.""" + df = pl.DataFrame( + { + "patient_id": ["P001"], + "dob": pl.Series([None], dtype=pl.Date), + "t1d_diagnosis_date": pl.Series([None], dtype=pl.Date), + "t1d_diagnosis_age": [12], + } + ) + + result = _fix_t1d_diagnosis_age(df) + + assert result["t1d_diagnosis_age"][0] == 12 + + def test_keeps_recorded_age_over_disagreeing_calculation(self): + """A real recorded diagnosis age is trusted over date arithmetic even + when both dates parse and disagree with it -- the recorded value is + the clinic's own entry, not something to silently override.""" + df = pl.DataFrame( + { + "patient_id": ["P001"], + "dob": [date(2005, 8, 20)], + "t1d_diagnosis_date": [date(2020, 3, 15)], # would calculate to 14 + "t1d_diagnosis_age": [15], + } + ) + + result = _fix_t1d_diagnosis_age(df) + + assert result["t1d_diagnosis_age"][0] == 15 + + +class TestStripStringWhitespace: + """Ticket 36: whitespace at the ends of a string never carries meaning. + + Product's cleaning already stripped every string column (step 2.16) while + patient's did not, so the same tracker's `file_name` and `sheet_name` + disagreed between `patient_data_cleaned` and `product_data_cleaned` -- + silently dropping rows from any join between the two arms. Stripping runs + before validation, so a value is not rejected for whitespace alone. + """ + + def test_strips_identifier_columns(self): + df_raw = pl.DataFrame( + { + "file_name": ["2024_Some Hospital A4D Tracker - final "], + "sheet_name": ["Dec24 "], + "patient_id": ["MY_AB001"], + } + ) + + result = clean_patient_data(df_raw, ErrorCollector()) + + assert result["file_name"].to_list() == ["2024_Some Hospital A4D Tracker - final"] + assert result["sheet_name"].to_list() == ["Dec24"] + + def test_whitespace_does_not_defeat_allowed_value_validation(self): + df_raw = pl.DataFrame({"patient_id": ["MY_AB001"], "sheet_name": ["Jan24"], "sex": ["F "]}) + + result = clean_patient_data(df_raw, ErrorCollector()) + + assert result["sex"].to_list() == ["F"] + + +def test_apply_type_conversions_keeps_the_year_of_a_space_separated_date(): + """Stripping a trailing time component must not eat a real date's year. + + 2017-era trackers record diagnosis dates as "Jun 2006"; splitting on the + first space left "Jun", which dateutil then completed with the *current* + year, so the row was sentinelled as a future date and the real value lost. + """ + df = pl.DataFrame( + { + "file_name": ["t.xlsx", "t.xlsx"], + "patient_id": ["P1", "P2"], + "t1d_diagnosis_date": ["Jun 2006", "2009-04-17 00:00:00"], + } + ) + + result = _apply_type_conversions(df, ErrorCollector()) + + assert result["t1d_diagnosis_date"].to_list() == [date(2006, 6, 1), date(2009, 4, 17)] + + +class TestHeightRangeValidation: + """Height cm-to-m conversion must not rescue implausible source values. + + Conversion applies only above 50, so a value between 2.3 and 50 is neither + metres nor centimetres and falls out of the [0, 2.3] bound as an error. + Dividing it by 100 instead manufactures a plausible-looking metre reading + from an unusable cell -- 120 such cells were published as 0.069 metres + (ticket 55). + """ + + def _validate(self, df: pl.DataFrame) -> pl.DataFrame: + return _apply_range_validation(df, ErrorCollector()) + + def test_centimetre_height_is_converted_to_metres(self): + df = pl.DataFrame({"height": [135.5], "file_name": ["f"], "patient_id": ["p"]}) + + assert self._validate(df)["height"].to_list() == [1.355] + + def test_metre_height_is_kept(self): + df = pl.DataFrame({"height": [1.75], "file_name": ["f"], "patient_id": ["p"]}) + + assert self._validate(df)["height"].to_list() == [1.75] + + def test_value_between_the_two_units_becomes_the_error_value(self): + df = pl.DataFrame( + { + "height": [2.43, 6.9, 13.0], + "file_name": ["f", "f", "f"], + "patient_id": ["p", "p", "p"], + } + ) + + assert self._validate(df)["height"].to_list() == [settings.error_val_numeric] * 3 + + def test_bmi_is_derived_from_the_validated_height(self): + """An unusable height voids the BMI rather than producing one. + + Deriving BMI before the height bound gave 60 / 2.43^2 = 10.16, which + then passed its own bound and reached production.""" + df = pl.DataFrame( + { + "height": [2.43, 1.75], + "weight": [60.0, 70.0], + "bmi": [None, None], + "file_name": ["f", "f"], + "patient_id": ["p", "q"], + } + ) + + result = self._validate(df) + + assert result["bmi"][0] == settings.error_val_numeric + assert result["bmi"][1] == pytest.approx(70.0 / 1.75**2) + + +class TestInsulinSubtypeDerivation: + """The 2024+ template's five insulin columns are tick boxes, but one clinic + ticks them by writing the drug's name (ticket 55).""" + + def _derive(self, **cols: list[str | None]) -> pl.DataFrame: + return _derive_insulin_fields(pl.DataFrame(cols)) + + def test_ticked_columns_become_the_subtype_list(self): + result = self._derive( + human_insulin_pre_mixed=["-"], + human_insulin_short_acting=["-"], + human_insulin_intermediate_acting=["-"], + analog_insulin_rapid_acting=["Y"], + analog_insulin_long_acting=["Y"], + ) + + assert result["insulin_subtype"].to_list() == ["rapid-acting,long-acting"] + assert result["insulin_type"].to_list() == ["analog insulin"] + + def test_a_drug_name_in_a_tick_column_counts_as_ticked(self): + """2024 Sarawak writes Novorapid/Glargine where the template wants Y.""" + result = self._derive( + human_insulin_pre_mixed=["-"], + human_insulin_short_acting=["-"], + human_insulin_intermediate_acting=["-"], + analog_insulin_rapid_acting=["Novorapid"], + analog_insulin_long_acting=["Glargine"], + ) + + assert result["insulin_subtype"].to_list() == ["rapid-acting,long-acting"] + + def test_negative_markers_do_not_count_as_ticked(self): + result = self._derive( + human_insulin_pre_mixed=["0"], + human_insulin_short_acting=["N"], + human_insulin_intermediate_acting=["-"], + analog_insulin_rapid_acting=[""], + analog_insulin_long_acting=["-"], + ) + + assert result["insulin_subtype"].to_list() == [""] + + def test_no_tick_at_all_leaves_the_subtype_empty(self): + """Allowed-value validation turns the empty string into "Undefined". + + This overstates what the clinic recorded on 17,418 rows; nulling them + instead was measured and reverted (ticket 55), so the behaviour is + pinned here deliberately rather than left unexamined.""" + result = self._derive( + human_insulin_pre_mixed=["-"], + human_insulin_short_acting=["-"], + human_insulin_intermediate_acting=["-"], + analog_insulin_rapid_acting=["-"], + analog_insulin_long_acting=["-"], + ) + + assert result["insulin_subtype"].to_list() == [""] + + +class TestExtractDateFromMeasurement: + """The closing parenthesis is optional, because the source often omits it. + + The 2017/2018 Mandalay, CDA and Mahosot trackers write `180(May-2017` + without ever closing the bracket -- 25 of the 30 cells in this population + -- and requiring the closer lost the measurement date on every one. + """ + + @staticmethod + def _extract(value: str) -> tuple[str | None, str | None]: + df = pl.DataFrame({"fbg_updated_mg": [value]}) + out = _extract_date_from_measurement(df, "fbg_updated_mg") + return out["fbg_updated_mg"][0], out["fbg_updated_date"][0] + + def test_closed_parenthesis(self): + assert self._extract("8.53 (28/8/2017)") == ("8.53", "28/8/2017") + + def test_unclosed_parenthesis(self): + assert self._extract("180(May-2017") == ("180", "May-2017") + + def test_doubled_opening_parenthesis_uses_the_last_one(self): + """The greedy prefix consumes the first `(`, so the date starts after + the second. The stray `(` is stripped from the value rather than kept, + because leaving it fails the numeric cast and loses the 196.""" + assert self._extract("196((Dec-2017)") == ("196", "Dec-2017") + + def test_no_parenthesis_leaves_the_value_alone(self): + assert self._extract("3") == ("3", None) + + def test_units_inside_the_value_are_preserved(self): + assert self._extract("148 mg/dl (Mar-18)") == ("148 mg/dl", "Mar-18") + + +class TestBuddhistEraConversion: + """Tests for _convert_buddhist_era_dates (ticket 61). + + Thai clinics keep their workbooks in a Thai-locale Excel, so a date arrives + with a Buddhist-era year (BE = CE + 543). Before this conversion existed, + _validate_dates saw a year centuries in the future and clobbered the cell + with the 9999-09-09 sentinel -- 381 cells across the real corpus, destroyed + rather than published oddly. + """ + + @staticmethod + def _df(dates: list[date | None], tracker_year: int = 2024) -> pl.DataFrame: + n = len(dates) + return pl.DataFrame( + { + "patient_id": [f"TH_CM{i:03d}" for i in range(n)], + "file_name": ["t.xlsx"] * n, + "tracker_year": [tracker_year] * n, + "t1d_diagnosis_date": dates, + }, + schema={ + "patient_id": pl.String, + "file_name": pl.String, + "tracker_year": pl.Int32, + "t1d_diagnosis_date": pl.Date, + }, + ) + + def test_converts_a_buddhist_year_to_gregorian(self): + collector = ErrorCollector() + + result = _convert_buddhist_era_dates(self._df([date(2567, 11, 11)]), collector) + + assert result["t1d_diagnosis_date"].to_list() == [date(2024, 11, 11)] + assert len(collector) == 1 + err = collector.errors[0] + assert err.error_code == "buddhist_era_converted" + assert err.column == "t1d_diagnosis_date" + assert err.original_value == "2567-11-11" + + def test_leaves_a_gregorian_date_untouched(self): + collector = ErrorCollector() + + result = _convert_buddhist_era_dates(self._df([date(2024, 3, 1), None]), collector) + + assert result["t1d_diagnosis_date"].to_list() == [date(2024, 3, 1), None] + assert len(collector) == 0 + + def test_converts_a_year_that_predates_its_tracker(self): + """A diagnosis or screening date legitimately predates its tracker, so + patient needs no lower band at all -- 2022 Hat Yai's 2560-01-01 becomes + 2017-01-01, which ticket 40 independently derived from that patient's + own D.O.B., recruitment and age at diagnosis.""" + collector = ErrorCollector() + + result = _convert_buddhist_era_dates( + self._df([date(2560, 1, 1)], tracker_year=2022), collector + ) + + assert result["t1d_diagnosis_date"].to_list() == [date(2017, 1, 1)] + + def test_leaves_a_year_that_still_lands_in_the_future(self): + """3035 and 5025 (2025 CDA, 2025 Surat Thani) decode to 2492 and 4482 -- + no calendar makes those a recorded date, so they stay for _validate_dates + to sentinel and become source-defect findings instead.""" + collector = ErrorCollector() + + result = _convert_buddhist_era_dates( + self._df([date(3035, 3, 1), date(5025, 5, 19)], tracker_year=2025), collector + ) + + assert result["t1d_diagnosis_date"].to_list() == [date(3035, 3, 1), date(5025, 5, 19)] + assert len(collector) == 0 + + def test_leaves_a_leap_day_that_does_not_exist_once_shifted(self): + """543 is not a multiple of 4, so a BE leap day can land on a non-leap + Gregorian year. Converting would have to invent a date, so the cell is + left for the ordinary implausible-date handling.""" + collector = ErrorCollector() + + result = _convert_buddhist_era_dates( + self._df([date(2568, 2, 29)], tracker_year=2025), collector + ) + + assert result["t1d_diagnosis_date"].to_list() == [date(2568, 2, 29)] + assert len(collector) == 0 + + def test_leaves_the_parse_failure_sentinel_alone(self): + collector = ErrorCollector() + + result = _convert_buddhist_era_dates(self._df([date(9999, 9, 9)]), collector) + + assert result["t1d_diagnosis_date"].to_list() == [date(9999, 9, 9)] + assert len(collector) == 0 + + def test_conversion_runs_before_the_future_date_sentinel(self): + """The whole point: end to end, a Buddhist-era cell must reach output as + a date rather than as 9999-09-09.""" + collector = ErrorCollector() + df = self._df([date(2567, 11, 11)]) + + result = _validate_dates(_convert_buddhist_era_dates(df, collector), collector) + + assert result["t1d_diagnosis_date"].to_list() == [date(2024, 11, 11)] diff --git a/tests/test_clean/test_product.py b/tests/test_clean/test_product.py new file mode 100644 index 0000000..69190fc --- /dev/null +++ b/tests/test_clean/test_product.py @@ -0,0 +1,1078 @@ +"""Tests for product cleaning helpers.""" + +from datetime import date + +import polars as pl + +from a4d.clean.product import ( + YEAR_FLOOR_DELTA, + _check_entry_dates_match_sheet, + _clean_received_from, + _compute_running_balance, + _extract_balance_from_received, + _fill_product_names_and_sort, + _format_dates, + _null_entry_date_residues, + _split_multi_product_cells, + _switch_misplaced_columns, + _validate_entry_dates, + clean_product_data, +) +from a4d.errors import ErrorCollector + + +def _entry_date_df( + products: list[str], + entry_dates: list[date | None], + table_year: int = 2024, +) -> pl.DataFrame: + n = len(products) + return pl.DataFrame( + { + "product": products, + "product_entry_date": entry_dates, + "product_table_year": [table_year] * n, + "product_sheet_name": ["Jun"] * n, + "file_name": ["t.xlsx"] * n, + }, + schema={ + "product": pl.String, + "product_entry_date": pl.Date, + "product_table_year": pl.Int32, + "product_sheet_name": pl.String, + "file_name": pl.String, + }, + ) + + +def test_validate_entry_dates_flags_future_dates_within_window(): + df = _entry_date_df( + products=["P1", "P2", "P3"], + entry_dates=[date(2024, 6, 1), date(2099, 3, 15), None], + ) + collector = ErrorCollector() + + result = _validate_entry_dates(df, collector) + + parsed = result["product_entry_date"].to_list() + assert parsed[0] == date(2024, 6, 1) + assert parsed[1] == date(9999, 9, 9) + assert parsed[2] is None + assert len(collector) == 1 + err = collector.errors[0] + assert err.column == "product_entry_date" + assert err.error_code == "invalid_value" + assert err.patient_id == "P2" + + +def test_validate_entry_dates_converts_buddhist_era_dates_to_gregorian(): + """Ticket 61: a BE date is the clinic's calendar, not its error, so it is + converted like any other recovery -- and logged, so the conversion is + auditable rather than silent. The parse-failure sentinel stays put.""" + df = _entry_date_df( + products=["P1", "P2"], + entry_dates=[date(2567, 11, 11), date(9999, 9, 9)], + ) + collector = ErrorCollector() + + result = _validate_entry_dates(df, collector) + + parsed = result["product_entry_date"].to_list() + assert parsed[0] == date(2024, 11, 11) + assert parsed[1] == date(9999, 9, 9) + assert len(collector) == 1 + assert collector.errors[0].error_code == "buddhist_era_converted" + assert collector.errors[0].original_value == "2567-11-11" + + +def test_validate_entry_dates_leaves_a_buddhist_leap_day_that_cannot_shift(): + """543 is not a multiple of 4, so 2568-02-29 would have to become a + 29 February 2025 that does not exist. Left unconverted and sentinelled as + an implausible era date rather than invented.""" + df = _entry_date_df( + products=["P1"], + entry_dates=[date(2568, 2, 29)], + table_year=2025, + ) + collector = ErrorCollector() + + result = _validate_entry_dates(df, collector) + + assert result["product_entry_date"].to_list() == [date(9999, 9, 9)] + assert {e.error_code for e in collector.errors} == {"implausible_era_date"} + + +def test_validate_entry_dates_sentinels_far_future_outside_the_buddhist_band(): + """A year above BUDDHIST_ERA_THRESHOLD is only exempt if it is this + tracker's own Buddhist-era year. Ticket 32 found the blanket >= 2400 skip + let two corrupt Excel serials (1339576, 411384) reach BigQuery as + 5567-08-19 and 3026-04-30, and a Hat Yai typo publish 2525 in a 2025 + tracker whose Buddhist year is 2568.""" + df = _entry_date_df( + products=["P1", "P2", "P3"], + entry_dates=[date(2567, 11, 11), date(5567, 8, 19), date(2525, 10, 2)], + ) + collector = ErrorCollector() + + result = _validate_entry_dates(df, collector) + + parsed = result["product_entry_date"].to_list() + assert parsed[0] == date(2024, 11, 11) + assert parsed[1] == date(9999, 9, 9) + assert parsed[2] == date(9999, 9, 9) + assert len(collector) == 3 + assert {e.patient_id for e in collector.errors if e.error_code == "implausible_era_date"} == { + "P2", + "P3", + } + + +def test_validate_entry_dates_converts_at_both_buddhist_band_edges(): + """The band mirrors the Gregorian window: BE equivalents of + [table_year - YEAR_FLOOR_DELTA, table_year]. Every year inside it converts; + the edges are where an off-by-one would show.""" + df = _entry_date_df( + products=["P1", "P2"], + entry_dates=[date(2024 + 543 - YEAR_FLOOR_DELTA, 1, 1), date(2024 + 543, 12, 31)], + ) + collector = ErrorCollector() + + result = _validate_entry_dates(df, collector) + + assert result["product_entry_date"].to_list() == [ + date(2024 - YEAR_FLOOR_DELTA, 1, 1), + date(2024, 12, 31), + ] + assert {e.error_code for e in collector.errors} == {"buddhist_era_converted"} + + +def test_validate_entry_dates_does_not_relog_the_sentinel(): + """9999-09-09 is the pipeline's own parse-failure sentinel, so re-flagging + it would double-log a cell some earlier step already reported.""" + df = _entry_date_df(products=["P1"], entry_dates=[date(9999, 9, 9)]) + collector = ErrorCollector() + + result = _validate_entry_dates(df, collector) + + assert result["product_entry_date"].to_list() == [date(9999, 9, 9)] + assert len(collector) == 0 + + +def test_validate_entry_dates_missing_columns_is_noop(): + df = pl.DataFrame({"product": ["P1"]}) + collector = ErrorCollector() + + result = _validate_entry_dates(df, collector) + + assert result.equals(df) + assert len(collector) == 0 + + +def test_validate_entry_dates_logs_year_floor_but_preserves_date(): + """Year-floor violations (parsed.year < tracker_year - YEAR_FLOOR_DELTA) are + logged via error_collector but the parsed date is preserved in + product_entry_date. Sentinelling it instead would drop the row out of + chronological order and distort the running balance accumulated in step + 2.15, so the date is preserved and the audit trail carries the flag.""" + df = _entry_date_df( + products=["P1", "P2", "P3"], + entry_dates=[date(2024, 6, 1), date(1967, 2, 5), None], + ) + collector = ErrorCollector() + + result = _validate_entry_dates(df, collector) + + parsed = result["product_entry_date"].to_list() + assert parsed[0] == date(2024, 6, 1) + assert parsed[1] == date(1967, 2, 5) + assert parsed[2] is None + assert len(collector) == 1 + err = collector.errors[0] + assert err.column == "product_entry_date" + assert err.error_code == "invalid_value" + assert err.patient_id == "P2" + assert "before" in err.error_message + + +def test_null_entry_date_residues_nulls_amount_left_marker(): + """End-of-block "Amount Left" markers (case-insensitive, whitespace-trimmed) + are nulled before parsing. Regression for 2018 Mahosot Nov18/Dec18 where + six rows per sheet had "Amount Left" in the entry_date column, surfacing + as 9999-09-09 parse-failure sentinels in cleaned output.""" + df = pl.DataFrame( + { + "product_entry_date": [ + "Amount Left", + "amount left", + " AMOUNT LEFT ", + "2018-12-05", + ], + }, + schema={"product_entry_date": pl.String}, + ) + + out = _null_entry_date_residues(df) + + assert out["product_entry_date"].to_list() == [None, None, None, "2018-12-05"] + + +def test_null_entry_date_residues_nulls_tiny_excel_serials(): + """Numeric cells below the 2000-01-01 floor (Excel serial 36526) are + nulled. Regression for 2025 NPH Jul25/Aug25/Dec25 row 81 where a stray + "30" surfaced as 1900-01-29 via the pre-Mar-1900 leap-year-bug epoch.""" + df = pl.DataFrame( + { + "product_entry_date": ["30", "59", "366", "36525", "36526", "45000"], + }, + schema={"product_entry_date": pl.String}, + ) + + out = _null_entry_date_residues(df) + + parsed = out["product_entry_date"].to_list() + assert parsed[0] is None # 30 — within Excel leap-year-bug range + assert parsed[1] is None # 59 — top of leap-year-bug range + assert parsed[2] is None # 366 — first day of 1901 + assert parsed[3] is None # 36525 — 1999-12-31, just below floor + assert parsed[4] == "36526" # 2000-01-01 — at floor, preserved + assert parsed[5] == "45000" # ~2023, well above floor + + +def test_null_entry_date_residues_preserves_real_dates(): + """Standard date string formats survive the residue scrub unchanged. + Regression guard: marker / tiny-serial detection must not catch + legitimate inputs.""" + df = pl.DataFrame( + { + "product_entry_date": [ + "2018-12-05", + "05/12/2018", + "Mar-18", + "24 Feb 2020", + None, + ], + }, + schema={"product_entry_date": pl.String}, + ) + + out = _null_entry_date_residues(df) + + assert out["product_entry_date"].to_list() == [ + "2018-12-05", + "05/12/2018", + "Mar-18", + "24 Feb 2020", + None, + ] + + +def test_null_entry_date_residues_missing_column_is_noop(): + df = pl.DataFrame({"product": ["P1"]}) + out = _null_entry_date_residues(df) + assert out.equals(df) + + +def test_format_dates_residue_cells_become_null_after_parsing(): + """End-to-end: residue cells flowing through _format_dates emerge as + NULL in the parsed Date column, not 9999-09-09 or 1900-01-29.""" + df = pl.DataFrame( + { + "product": ["P1", "P2", "P3", "P4", "P5"], + "product_entry_date": [ + "Amount Left", + "30", + "2018-12-05", + "45000", + None, + ], + "file_name": ["t.xlsx"] * 5, + }, + schema={ + "product": pl.String, + "product_entry_date": pl.String, + "file_name": pl.String, + }, + ) + collector = ErrorCollector() + + out = _format_dates(df, collector) + + parsed = out["product_entry_date"].to_list() + assert parsed[0] is None # "Amount Left" — nulled + assert parsed[1] is None # tiny serial 30 — nulled + assert parsed[2] == date(2018, 12, 5) + assert parsed[3] is not None # 45000 → real post-2000 date + assert parsed[3].year >= 2000 + assert parsed[4] is None + # No parse-failure errors logged because residue cells were nulled + # before parse_date_flexible saw them. + assert all(err.error_code != "invalid_value" for err in collector.errors) + + +def test_switch_misplaced_columns_scoped_per_sheet(): + """Swap fires only for sheets with 'Remaining Stock'; clean sheets in + the same file pass through untouched (regression for 2018_Penang_DC).""" + df = pl.DataFrame( + { + "product_sheet_name": ["Jul18", "Jul18", "Nov18", "Nov18"], + "product": ["A", "A", "A", "A"], + "product_units_received": ["70", None, "Remaining Stock", None], + "product_received_from": ["DKSH", None, "72", None], + }, + schema={ + "product_sheet_name": pl.String, + "product": pl.String, + "product_units_received": pl.String, + "product_received_from": pl.String, + }, + ) + + out = _switch_misplaced_columns(df) + + jul = out.filter(pl.col("product_sheet_name") == "Jul18") + assert jul["product_units_received"].to_list() == ["70", None] + assert jul["product_received_from"].to_list() == ["DKSH", None] + + nov = out.filter(pl.col("product_sheet_name") == "Nov18") + assert nov["product_units_received"].to_list() == ["72", None] + assert nov["product_received_from"].to_list() == ["Remaining Stock", None] + + +def test_extract_balance_from_received_scoped_per_sheet(): + """Rewrite fires only for sheets containing 'Balance' markers; clean + sheets in the same file pass through untouched (regression for 2018 + Preah Kossamak / 2019 Sultanah Bahiyah_DC / 2020 LFHC_DC, where Aug18 + donor names were blanked because Sep18+ used the Balance convention).""" + df = pl.DataFrame( + { + "product_sheet_name": ["Aug18", "Sep18", "Sep18"], + "product": ["A", "A", "A"], + "product_units_received": ["20", "Start Balance", None], + "product_units_released": [None, "2", None], + "product_received_from": ["GE100", None, None], + }, + schema={ + "product_sheet_name": pl.String, + "product": pl.String, + "product_units_received": pl.String, + "product_units_released": pl.String, + "product_received_from": pl.String, + }, + ) + + out = _extract_balance_from_received(df) + + aug = out.filter(pl.col("product_sheet_name") == "Aug18") + assert aug["product_received_from"].to_list() == ["GE100"] + assert aug["product_units_released"].to_list() == [None] + + sep = out.filter(pl.col("product_sheet_name") == "Sep18") + assert sep["product_received_from"].to_list() == ["2", None] + assert sep["product_units_released"].to_list() == [None, None] + + +def test_extract_balance_from_received_clears_released_on_all_balance_sheet(): + """Sheets where every row is a Balance marker still get units_released + cleared. Regression for 2019 PKH Oct19 / 2020 PKH May20 (18 rows): the + sheet_triggered predicate uses received_from.is_null().any().over(sheet), + which flips False after the first .with_columns() populates received_from + on every Balance row — so without caching, the second call's released- + clear pass becomes a no-op.""" + df = pl.DataFrame( + { + "product_sheet_name": ["Oct19", "Oct19"], + "product": ["GE100", "GE100"], + "product_units_received": ["Start Balance", "End Balance"], + "product_units_released": ["1", "1"], + "product_received_from": [None, None], + }, + schema={ + "product_sheet_name": pl.String, + "product": pl.String, + "product_units_received": pl.String, + "product_units_released": pl.String, + "product_received_from": pl.String, + }, + ) + + out = _extract_balance_from_received(df) + + assert out["product_received_from"].to_list() == ["1", "1"] + assert out["product_units_released"].to_list() == [None, None] + + +def test_extract_balance_from_received_preserves_supplier_on_triggered_sheet(): + """Non-Balance rows on a triggered sheet keep their supplier name. + Regression for the Mahosot 2020 DKSH stock-receipt rows: the + ``(?i)Balance`` trigger substring-matches "START BALANCE" / "END BALANCE", + firing on every standard sheet. Before the fix, the case_when's catch-all + nulled ``product_received_from`` on every non-Balance row in the triggered + sheet, blanking legitimate supplier names. After the fix the catch-all + preserves received_from.""" + df = pl.DataFrame( + { + "product_sheet_name": ["Mar20", "Mar20", "Mar20"], + "product": ["Performa", "Performa", "Performa"], + "product_units_received": ["START BALANCE", "100", "END BALANCE"], + "product_units_released": [None, None, None], + "product_received_from": ["43", "DKSH", "87"], + }, + schema={ + "product_sheet_name": pl.String, + "product": pl.String, + "product_units_received": pl.String, + "product_units_released": pl.String, + "product_received_from": pl.String, + }, + ) + + out = _extract_balance_from_received(df) + + # Balance-marker rows pass through unchanged (received_from non-null, + # so no released value to relocate; arm 2 is a no-op rewrite). + # Change row in the middle keeps DKSH instead of being nulled. + assert out["product_received_from"].to_list() == ["43", "DKSH", "87"] + # units_released stays null on all rows (was null going in). + assert out["product_units_released"].to_list() == [None, None, None] + + +def test_extract_balance_from_received_nulls_total_subtotal_label(): + """Typist subtotal-row labels ("Total" in product_received_from) are + nulled, while legitimate supplier names on the same triggered sheet + survive. Regression for the 47 corpus-wide subtotal rows in 2019 + Penang DC / VNCH / Mandalay etc. that the supplier-preservation fix + inadvertently kept. "Total" is the label the typist puts on the + end-of-product-block subtotal row, never a supplier.""" + # Sheet must be "triggered": need at least one Balance marker AND at + # least one row with null received_from. Last row supplies the null. + df = pl.DataFrame( + { + "product_sheet_name": ["Mar20", "Mar20", "Mar20", "Mar20", "Mar20"], + "product": ["Performa", "Performa", "Performa", "Performa", "Performa"], + "product_units_received": ["START BALANCE", "100", "0", "END BALANCE", "50"], + "product_units_released": [None, None, None, None, None], + "product_received_from": ["43", "DKSH", "Total", "87", None], + }, + schema={ + "product_sheet_name": pl.String, + "product": pl.String, + "product_units_received": pl.String, + "product_units_released": pl.String, + "product_received_from": pl.String, + }, + ) + + out = _extract_balance_from_received(df) + + # DKSH (legitimate supplier) survives; Total (subtotal label) is nulled; + # null stays null; balance markers' received_from is unchanged. + assert out["product_received_from"].to_list() == ["43", "DKSH", None, "87", None] + + +def test_clean_received_from_copies_start_balance_and_strips_numeric_supplier(): + """Pin _clean_received_from's two paths: (1) START rows copy + received_from into product_balance; non-START rows preserve their + existing balance (defensive — earlier code wiped them, which only + happened to be a no-op because nothing populates product_balance + before this step). (2) numeric-only product_received_from is nulled + while alphabetic / mixed-alphanumeric values survive.""" + df = pl.DataFrame( + { + "product_units_received": ["START", "5", "3", "2"], + "product_received_from": ["DKSH", "DKSH", "43", "DKSH 43"], + "product_balance": [None, None, None, None], + "product_sheet_name": ["Jan", "Jan", "Jan", "Jan"], + }, + schema={ + "product_units_received": pl.String, + "product_received_from": pl.String, + "product_balance": pl.String, + "product_sheet_name": pl.String, + }, + ) + + out = _clean_received_from(df) + + # START row gets received_from copied into balance; other rows keep their + # original (null in production after apply_schema seed). + assert out["product_balance"].to_list() == ["DKSH", None, None, None] + # Alpha-strip on received_from: numeric-only "43" → null, alpha + mixed survive. + assert out["product_received_from"].to_list() == ["DKSH", "DKSH", None, "DKSH 43"] + + +def test_clean_received_from_no_start_preserves_balance(): + """When no row contains START, product_balance must be untouched. + Only the alpha-strip on received_from runs.""" + df = pl.DataFrame( + { + "product_units_received": ["5", "3"], + "product_received_from": ["DKSH", "43"], + "product_balance": ["100", "200"], # pre-existing sentinels + "product_sheet_name": ["Jan", "Jan"], + }, + schema={ + "product_units_received": pl.String, + "product_received_from": pl.String, + "product_balance": pl.String, + "product_sheet_name": pl.String, + }, + ) + + out = _clean_received_from(df) + + assert out["product_balance"].to_list() == ["100", "200"] + assert out["product_received_from"].to_list() == ["DKSH", None] + + +def test_clean_received_from_preserves_balance_on_sheets_without_start(): + """Two-sheet frame, START only in sheet A. Sheet B's pre-existing + product_balance must be preserved — a START in one sheet must not + propagate balance assignments (or wipes) to other sheets.""" + df = pl.DataFrame( + { + "product_units_received": ["START", "5", "3", "2"], + "product_received_from": ["DKSH", "DKSH", "Acme", "BMS"], + "product_balance": [None, None, "50", "75"], + "product_sheet_name": ["Jan", "Jan", "Feb", "Feb"], + }, + schema={ + "product_units_received": pl.String, + "product_received_from": pl.String, + "product_balance": pl.String, + "product_sheet_name": pl.String, + }, + ) + + out = _clean_received_from(df) + + # Sheet Jan: START row copies received_from to balance; other Jan row keeps null. + # Sheet Feb: no START → both rows preserve their existing balance ("50", "75"). + assert out["product_balance"].to_list() == ["DKSH", None, "50", "75"] + + +def test_format_dates_normalizes_separator_typos(): + """Cells with non-standard separators (period after day, underscore + before year, repeated dashes) parse to the intended date after + pre-normalization (regression for 2020 CDA Feb20, 2024 Likas Jan24, + 2022 Mukdahan Apr22). The Excel-datetime time-strip ('YYYY-MM-DD + HH:MM:SS') and the canonical 'dd-Mon-yyyy' format must continue to + parse correctly.""" + df = pl.DataFrame( + { + "product": ["P1", "P2", "P3", "P4", "P5"], + "product_entry_date": [ + "24.Feb 2020", # CDA period-after-day typo + "22-Jan_2024", # Likas underscore-before-year typo + "16-Apr--2022", # Mukdahan double-dash typo + "2020-02-24 00:00:00", # Excel datetime cast — time must strip + "16-Apr-2022", # canonical dd-Mon-yyyy — must parse + ], + }, + schema={ + "product": pl.String, + "product_entry_date": pl.String, + }, + ) + + out = _format_dates(df, ErrorCollector()) + + assert out["product_entry_date"].to_list() == [ + date(2020, 2, 24), + date(2024, 1, 22), + date(2022, 4, 16), + date(2020, 2, 24), + date(2022, 4, 16), + ] + + +def test_format_dates_preserves_year_typo_sentinels(): + """Year typos (real Excel datetime with wrong year, 5-digit year + strings) must continue to fail parsing — separator normalization + must not accidentally recover them. Mahosot's datetime(2009,12,4) + parses to a real 2009 date (Python validates it later via + _validate_entry_dates); NPH's 5-digit year falls through to the + error path.""" + df = pl.DataFrame( + { + "product": ["Mahosot", "NPH"], + "product_entry_date": [ + "2009-12-04 00:00:00", # Mahosot Excel datetime, year typo + "1 jun 20203", # NPH 5-digit year + ], + }, + schema={ + "product": pl.String, + "product_entry_date": pl.String, + }, + ) + + out = _format_dates(df, ErrorCollector()) + + parsed = out["product_entry_date"].to_list() + # Mahosot: real datetime parses to 2009-12-04 (year-floor catches later). + assert parsed[0] == date(2009, 12, 4) + # NPH: 5-digit year is unparseable; parse_date_column emits a sentinel + # error_val_date or None — either way, not a valid 2020/2023 date. + assert parsed[1] is None or parsed[1] != date(2023, 6, 1) + assert parsed[1] != date(2020, 6, 1) + + +def test_split_multi_product_cells_extracts_box_count_to_units_received(): + """Pins the \\d+ count regex: a count starting with the digit '1' + followed by '0' must be extracted whole as '10', not '1'.""" + df = pl.DataFrame( + { + "product": ["Accu-Check Strips (10 box)"], + "product_received_from": ["DKSH"], + "product_released_to": [None], + "product_units_received": [None], + "product_units_released": [None], + }, + schema={ + "product": pl.String, + "product_received_from": pl.String, + "product_released_to": pl.String, + "product_units_received": pl.String, + "product_units_released": pl.String, + }, + ) + + out = _split_multi_product_cells(df) + + assert out["product_units_notes"].to_list() == ["10 box"] + assert out["product_units_received"].to_list() == ["10"] + assert out["product_units_released"].to_list() == [None] + + +def test_split_multi_product_cells_extracts_unit_count_to_released_when_released_to_set(): + """Same shape as the box test but routes the count to product_units_released + when product_released_to is set and product_received_from is null.""" + df = pl.DataFrame( + { + "product": ["Accu-Check Strips (10 unit)"], + "product_received_from": [None], + "product_released_to": ["KD_EW001"], + "product_units_received": [None], + "product_units_released": [None], + }, + schema={ + "product": pl.String, + "product_received_from": pl.String, + "product_released_to": pl.String, + "product_units_received": pl.String, + "product_units_released": pl.String, + }, + ) + + out = _split_multi_product_cells(df) + + assert out["product_units_notes"].to_list() == ["10 unit"] + assert out["product_units_released"].to_list() == ["10"] + assert out["product_units_received"].to_list() == [None] + + +def test_misplaced_datetime_cell_in_units_received_logs_and_recodes_to_zero(): + """Pin the handling of a date typo'd into the units_received column. + + Reading the cell's underlying Excel serial (43644.0 for 2019-06-28) would + pollute the balance trajectory by tens of thousands. Instead the value + stringifies, the Float64 cast in step 2.11 fails, and step 2.12 recodes the + null to 0.0. Step 2.11 also emits one ``type_conversion`` ErrorCollector + entry, so the discarded cell is recoverable from the log. + Regression for the 22 real-divergence rows in + 2019_Sultanah Bahiyah_DC / 2019_Penang_DC / 2020 Mandalay + surfaced by Ali_internship/product_balance_diff_v1.ipynb.""" + df = pl.DataFrame( + { + "file_name": ["2019_SB.xlsx"], + "product_sheet_name": ["Jun19"], + "product": ["Accu-Chek Performa Test Strip"], + "product_entry_date": [None], + "product_units_received": ["2019-06-28 00:00:00"], # typo cell + "product_units_released": [None], + "product_received_from": [None], + "product_released_to": [None], + "product_units_returned": [None], + "product_returned_by": [None], + "product_balance": [None], + "product_table_month": ["06"], + "product_table_year": [2019.0], + "product_balance_status": [None], + "product_category": [None], + "product_unit_capacity": [None], + "product_units_notes": [None], + "orig_product_released_to": [None], + "product_remarks": [None], + }, + schema={ + "file_name": pl.String, + "product_sheet_name": pl.String, + "product": pl.String, + "product_entry_date": pl.String, + "product_units_received": pl.String, + "product_units_released": pl.String, + "product_received_from": pl.String, + "product_released_to": pl.String, + "product_units_returned": pl.String, + "product_returned_by": pl.String, + "product_balance": pl.String, + "product_table_month": pl.String, + "product_table_year": pl.Float64, + "product_balance_status": pl.String, + "product_category": pl.String, + "product_unit_capacity": pl.String, + "product_units_notes": pl.String, + "orig_product_released_to": pl.String, + "product_remarks": pl.String, + }, + ) + + collector = ErrorCollector() + out = clean_product_data(df, collector) + + assert out["product_units_received"].to_list() == [0.0] + + units_errors = [ + e + for e in collector.errors + if e.column == "product_units_received" and e.function_name == "_clean_units_received" + ] + assert len(units_errors) == 1 + err = units_errors[0] + assert err.error_code == "type_conversion" + assert err.original_value == "2019-06-28 00:00:00" + assert err.file_name == "2019_SB.xlsx" + assert err.patient_id == "Accu-Chek Performa Test Strip" + + +def test_running_balance_eliminates_float_residue(): + """Pin .round(10) on the cumsum result. Without it, Python's vectorized + cumsum yields 2.220446e-16 instead of 0 on rows where the running delta + sum hits a non-binary-clean target (e.g. 1.8 - 0.4 - 1.4 = 2.22e-16 + because 0.4+1.4 in float64 is 1.7999999999999998). Regression for the 600 + FP-noise rows surfaced + by Ali_internship/product_balance_diff_v3.ipynb (V3 corpus example: + 2021_Lao Friends, Feb21, Mixtard 30 Penfill 3ml (5s)).""" + df = pl.DataFrame( + { + "product_sheet_name": ["Feb21"] * 4, + "product": ["Mixtard"] * 4, + "product_balance": [1.8, None, None, None], + "product_balance_status": ["start", "change", "change", "end"], + "product_units_received": [0.0, 0.0, 0.0, 0.0], + "product_units_released": [0.0, 0.4, 1.4, 0.0], + "product_units_returned": [0.0, 0.0, 0.0, 0.0], + "product_table_year": [2020, 2020, 2020, 2020], # < 2021 to skip end-row zeroing + }, + schema={ + "product_sheet_name": pl.String, + "product": pl.String, + "product_balance": pl.Float64, + "product_balance_status": pl.String, + "product_units_received": pl.Float64, + "product_units_released": pl.Float64, + "product_units_returned": pl.Float64, + "product_table_year": pl.Int32, + }, + ) + + out = _compute_running_balance(df) + + # Strict equality — without .round(10) the 3rd row would be 2.22e-16. + assert out["product_balance"].to_list() == [1.8, 1.4, 0.0, 0.0] + + +def test_fill_product_names_and_sort_treats_sentinel_as_null_for_rank(): + """Pin the sentinel-rank fix. The 9999-09-09 parse-failure sentinel must + be treated as null when computing the per-(sheet, product) rank, so it + falls into the 'preserve input order' branch rather than sorting to the + dense_d+1 end-of-changes position. Real-world driver: the 11 Tier-2 rows + in product_balance documented in product_balance_investigation.md: the + sentinel was sorting after every valid date, distorting the cumulative + balance order.""" + df = pl.DataFrame( + { + "product_sheet_name": ["Jun24"] * 5, + "product": ["P"] * 5, + "product_entry_date": [ + None, + date(2024, 1, 15), + date(9999, 9, 9), + date(2024, 3, 15), + None, + ], + "product_balance_status": ["start", "change", "change", "change", "end"], + "product_table_month": [6] * 5, + "index": [1, 2, 3, 4, 5], + }, + schema={ + "product_sheet_name": pl.String, + "product": pl.String, + "product_entry_date": pl.Date, + "product_balance_status": pl.String, + "product_table_month": pl.Int32, + "index": pl.Int64, + }, + ) + + out = _fill_product_names_and_sort(df) + + # After fix: sentinel sits between the two valid mid-rows in input order + # (rank=row_n=3 ties with rank for 2024-03-15=dense_d+1=3, stable sort + # keeps the sentinel at input position 3, valid date at position 4). + statuses = out["product_balance_status"].to_list() + assert statuses == ["start", "change", "change", "change", "end"] + dates = out["product_entry_date"].to_list() + assert dates[0] is None # start + assert dates[1] == date(2024, 1, 15) # earlier valid date + assert dates[2] == date(9999, 9, 9) # sentinel — between A and B + assert dates[3] == date(2024, 3, 15) # later valid date + assert dates[4] is None # end + + +def _check_dates_df( + entry_dates: list[date | None], + table_month: int = 6, + table_year: int = 2024, +) -> pl.DataFrame: + """Fixture for _check_entry_dates_match_sheet. Mirrors the post-2.6 schema: + table_month is still String, table_year is still Float64 (the schema cast + at step 2.16 hasn't run yet).""" + n = len(entry_dates) + return pl.DataFrame( + { + "product": [f"P{i}" for i in range(n)], + "product_entry_date": entry_dates, + "product_table_year": [float(table_year)] * n, + "product_table_month": [f"{table_month:02d}"] * n, + "product_sheet_name": [f"{table_month:02d}"] * n, + "file_name": ["t.xlsx"] * n, + }, + schema={ + "product": pl.String, + "product_entry_date": pl.Date, + "product_table_year": pl.Float64, + "product_table_month": pl.String, + "product_sheet_name": pl.String, + "file_name": pl.String, + }, + ) + + +def test_check_entry_dates_logs_month_mismatch(): + df = _check_dates_df( + entry_dates=[date(2024, 6, 15), date(2024, 3, 15)], + table_month=6, + table_year=2024, + ) + collector = ErrorCollector() + + _check_entry_dates_match_sheet(df, collector) + + assert len(collector) == 1 + err = collector.errors[0] + assert err.column == "product_entry_date" + assert err.error_code == "invalid_value" + assert err.function_name == "check_entry_dates" + assert err.patient_id == "P1" + + +def test_check_entry_dates_logs_year_mismatch(): + df = _check_dates_df( + entry_dates=[date(2023, 6, 15)], # year mismatch even though month matches + table_month=6, + table_year=2024, + ) + collector = ErrorCollector() + _check_entry_dates_match_sheet(df, collector) + assert len(collector) == 1 + + +def test_check_entry_dates_skips_sentinel_buddhist_and_null(): + df = _check_dates_df( + entry_dates=[ + None, # null — skip + date(9999, 9, 9), # parse-failure sentinel — skip + date(2567, 11, 11), # Buddhist-era — skip + date(2024, 6, 15), # match — skip + ], + table_month=6, + table_year=2024, + ) + collector = ErrorCollector() + _check_entry_dates_match_sheet(df, collector) + assert len(collector) == 0 + + +def test_check_entry_dates_no_log_on_match(): + df = _check_dates_df( + entry_dates=[date(2024, 6, 1), date(2024, 6, 30)], + table_month=6, + table_year=2024, + ) + collector = ErrorCollector() + _check_entry_dates_match_sheet(df, collector) + assert len(collector) == 0 + + +def test_fat_finger_future_falls_into_input_order_rank_after_validation(): + """Pin the deliberate _validate_entry_dates × _fill_product_names_and_sort + interaction. `2099-03-15` is clobbered to the 9999-09-09 sentinel inside + _validate_entry_dates, and the sentinel is then treated as null when + ranking -- so a fat-fingered year holds its input position instead of + sorting to the end of the ledger and distorting the running balance. + + The test exists so any future revert of either _validate_entry_dates' + future-year guard or the sentinel-rank fix in + _fill_product_names_and_sort breaks loudly.""" + collector = ErrorCollector() + df = pl.DataFrame( + { + "product": ["P", "P", "P", "P"], + "product_entry_date": [None, date(2099, 3, 15), date(2024, 6, 1), None], + "product_balance_status": ["start", "change", "change", "end"], + "product_table_year": [2024] * 4, + "product_table_month": [6] * 4, + "product_sheet_name": ["Jun24"] * 4, + "file_name": ["t.xlsx"] * 4, + "index": [1, 2, 3, 4], + }, + schema={ + "product": pl.String, + "product_entry_date": pl.Date, + "product_balance_status": pl.String, + "product_table_year": pl.Int32, + "product_table_month": pl.Int32, + "product_sheet_name": pl.String, + "file_name": pl.String, + "index": pl.Int64, + }, + ) + + validated = _validate_entry_dates(df, collector) + out = _fill_product_names_and_sort(validated) + + # _validate_entry_dates rewrote the 2099 future to the sentinel. + assert validated["product_entry_date"].to_list()[1] == date(9999, 9, 9) + assert len(collector) == 1 + + # In the sorted output, the (now-sentinel) row sits at its input position + # 2 — not the dense-rank tail. Statuses confirm row ordering. + assert out["product_balance_status"].to_list() == [ + "start", + "change", + "change", + "end", + ] + dates = out["product_entry_date"].to_list() + assert dates[0] is None # start + assert dates[1] == date(9999, 9, 9) # was 2099-03-15, now sentinel-as-null + assert dates[2] == date(2024, 6, 1) # valid mid-row + assert dates[3] is None # end + + +def test_clean_product_data_handles_columnless_raw_frame(): + """Trackers from pre-product-tracking years extract to a 0-column, + 0-row frame (no product section in any sheet). Cleaning must hand back + an empty, schema-conformant frame rather than crash on a missing + "product" column (ticket 14).""" + df_raw = pl.DataFrame() + collector = ErrorCollector() + + out = clean_product_data(df_raw, collector) + + assert out.height == 0 + assert out.width == 20 + assert "product" in out.columns + assert len(collector) == 0 + + +def _balance_reconciliation_frame(source_closing, released, status=None): + """Fixture for the closing-balance reconciliation check (ticket 36). + + Two transactions off a start balance of 100, with the source's own + recorded closing balance supplied by the caller so a test can make it + agree or disagree with what step 2.15 computes. + """ + n = 3 + return pl.DataFrame( + { + "file_name": ["2024_Test.xlsx"] * n, + "index": [1, 2, 3], + "product_sheet_name": ["Jan24"] * n, + "product": ["Test Strips"] * n, + "product_balance": [100.0, None, source_closing], + "product_balance_status": status or ["start", "change", "end"], + "product_units_received": [0.0, 0.0, 0.0], + "product_units_released": released, + "product_units_returned": [0.0] * n, + "product_table_year": [2020] * n, + }, + schema={ + "file_name": pl.String, + "index": pl.Int64, + "product_sheet_name": pl.String, + "product": pl.String, + "product_balance": pl.Float64, + "product_balance_status": pl.String, + "product_units_received": pl.Float64, + "product_units_released": pl.Float64, + "product_units_returned": pl.Float64, + "product_table_year": pl.Int32, + }, + ) + + +def _reconciliation_errors(collector): + return [e for e in collector.errors if e.error_code == "balance_reconciliation"] + + +def test_running_balance_logs_when_closing_disagrees_with_source(): + """Ticket 36: the running balance is recomputed from the start balance plus + transactions, in Python's chronological row order rather than the source's + data-entry order, so intermediate balances legitimately differ from the + tracker's. The *closing* balance is order-independent, so a disagreement + there is a real source arithmetic or data-entry problem and must surface.""" + # 100 - 8 - 4 = 88 computed, but the tracker recorded 90. + df = _balance_reconciliation_frame(source_closing=90.0, released=[0.0, 8.0, 4.0]) + collector = ErrorCollector() + + out = _compute_running_balance(df, collector) + + assert out["product_balance"].to_list() == [100.0, 92.0, 88.0] + errors = _reconciliation_errors(collector) + assert len(errors) == 1 + assert errors[0].column == "product_balance" + assert errors[0].function_name == "_compute_running_balance" + assert "90" in errors[0].error_message + assert "88" in errors[0].error_message + + +def test_running_balance_silent_when_closing_matches_source(): + """No warning when the recomputed ledger lands where the tracker says.""" + df = _balance_reconciliation_frame(source_closing=88.0, released=[0.0, 8.0, 4.0]) + collector = ErrorCollector() + + _compute_running_balance(df, collector) + + assert _reconciliation_errors(collector) == [] + + +def test_running_balance_silent_when_source_records_no_closing_balance(): + """Most groups record only a start balance and leave the rest blank -- that + is not a disagreement, and warning on it would drown the real signal.""" + df = _balance_reconciliation_frame(source_closing=None, released=[0.0, 8.0, 4.0]) + collector = ErrorCollector() + + _compute_running_balance(df, collector) + + assert _reconciliation_errors(collector) == [] + + +def test_running_balance_reconciliation_is_optional(): + """Callers without an ErrorCollector (existing tests, ad-hoc use) still work.""" + df = _balance_reconciliation_frame(source_closing=90.0, released=[0.0, 8.0, 4.0]) + + out = _compute_running_balance(df) + + assert out["product_balance"].to_list() == [100.0, 92.0, 88.0] diff --git a/tests/test_clean/test_transformers.py b/tests/test_clean/test_transformers.py new file mode 100644 index 0000000..7e613b3 --- /dev/null +++ b/tests/test_clean/test_transformers.py @@ -0,0 +1,888 @@ +"""Tests for data transformation functions.""" + +import polars as pl +import pytest + +from a4d.clean.transformers import ( + apply_transformation, + correct_decimal_sign_multiple, + extract_regimen, + fix_bmi, + fix_sex, + fix_testing_frequency, + replace_range_with_mean, + split_bp_in_sys_and_dias, + str_to_lower, +) +from a4d.config import settings + + +def test_extract_regimen_basal(): + """Test extraction of basal-bolus regimen.""" + df = pl.DataFrame( + { + "insulin_regimen": [ + "Basal-bolus", + "basal bolus", + "BASAL", + "Some basal text", + ] + } + ) + + result = extract_regimen(df) + + # All should be standardized to "Basal-bolus (MDI)" + assert all(v == "Basal-bolus (MDI)" for v in result["insulin_regimen"].to_list()) + + +def test_extract_regimen_premixed(): + """Test extraction of premixed regimen.""" + df = pl.DataFrame( + { + "insulin_regimen": [ + "Premixed", + "PREMIXED 30/70", + "premixed bd", + ] + } + ) + + result = extract_regimen(df) + + assert all(v == "Premixed 30/70 BD" for v in result["insulin_regimen"].to_list()) + + +def test_extract_regimen_self_mixed(): + """Test extraction of self-mixed regimen.""" + df = pl.DataFrame( + { + "insulin_regimen": [ + "Self-mixed", + "SELF-MIXED BD", + "self-mixed", # Must have hyphen to match + ] + } + ) + + result = extract_regimen(df) + + assert all(v == "Self-mixed BD" for v in result["insulin_regimen"].to_list()) + + +def test_extract_regimen_conventional(): + """Test extraction of conventional regimen.""" + df = pl.DataFrame( + { + "insulin_regimen": [ + "Conventional", + "Modified CONVENTIONAL TID", + "conventional tid", + ] + } + ) + + result = extract_regimen(df) + + assert all(v == "Modified conventional TID" for v in result["insulin_regimen"].to_list()) + + +def test_extract_regimen_missing_column(): + """Test that missing column is handled gracefully.""" + df = pl.DataFrame({"other": ["value"]}) + + result = extract_regimen(df) + + assert result.equals(df) + + +def test_extract_regimen_preserves_nulls(): + """Test that nulls are preserved.""" + df = pl.DataFrame( + { + "insulin_regimen": ["Basal-bolus", None, "Premixed"], + } + ) + + result = extract_regimen(df) + + assert result["insulin_regimen"][0] == "Basal-bolus (MDI)" + assert result["insulin_regimen"][1] is None + assert result["insulin_regimen"][2] == "Premixed 30/70 BD" + + +def test_extract_regimen_no_match(): + """Test values that don't match any pattern.""" + df = pl.DataFrame( + { + "insulin_regimen": [ + "Unknown regimen", + "Other", + ] + } + ) + + result = extract_regimen(df) + + # Values that don't match are left exactly as the source wrote them + # (ticket 29). + assert result["insulin_regimen"].to_list() == ["Unknown regimen", "Other"] + + +def test_str_to_lower(): + """Test string lowercasing.""" + df = pl.DataFrame( + { + "status": ["ACTIVE", "Inactive", "Transferred", "MixedCase"], + } + ) + + result = str_to_lower(df, "status") + + assert result["status"].to_list() == ["active", "inactive", "transferred", "mixedcase"] + + +def test_str_to_lower_preserves_nulls(): + """Test that nulls are preserved.""" + df = pl.DataFrame( + { + "status": ["ACTIVE", None, "Inactive"], + } + ) + + result = str_to_lower(df, "status") + + assert result["status"][0] == "active" + assert result["status"][1] is None + assert result["status"][2] == "inactive" + + +def test_str_to_lower_missing_column(): + """Test that missing column is handled gracefully.""" + df = pl.DataFrame({"other": ["VALUE"]}) + + result = str_to_lower(df, "nonexistent") + + assert result.equals(df) + + +def test_apply_transformation_extract_regimen(): + """Test applying extract_regimen transformation.""" + df = pl.DataFrame( + { + "insulin_regimen": ["Basal-bolus", "Premixed"], + } + ) + + result = apply_transformation(df, "insulin_regimen", "extract_regimen") + + assert result["insulin_regimen"].to_list() == ["Basal-bolus (MDI)", "Premixed 30/70 BD"] + + +def test_apply_transformation_str_to_lower(): + """Test applying str_to_lower transformation (both naming conventions).""" + df = pl.DataFrame( + { + "status": ["ACTIVE", "INACTIVE"], + } + ) + + # The `stringr::` prefix is an accepted alias; the config files carry it. + result = apply_transformation(df, "status", "stringr::str_to_lower") + assert result["status"].to_list() == ["active", "inactive"] + + # Reset + df = pl.DataFrame({"status": ["ACTIVE", "INACTIVE"]}) + + # Test with Python function name + result = apply_transformation(df, "status", "str_to_lower") + assert result["status"].to_list() == ["active", "inactive"] + + +def test_apply_transformation_unknown_function(): + """Test that unknown function raises error.""" + df = pl.DataFrame({"column": ["value"]}) + + with pytest.raises(ValueError, match="Unknown transformation function"): + apply_transformation(df, "column", "unknown_function") + + +def test_correct_decimal_sign_multiple(): + """Test correcting decimal signs for multiple columns.""" + df = pl.DataFrame( + { + "weight": ["70,5", "80,2"], + "height": ["1,75", "1,80"], + "hba1c": ["7,2", "6,8"], + } + ) + + result = correct_decimal_sign_multiple(df, ["weight", "height", "hba1c"]) + + assert result["weight"].to_list() == ["70.5", "80.2"] + assert result["height"].to_list() == ["1.75", "1.80"] + assert result["hba1c"].to_list() == ["7.2", "6.8"] + + +def test_correct_decimal_sign_multiple_missing_columns(): + """Test that missing columns are handled gracefully.""" + df = pl.DataFrame( + { + "weight": ["70,5", "80,2"], + } + ) + + # Should not raise error even though height and hba1c don't exist + result = correct_decimal_sign_multiple(df, ["weight", "height", "hba1c"]) + + assert result["weight"].to_list() == ["70.5", "80.2"] + + +def test_extract_regimen_order_matters(): + """Transformations are applied in order, and the first match wins. + + A value matching more than one pattern must resolve to the earlier one, so + the order in the config is part of the contract rather than incidental. + """ + df = pl.DataFrame( + { + "insulin_regimen": [ + "basal premixed", # Both patterns match + ] + } + ) + + result = extract_regimen(df) + + # "basal" is checked first in the code, so it should match that + assert result["insulin_regimen"][0] == "Basal-bolus (MDI)" + + +def test_fix_sex_female_synonyms(): + """Test that female synonyms are mapped to 'F'.""" + df = pl.DataFrame( + { + "sex": [ + "Female", + "FEMALE", + "girl", + "Woman", + "fem", + "Feminine", + "f", + "F", + ] + } + ) + + result = fix_sex(df) + + # All should be mapped to "F" + assert all(v == "F" for v in result["sex"].to_list()) + + +def test_fix_sex_male_synonyms(): + """Test that male synonyms are mapped to 'M'.""" + df = pl.DataFrame( + { + "sex": [ + "Male", + "MALE", + "boy", + "Man", + "masculine", + "m", + "M", + ] + } + ) + + result = fix_sex(df) + + # All should be mapped to "M" + assert all(v == "M" for v in result["sex"].to_list()) + + +def test_fix_sex_invalid_values(): + """Test that invalid values are set to 'Undefined'.""" + df = pl.DataFrame( + { + "sex": [ + "invalid", + "unknown", + "other", + "X", + ] + } + ) + + result = fix_sex(df) + + # All should be set to "Undefined" + assert all(v == "Undefined" for v in result["sex"].to_list()) + + +def test_fix_sex_preserves_nulls(): + """Test that null and empty values are preserved as null.""" + df = pl.DataFrame( + { + "sex": ["Female", None, "", "Male"], + } + ) + + result = fix_sex(df) + + assert result["sex"][0] == "F" + assert result["sex"][1] is None + assert result["sex"][2] is None + assert result["sex"][3] == "M" + + +def test_fix_sex_case_insensitive(): + """Test that matching is case-insensitive.""" + df = pl.DataFrame( + { + "sex": [ + "FEMALE", + "female", + "Female", + "FeMaLe", + "MALE", + "male", + "Male", + "MaLe", + ] + } + ) + + result = fix_sex(df) + + assert result["sex"].to_list() == ["F", "F", "F", "F", "M", "M", "M", "M"] + + +def test_fix_sex_missing_column(): + """Test that missing column is handled gracefully.""" + df = pl.DataFrame({"other": ["value"]}) + + result = fix_sex(df) + + assert result.equals(df) + + +def test_fix_sex_maps_every_known_synonym(): + """Every spelling of sex observed in the trackers maps to M, F or Undefined.""" + df = pl.DataFrame( + { + "sex": [ + # Female synonyms + "female", + "girl", + "woman", + "fem", + "feminine", + "f", + # Male synonyms + "male", + "boy", + "man", + "masculine", + "m", + # Invalid + "other", + "unknown", + # Null/empty + None, + "", + ] + } + ) + + result = fix_sex(df) + + expected = [ + "F", + "F", + "F", + "F", + "F", + "F", + "M", + "M", + "M", + "M", + "M", + "Undefined", + "Undefined", + None, + None, + ] + assert result["sex"].to_list() == expected + + +def test_fix_bmi_basic_calculation(): + """Test basic BMI calculation from weight and height.""" + df = pl.DataFrame( + { + "weight": [70.0, 80.0, 65.0], + "height": [1.75, 1.80, 1.60], + } + ) + + result = fix_bmi(df) + + # BMI = weight / height^2 + assert "bmi" in result.columns + assert result["bmi"][0] == pytest.approx(22.857, abs=0.001) # 70 / 1.75^2 = 22.857 + assert result["bmi"][1] == pytest.approx(24.691, abs=0.001) # 80 / 1.80^2 = 24.691 + assert result["bmi"][2] == pytest.approx(25.391, abs=0.001) # 65 / 1.60^2 = 25.391 + + +def test_fix_bmi_replaces_existing(): + """Test that calculated BMI replaces existing BMI value.""" + df = pl.DataFrame( + { + "weight": [70.0], + "height": [1.75], + "bmi": [999.9], # Wrong BMI that should be replaced + } + ) + + result = fix_bmi(df) + + # Should replace wrong BMI with correct calculation + assert result["bmi"][0] == pytest.approx(22.857, abs=0.001) + + +def test_fix_bmi_null_weight(): + """Test that null weight results in null BMI.""" + df = pl.DataFrame( + { + "weight": [None, 70.0], + "height": [1.75, 1.75], + } + ) + + result = fix_bmi(df) + + assert result["bmi"][0] is None + assert result["bmi"][1] is not None + + +def test_fix_bmi_null_height(): + """Test that null height results in null BMI.""" + df = pl.DataFrame( + { + "weight": [70.0, 70.0], + "height": [None, 1.75], + } + ) + + result = fix_bmi(df) + + assert result["bmi"][0] is None + assert result["bmi"][1] is not None + + +def test_fix_bmi_error_value_weight(): + """Test that error value weight results in error value BMI.""" + df = pl.DataFrame( + { + "weight": [settings.error_val_numeric, 70.0], + "height": [1.75, 1.75], + } + ) + + result = fix_bmi(df) + + assert result["bmi"][0] == settings.error_val_numeric + assert result["bmi"][1] == pytest.approx(22.857, abs=0.001) + + +def test_fix_bmi_error_value_height(): + """Test that error value height results in error value BMI.""" + df = pl.DataFrame( + { + "weight": [70.0, 70.0], + "height": [settings.error_val_numeric, 1.75], + } + ) + + result = fix_bmi(df) + + assert result["bmi"][0] == settings.error_val_numeric + assert result["bmi"][1] == pytest.approx(22.857, abs=0.001) + + +def test_fix_bmi_missing_columns(): + """Test that missing weight or height columns are handled gracefully.""" + # Missing both + df = pl.DataFrame({"other": [1, 2, 3]}) + result = fix_bmi(df) + assert result.equals(df) + + # Missing weight + df = pl.DataFrame({"height": [1.75, 1.80]}) + result = fix_bmi(df) + assert result.equals(df) + + # Missing height + df = pl.DataFrame({"weight": [70.0, 80.0]}) + result = fix_bmi(df) + assert result.equals(df) + + +def test_fix_bmi_derives_from_weight_and_height(): + """BMI is recomputed from weight and height, replacing any recorded value.""" + df = pl.DataFrame( + { + "weight": [70.0, None, settings.error_val_numeric, 80.0, 65.0], + "height": [1.75, 1.80, 1.75, None, settings.error_val_numeric], + } + ) + + result = fix_bmi(df) + + # Row 0: Normal calculation + assert result["bmi"][0] == pytest.approx(22.857, abs=0.001) + # Row 1: Null weight → null BMI + assert result["bmi"][1] is None + # Row 2: Error weight → error BMI + assert result["bmi"][2] == settings.error_val_numeric + # Row 3: Null height → null BMI + assert result["bmi"][3] is None + # Row 4: Error height → error BMI + assert result["bmi"][4] == settings.error_val_numeric + + +def test_fix_bmi_height_cm_conversion(): + """Test that height in cm is converted to m before BMI calculation. + + Above 50 the value can only be centimetres; below 2.3 it can only be + metres. Real case: Lao Friends Hospital has height=135.5cm, weight=30.7kg. + """ + df = pl.DataFrame( + { + "weight": [30.7, 70.0, 80.0], + "height": [135.5, 175.0, 1.80], # cm, cm, m + } + ) + + result = fix_bmi(df) + + # Row 0: 135.5cm → 1.355m → BMI = 30.7 / 1.355² = 16.72 + assert result["bmi"][0] == pytest.approx(16.72, abs=0.01) + # Row 1: 175cm → 1.75m → BMI = 70 / 1.75² = 22.86 + assert result["bmi"][1] == pytest.approx(22.86, abs=0.01) + # Row 2: 1.80m stays as-is → BMI = 80 / 1.80² = 24.69 + assert result["bmi"][2] == pytest.approx(24.69, abs=0.01) + + +# Tests for replace_range_with_mean + + +def test_replace_range_with_mean_basic(): + """Test basic range mean calculation.""" + assert replace_range_with_mean("0-2") == pytest.approx(1.0) + assert replace_range_with_mean("2-3") == pytest.approx(2.5) + assert replace_range_with_mean("1-5") == pytest.approx(3.0) + + +def test_replace_range_with_mean_larger_ranges(): + """Test larger range values.""" + assert replace_range_with_mean("10-20") == pytest.approx(15.0) + assert replace_range_with_mean("0-10") == pytest.approx(5.0) + + +def test_replace_range_with_mean_same_values(): + """Test range where both values are the same.""" + assert replace_range_with_mean("0-0") == pytest.approx(0.0) + assert replace_range_with_mean("5-5") == pytest.approx(5.0) + + +def test_replace_range_with_mean_decimals(): + """Test ranges with decimal values.""" + assert replace_range_with_mean("1.5-2.5") == pytest.approx(2.0) + assert replace_range_with_mean("0.5-1.5") == pytest.approx(1.0) + + +# Tests for fix_testing_frequency + + +def test_fix_testing_frequency_passthrough(): + """Test that normal values pass through unchanged.""" + df = pl.DataFrame( + { + "patient_id": ["P1", "P2", "P3"], + "testing_frequency": ["2", "1.5", "3"], + } + ) + + result = fix_testing_frequency(df) + + assert result["testing_frequency"].to_list() == ["2", "1.5", "3"] + + +def test_fix_testing_frequency_range_replacement(): + """Test that ranges are replaced with mean.""" + df = pl.DataFrame( + { + "patient_id": ["P1", "P2", "P3"], + "testing_frequency": ["0-2", "2-3", "1-5"], + } + ) + + result = fix_testing_frequency(df) + + assert result["testing_frequency"].to_list() == ["1", "2.5", "3"] + + +def test_fix_testing_frequency_mixed(): + """Test mixed normal values and ranges.""" + df = pl.DataFrame( + { + "patient_id": ["P1", "P2", "P3", "P4"], + "testing_frequency": ["2", "0-2", "1.5", "2-3"], + } + ) + + result = fix_testing_frequency(df) + + assert result["testing_frequency"].to_list() == ["2", "1", "1.5", "2.5"] + + +def test_fix_testing_frequency_null_handling(): + """Test that null and empty values are preserved.""" + df = pl.DataFrame( + { + "patient_id": ["P1", "P2", "P3"], + "testing_frequency": [None, "", "2"], + } + ) + + result = fix_testing_frequency(df) + + assert result["testing_frequency"][0] is None + assert result["testing_frequency"][1] is None + assert result["testing_frequency"][2] == "2" + + +def test_fix_testing_frequency_whole_numbers(): + """Test that whole number means don't have decimal points.""" + df = pl.DataFrame( + { + "patient_id": ["P1", "P2"], + "testing_frequency": ["0-2", "1-3"], + } + ) + + result = fix_testing_frequency(df) + + # 0-2 mean is 1.0, should be "1" not "1.0" + # 1-3 mean is 2.0, should be "2" not "2.0" + assert result["testing_frequency"][0] == "1" + assert result["testing_frequency"][1] == "2" + + +def test_fix_testing_frequency_missing_column(): + """Test that missing column is handled gracefully.""" + df = pl.DataFrame({"other": [1, 2, 3]}) + + result = fix_testing_frequency(df) + + assert result.equals(df) + + +def test_fix_testing_frequency_large_range(): + """Test larger ranges.""" + df = pl.DataFrame( + { + "patient_id": ["P1"], + "testing_frequency": ["0-10"], + } + ) + + result = fix_testing_frequency(df) + + assert result["testing_frequency"][0] == "5" + + +def test_fix_testing_frequency_preserves_other_columns(): + """Test that other columns are preserved.""" + df = pl.DataFrame( + { + "patient_id": ["P1", "P2"], + "testing_frequency": ["0-2", "3"], + "other_col": ["A", "B"], + } + ) + + result = fix_testing_frequency(df) + + assert "patient_id" in result.columns + assert "other_col" in result.columns + assert result["other_col"].to_list() == ["A", "B"] + + +# Tests for split_bp_in_sys_and_dias + + +def test_split_bp_valid_format(): + """Test splitting valid blood pressure format.""" + df = pl.DataFrame( + { + "blood_pressure_mmhg": ["96/55", "101/57", "120/80"], + } + ) + + result = split_bp_in_sys_and_dias(df) + + assert "blood_pressure_sys_mmhg" in result.columns + assert "blood_pressure_dias_mmhg" in result.columns + assert "blood_pressure_mmhg" not in result.columns + + assert result["blood_pressure_sys_mmhg"].to_list() == ["96", "101", "120"] + assert result["blood_pressure_dias_mmhg"].to_list() == ["55", "57", "80"] + + +def test_split_bp_strips_whitespace_around_the_separator(): + """A clinician who writes "70 / 40" has recorded a blood pressure, and the + fragments must reach numeric conversion without their padding (ticket 53). + + Polars' numeric cast rejects surrounding whitespace, so ``" 40"`` failed + conversion and the cleaned output carried the 999999 error sentinel + instead. 465 cells across 7 real trackers + were lost this way -- every affected source value has a space beside the + slash and none has anything else wrong with it. + """ + df = pl.DataFrame( + { + "blood_pressure_mmhg": ["70 / 40", "103/ 69", "116 / 66", "105/ 62"], + } + ) + + result = split_bp_in_sys_and_dias(df) + + assert result["blood_pressure_sys_mmhg"].to_list() == ["70", "103", "116", "105"] + assert result["blood_pressure_dias_mmhg"].to_list() == ["40", "69", "66", "62"] + + +def test_split_bp_invalid_no_slash(): + """Test that values without slash are replaced with error value.""" + df = pl.DataFrame( + { + "blood_pressure_mmhg": ["96", "1,6", ""], + } + ) + + result = split_bp_in_sys_and_dias(df) + + error_val = str(int(settings.error_val_numeric)) + assert result["blood_pressure_sys_mmhg"].to_list() == [error_val, error_val, error_val] + assert result["blood_pressure_dias_mmhg"].to_list() == [error_val, error_val, error_val] + + +def test_split_bp_mixed_valid_invalid(): + """Test mixed valid and invalid values.""" + df = pl.DataFrame( + { + "blood_pressure_mmhg": ["96/55", "invalid", "120/80"], + } + ) + + result = split_bp_in_sys_and_dias(df) + + error_val = str(int(settings.error_val_numeric)) + assert result["blood_pressure_sys_mmhg"].to_list() == ["96", error_val, "120"] + assert result["blood_pressure_dias_mmhg"].to_list() == ["55", error_val, "80"] + + +def test_split_bp_null_values(): + """Test that null values are preserved.""" + df = pl.DataFrame( + { + "blood_pressure_mmhg": ["96/55", None, "120/80"], + } + ) + + result = split_bp_in_sys_and_dias(df) + + assert result["blood_pressure_sys_mmhg"][0] == "96" + assert result["blood_pressure_sys_mmhg"][1] is None + assert result["blood_pressure_sys_mmhg"][2] == "120" + + +def test_split_bp_missing_column(): + """Test that missing column is handled gracefully.""" + df = pl.DataFrame({"other": [1, 2, 3]}) + + result = split_bp_in_sys_and_dias(df) + + assert result.equals(df) + + +def test_split_bp_drops_original_column(): + """Test that original blood_pressure_mmhg column is dropped.""" + df = pl.DataFrame( + { + "blood_pressure_mmhg": ["96/55", "120/80"], + } + ) + + result = split_bp_in_sys_and_dias(df) + + assert "blood_pressure_mmhg" not in result.columns + + +def test_split_bp_preserves_other_columns(): + """Test that other columns are preserved.""" + df = pl.DataFrame( + { + "patient_id": ["P1", "P2"], + "blood_pressure_mmhg": ["96/55", "120/80"], + "other_col": ["A", "B"], + } + ) + + result = split_bp_in_sys_and_dias(df) + + assert "patient_id" in result.columns + assert "other_col" in result.columns + assert result["patient_id"].to_list() == ["P1", "P2"] + assert result["other_col"].to_list() == ["A", "B"] + + +def test_split_bp_multiple_invalid(): + """Test multiple invalid values log warning.""" + df = pl.DataFrame( + { + "blood_pressure_mmhg": ["invalid1", "invalid2", "96/55"], + } + ) + + result = split_bp_in_sys_and_dias(df) + + error_val = str(int(settings.error_val_numeric)) + assert result["blood_pressure_sys_mmhg"][0] == error_val + assert result["blood_pressure_sys_mmhg"][1] == error_val + assert result["blood_pressure_sys_mmhg"][2] == "96" + + +def test_extract_regimen_preserves_case_of_unmatched_values(): + """Ticket 29: a value matching none of the patterns is left untouched. + + An earlier version got case-insensitivity by lowercasing the column first, + which + permanently mangled every value none of the four patterns matched -- + "NPH" -> "nph", "Other" -> "other" -- on 1,309 real cleaned rows. + """ + df = pl.DataFrame({"insulin_regimen": ["NPH", "Other", "Others", "Glargine", "Mix -bolus"]}) + + result = extract_regimen(df) + + assert result["insulin_regimen"].to_list() == [ + "NPH", + "Other", + "Others", + "Glargine", + "Mix -bolus", + ] diff --git a/tests/test_clean/test_validators.py b/tests/test_clean/test_validators.py new file mode 100644 index 0000000..b01ead6 --- /dev/null +++ b/tests/test_clean/test_validators.py @@ -0,0 +1,780 @@ +"""Tests for schema and validation utilities.""" + +import polars as pl +import pytest + +from a4d.clean.validators import ( + fix_patient_id, + load_validation_rules, + validate_all_columns, + validate_allowed_values, + validate_column_from_rules, +) +from a4d.config import settings +from a4d.errors import ErrorCollector + + +def test_load_validation_rules(): + """Test loading validation rules from YAML.""" + rules = load_validation_rules() + + # Check that rules were loaded + assert isinstance(rules, dict) + assert len(rules) > 0 + + # Check a specific column rule (new simplified structure) + assert "status" in rules + assert "allowed_values" in rules["status"] + assert "replace_invalid" in rules["status"] + assert isinstance(rules["status"]["allowed_values"], list) + assert len(rules["status"]["allowed_values"]) > 0 + + # Check another column + assert "clinic_visit" in rules + assert rules["clinic_visit"]["allowed_values"] == ["N", "Y"] + assert rules["clinic_visit"]["replace_invalid"] is True + + +def test_validate_allowed_values_all_valid(): + """Test validation when all values are valid.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 3, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003"], + "status": ["Active", "Inactive", "Active"], + } + ) + + collector = ErrorCollector() + + result = validate_allowed_values( + df=df, + column="status", + allowed_values=["Active", "Inactive", "Transferred"], + error_collector=collector, + replace_invalid=True, + ) + + assert result["status"].to_list() == ["Active", "Inactive", "Active"] + assert len(collector) == 0 + + +def test_validate_allowed_values_with_invalid(): + """Test validation when some values are invalid.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 4, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003", "XX_YY004"], + "status": ["Active", "INVALID", "Inactive", "BAD_VALUE"], + } + ) + + collector = ErrorCollector() + + result = validate_allowed_values( + df=df, + column="status", + allowed_values=["Active", "Inactive"], + error_collector=collector, + replace_invalid=True, + ) + + assert result["status"].to_list() == [ + "Active", + settings.error_val_character, + "Inactive", + settings.error_val_character, + ] + assert len(collector) == 2 + + # Check error details + # Note: file_name and patient_id are "unknown" placeholders in validate_allowed_values + # They get filled in during bulk processing operations + errors_df = collector.to_dataframe() + # Order is not guaranteed, so check using sets + assert set(errors_df["original_value"].to_list()) == {"INVALID", "BAD_VALUE"} + assert errors_df["column"].to_list() == ["status", "status"] + assert errors_df["error_code"].to_list() == ["invalid_value", "invalid_value"] + + +def test_validate_allowed_values_preserves_nulls(): + """Test that nulls are preserved and not logged as errors.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 3, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003"], + "status": ["Active", None, "Inactive"], + } + ) + + collector = ErrorCollector() + + result = validate_allowed_values( + df=df, + column="status", + allowed_values=["Active", "Inactive"], + error_collector=collector, + replace_invalid=True, + ) + + assert result["status"].to_list() == ["Active", None, "Inactive"] + assert len(collector) == 0 + + +def test_validate_allowed_values_no_replace(): + """Test validation without replacing invalid values.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 2, + "patient_id": ["XX_YY001", "XX_YY002"], + "status": ["Active", "INVALID"], + } + ) + + collector = ErrorCollector() + + result = validate_allowed_values( + df=df, + column="status", + allowed_values=["Active"], + error_collector=collector, + replace_invalid=False, + ) + + # Invalid value should NOT be replaced + assert result["status"].to_list() == ["Active", "INVALID"] + # But it should still be logged + assert len(collector) == 1 + + +def test_validate_allowed_values_missing_column(): + """Test that missing columns are handled gracefully.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"], + "patient_id": ["XX_YY001"], + } + ) + + collector = ErrorCollector() + + result = validate_allowed_values( + df=df, + column="nonexistent", + allowed_values=["Active"], + error_collector=collector, + ) + + assert result.equals(df) + assert len(collector) == 0 + + +def test_validate_allowed_values_ignores_existing_errors(): + """Test that existing error values are not re-logged.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 3, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003"], + "status": ["Active", settings.error_val_character, "INVALID"], + } + ) + + collector = ErrorCollector() + + result = validate_allowed_values( + df=df, + column="status", + allowed_values=["Active", "Inactive"], + error_collector=collector, + replace_invalid=True, + ) + + # Only "INVALID" should be logged, not the existing error value + assert len(collector) == 1 + assert result["status"].to_list() == [ + "Active", + settings.error_val_character, + settings.error_val_character, + ] + + +def test_validate_column_from_rules(): + """Test validation using rules from data_cleaning.yaml.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 3, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003"], + "clinic_visit": ["Y", "N", "INVALID"], + } + ) + + rules = load_validation_rules() + collector = ErrorCollector() + + result = validate_column_from_rules( + df=df, + column="clinic_visit", + rules=rules["clinic_visit"], + error_collector=collector, + ) + + # "INVALID" should be replaced with error value + assert result["clinic_visit"].to_list() == ["Y", "N", settings.error_val_character] + assert len(collector) == 1 + + +def test_validate_column_from_rules_missing_column(): + """Test validation with missing column.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"], + "patient_id": ["XX_YY001"], + } + ) + + rules = load_validation_rules() + collector = ErrorCollector() + + result = validate_column_from_rules( + df=df, + column="nonexistent", + rules=rules["clinic_visit"], + error_collector=collector, + ) + + assert result.equals(df) + assert len(collector) == 0 + + +def test_validate_all_columns(): + """Test validation of all columns with rules. + + Note: Validation uses case-insensitive matching and normalizes to canonical values. + For example, "active" becomes "Active", "y" becomes "Y". + """ + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 3, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003"], + "clinic_visit": ["Y", "N", "INVALID1"], + "patient_consent": ["Y", "INVALID2", "N"], + "status": ["active", "INVALID3", "inactive"], # Lowercase input + } + ) + + collector = ErrorCollector() + + result = validate_all_columns(df, collector) + + # All invalid values should be replaced + # Valid values should be normalized to canonical form (Title Case for status) + assert result["clinic_visit"].to_list() == ["Y", "N", settings.error_val_character] + assert result["patient_consent"].to_list() == ["Y", settings.error_val_character, "N"] + assert result["status"].to_list() == ["Active", settings.error_val_character, "Inactive"] + + # Should have logged 3 errors (one per invalid value) + assert len(collector) == 3 + + +def test_validate_all_columns_only_validates_existing(): + """Test that validation only processes columns that exist in DataFrame.""" + df = pl.DataFrame( + { + "file_name": ["test.xlsx"], + "patient_id": ["XX_YY001"], + "clinic_visit": ["Y"], + # Many other columns from rules don't exist + } + ) + + collector = ErrorCollector() + + # Should not raise error even though many rule columns don't exist + result = validate_all_columns(df, collector) + + assert "clinic_visit" in result.columns + assert len(collector) == 0 + + +def test_validate_allowed_values_case_insensitive(): + """Test that validation is case-insensitive and normalizes to canonical values. + + - "y" matches "Y" (case-insensitive) + - Returns canonical value "Y" (not the input "y") + """ + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 3, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003"], + "clinic_visit": ["Y", "y", "N"], # Mixed case + } + ) + + collector = ErrorCollector() + + result = validate_allowed_values( + df=df, + column="clinic_visit", + allowed_values=["Y", "N"], + error_collector=collector, + replace_invalid=True, + ) + + # Lowercase "y" should match "Y" and be normalized to canonical "Y" + assert result["clinic_visit"].to_list() == ["Y", "Y", "N"] + assert len(collector) == 0 # No errors - "y" is valid + + +def test_validate_allowed_values_csv_subset(): + """CSV values whose every token is in allowed_values emit canonical CSV.""" + allowed = ["Pre-mixed", "Short-acting", "Intermediate-acting", "Rapid-acting", "Long-acting"] + df = pl.DataFrame( + { + "file_name": ["test.xlsx"] * 5, + "patient_id": ["XX_YY001", "XX_YY002", "XX_YY003", "XX_YY004", "XX_YY005"], + "insulin_subtype": [ + "rapid-acting", # single valid + "pre-mixed,rapid-acting", # two-token CSV valid + "pre-mixed,rapid-acting,long-acting", # three-token CSV valid + "pre-mixed,unknown-thing", # CSV with one bad token + "not-in-list", # single invalid + ], + } + ) + + collector = ErrorCollector() + result = validate_allowed_values( + df=df, + column="insulin_subtype", + allowed_values=allowed, + error_collector=collector, + replace_invalid=True, + allow_csv_subset=True, + ) + + assert result["insulin_subtype"].to_list() == [ + "Rapid-acting", + "Pre-mixed,Rapid-acting", + "Pre-mixed,Rapid-acting,Long-acting", + settings.error_val_character, + settings.error_val_character, + ] + assert len(collector) == 2 + + +def test_validate_allowed_values_csv_subset_disabled(): + """Without allow_csv_subset, CSV values are treated as single strings and fail.""" + allowed = ["Pre-mixed", "Rapid-acting"] + df = pl.DataFrame( + { + "file_name": ["test.xlsx"], + "patient_id": ["XX_YY001"], + "insulin_subtype": ["pre-mixed,rapid-acting"], + } + ) + + collector = ErrorCollector() + result = validate_allowed_values( + df=df, + column="insulin_subtype", + allowed_values=allowed, + error_collector=collector, + replace_invalid=True, + ) + + assert result["insulin_subtype"].to_list() == [settings.error_val_character] + + +# Tests for fix_patient_id + + +def test_fix_patient_id_valid_ids(): + """Test that valid patient IDs are not changed.""" + df = pl.DataFrame( + { + "patient_id": ["KD_EW004", "AB_CD123", "XY_ZW999"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"].to_list() == ["KD_EW004", "AB_CD123", "XY_ZW999"] + assert len(collector) == 0 + + +def test_fix_patient_id_hyphen_normalization(): + """Test that hyphens are replaced with underscores.""" + df = pl.DataFrame( + { + "patient_id": ["KD-EW004", "AB-CD123"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"].to_list() == ["KD_EW004", "AB_CD123"] + assert len(collector) == 0 # Normalization doesn't generate errors + + +def test_fix_patient_id_overlong_without_candidate_is_sentinelled(): + """An over-length ID is never truncated into an identity nobody wrote. + + Truncating to 8 characters manufactures an ID that appears in no source + workbook (ticket 47: KH_NPH026 -> KH_NPH02, which then collected four + different patients' records). + """ + df = pl.DataFrame( + { + "patient_id": ["KD_EW004XY", "KD_EW004ABC", "VERYLONGID"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"].to_list() == ["Undefined", "Undefined", "Undefined"] + assert len(collector) == 3 + + +def test_fix_patient_id_recovers_from_the_tracker_s_own_spelling(): + """A one-edit typo resolves to the ID the same tracker spells correctly.""" + df = pl.DataFrame( + { + "patient_id": ["KH_NPH026", "KH_NP026", "KH_NP027"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"].to_list() == ["KH_NP026", "KH_NP026", "KH_NP027"] + # Recovery is still a defect in the source workbook, so it is reported. + assert len(collector) == 1 + assert "KH_NPH026" in collector.errors[0].original_value + + +def test_fix_patient_id_recovers_a_short_id_too(): + """Recovery is not limited to the over-length branch.""" + df = pl.DataFrame({"patient_id": ["MM_NO97", "MM_NO097"]}) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"].to_list() == ["MM_NO097", "MM_NO097"] + + +def test_fix_patient_id_ambiguous_candidates_are_sentinelled(): + """Two candidates one edit away means the intended patient is unknowable.""" + df = pl.DataFrame({"patient_id": ["KH_NP02", "KH_NP021", "KH_NP023"]}) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"][0] == "Undefined" + + +def test_fix_patient_id_candidate_must_be_in_the_same_tracker(): + """No candidate in the frame means no recovery, however plausible the guess. + + 2026_NOGH writes MM_NO97 in its Patient List and every month sheet, and + has no MM_NO097 anywhere -- so nothing licenses inventing one. + """ + df = pl.DataFrame({"patient_id": ["MM_NO97", "MM_NO096", "MM_NO001"]}) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"][0] == "Undefined" + + +def test_fix_patient_id_invalid_too_short_first_part(): + """Test that IDs with < 2 letters in first part are replaced.""" + df = pl.DataFrame( + { + "patient_id": ["K_EW004", "A_CD123"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"].to_list() == ["Undefined", "Undefined"] + assert len(collector) == 2 + + +def test_fix_patient_id_invalid_too_short_second_part(): + """Test that IDs with < 2 letters in second part are replaced.""" + df = pl.DataFrame( + { + "patient_id": ["KD_E004", "AB_C123"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"].to_list() == ["Undefined", "Undefined"] + assert len(collector) == 2 + + +def test_fix_patient_id_invalid_wrong_digits(): + """Test that IDs without exactly 3 digits are replaced.""" + df = pl.DataFrame( + { + "patient_id": ["KD_EW04", "KD_EW0", "KD_EW0001"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + # All invalid (2 digits, 1 digit, 4 digits), and none has a well-formed + # ID in the same frame to recover against. + assert result["patient_id"].to_list() == ["Undefined", "Undefined", "Undefined"] + + +def test_fix_patient_id_invalid_digits_in_letter_positions(): + """Test that IDs with digits instead of letters are replaced.""" + df = pl.DataFrame( + { + "patient_id": ["11_EW004", "KD_E1004", "12_34567"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"].to_list() == ["Undefined", "Undefined", "Undefined"] + assert len(collector) == 3 + + +def test_fix_patient_id_invalid_letters_in_digit_positions(): + """Test that IDs with letters in digit positions are replaced.""" + df = pl.DataFrame( + { + "patient_id": ["KD_EWX04", "KD_EWABC"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"].to_list() == ["Undefined", "Undefined"] + assert len(collector) == 2 + + +def test_fix_patient_id_invalid_no_underscore(): + """Test that IDs without underscore are replaced.""" + df = pl.DataFrame( + { + "patient_id": ["KDEW004", "INVALID"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"].to_list() == ["Undefined", "Undefined"] + assert len(collector) == 2 + + +def test_fix_patient_id_null_values(): + """Test that null values are preserved.""" + df = pl.DataFrame( + { + "patient_id": ["KD_EW004", None, "AB_CD123"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"][0] == "KD_EW004" + assert result["patient_id"][1] is None + assert result["patient_id"][2] == "AB_CD123" + assert len(collector) == 0 + + +def test_fix_patient_id_empty_string(): + """Test that empty string is replaced with error value.""" + df = pl.DataFrame( + { + "patient_id": ["", "KD_EW004"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"][0] == "Undefined" + assert result["patient_id"][1] == "KD_EW004" + assert len(collector) == 1 + + +def test_fix_patient_id_missing_column(): + """Test that missing column is handled gracefully.""" + df = pl.DataFrame({"other": [1, 2, 3]}) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result.equals(df) + assert len(collector) == 0 + + +def test_fix_patient_id_mixed_valid_invalid(): + """Test mixed valid and invalid IDs.""" + df = pl.DataFrame( + { + "patient_id": [ + "KD_EW004", # Valid + "KD-AB123", # Valid after normalization + "INVALID", # Invalid, replaced + "KD_EW004XY", # Invalid, two edits from KD_EW004 + None, # Null preserved + ], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"][0] == "KD_EW004" + assert result["patient_id"][1] == "KD_AB123" + assert result["patient_id"][2] == "Undefined" + assert result["patient_id"][3] == "Undefined" + assert result["patient_id"][4] is None + assert len(collector) == 2 + + +def test_fix_patient_id_lowercase_letters(): + """Test that lowercase letters make ID invalid.""" + df = pl.DataFrame( + { + "patient_id": ["kd_ew004", "KD_ew004", "kd_EW004"], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + # All should be replaced (format requires uppercase) + assert result["patient_id"].to_list() == ["Undefined", "Undefined", "Undefined"] + assert len(collector) == 3 + + +def test_fix_patient_id_never_truncates_an_overlong_id(): + """Ticket 47: every malformed ID is recovered or sentinelled, never cut. + + A malformed ID is recovered against a well-formed ID in the same tracker + (edit distance 1, unique candidate only) or sentinelled -- so the output + only ever contains an ID some tracker actually spells. + """ + df = pl.DataFrame( + { + "patient_id": [ + "KD_EW004", # Valid + "KD-EW004", # Normalize - to _ + "11_EW004", # Digits instead of letters, two edits away + "KD_EW004XY", # Over-length, two edits away + None, # Null + "", # Empty + ], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + expected = [ + "KD_EW004", # Valid + "KD_EW004", # Normalized + "Undefined", # No unambiguous candidate + "Undefined", # Not truncated to KD_EW004 + None, # Null + "Undefined", # Empty + ] + assert result["patient_id"].to_list() == expected + assert len(collector) == 3 + + +def test_fix_patient_id_recovers_a_single_character_typo(): + """The other half of the same divergence: one edit, one candidate, recover. + + These four spellings are all one edit from the valid `KD_EW004` the same + frame carries, which is the 2023_NPH shape that opened ticket 47. + """ + df = pl.DataFrame( + { + "patient_id": [ + "KD_EW004", # Valid, the recovery target + "K_EW004", # Dropped letter + "KD_E004", # Dropped letter + "KD_EWX04", # Substituted character + "KD_E1004", # Substituted character + ], + } + ) + + collector = ErrorCollector() + result = fix_patient_id(df, collector) + + assert result["patient_id"].to_list() == ["KD_EW004"] * 5 + # Recovered, but still four defective source cells to report. + assert len(collector) == 4 + + +def test_validate_allowed_values_maps_a_configured_alias_to_its_canonical_form(): + """Ticket 29: the pre-2024 template's spelling is an alias, not a value. + + The source carries both "Active - Remote" (2020-2023 trackers) and + "Active Remote" (2024+ trackers, and the only spelling the current + Lookup List dropdown defines). One canonical label must reach the + reports, and which one is canonical is stated in the config rather than + implied by list order. + """ + collector = ErrorCollector() + df = pl.DataFrame({"status": ["Active - Remote", "active remote", "Active"]}) + + result = validate_allowed_values( + df=df, + column="status", + allowed_values=["Active", "Active Remote"], + aliases={"Active Remote": ["Active - Remote"]}, + error_collector=collector, + ) + + assert result["status"].to_list() == ["Active Remote", "Active Remote", "Active"] + + +def test_validate_allowed_values_rejects_two_allowed_values_that_sanitize_alike(): + """Two canonical values reducing to one key made the winner depend on + dict-insertion order (ticket 29). Fail loudly instead of picking one.""" + collector = ErrorCollector() + df = pl.DataFrame({"status": ["Active Remote"]}) + + with pytest.raises(ValueError, match="sanitize to the same key"): + validate_allowed_values( + df=df, + column="status", + allowed_values=["Active - Remote", "Active Remote"], + error_collector=collector, + ) + + +def test_validate_allowed_values_rejects_aliases_for_a_non_allowed_canonical(): + """Aliases declared under a label that is not a canonical value are a + config typo.""" + collector = ErrorCollector() + df = pl.DataFrame({"status": ["Active"]}) + + with pytest.raises(ValueError, match="not an allowed value"): + validate_allowed_values( + df=df, + column="status", + allowed_values=["Active"], + aliases={"Active Remotte": ["Active - Remote"]}, + error_collector=collector, + ) diff --git a/tests/test_cli/__init__.py b/tests/test_cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_cli/conftest.py b/tests/test_cli/conftest.py new file mode 100644 index 0000000..667d084 --- /dev/null +++ b/tests/test_cli/conftest.py @@ -0,0 +1,87 @@ +"""Fixtures for CLI tests, including a minimal valid dummy tracker file.""" + +from pathlib import Path + +import openpyxl +import pytest + + +@pytest.fixture +def dummy_tracker(tmp_path) -> Path: + """Create a minimal valid A4D Excel tracker file for testing. + + Structure follows the actual tracker format: + - Sheet "Jan24" (month abbreviation + 2-digit year) + - Row 1: empty (no header, data_start_row - 2 → header_2 path) + - Row 2: column headers (data_start_row - 1 → header_1 path) + - Row 3+: patient data rows (col A = numeric row number) + + The clinic_id is derived from the parent folder name ("TST"). + """ + clinic_dir = tmp_path / "TST" + clinic_dir.mkdir() + tracker_path = clinic_dir / "2024_Test_Clinic.xlsx" + + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "Jan24" + + # Row 1: empty title row → header_2 (≤2 non-None values triggers header_1-only path) + # Row 2: column headers → header_1 + # "Patient ID" in header_1 + empty header_2 → merge_headers uses header_1 only + ws.cell(2, 2).value = "Patient ID" + ws.cell(2, 3).value = "Name" + ws.cell(2, 4).value = "Sex" + ws.cell(2, 5).value = "Age" + + # Row 3+: data rows — col A must be numeric (find_data_start_row scans for first int/float) + ws.cell(3, 1).value = 1 + ws.cell(3, 2).value = "PT-001" + ws.cell(3, 3).value = "Test Patient One" + ws.cell(3, 4).value = "Female" + ws.cell(3, 5).value = 25 + + ws.cell(4, 1).value = 2 + ws.cell(4, 2).value = "PT-002" + ws.cell(4, 3).value = "Test Patient Two" + ws.cell(4, 4).value = "Male" + ws.cell(4, 5).value = 30 + + wb.save(tracker_path) + return tracker_path + + +@pytest.fixture +def dummy_tracker_dir(dummy_tracker) -> Path: + """Return the directory containing the dummy tracker (data root for batch mode).""" + return dummy_tracker.parent.parent + + +@pytest.fixture +def dummy_product_tracker(tmp_path) -> Path: + """Create a minimal valid A4D Excel tracker file with a product section. + + Headers use real synonyms_product.yaml canonical names so the CLI's + default (non-stub) column mapper resolves them without mocking. + """ + clinic_dir = tmp_path / "TST" + clinic_dir.mkdir() + tracker_path = clinic_dir / "2024_Test_Clinic.xlsx" + + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "Jan24" + + rows = [ + [None, None, None, None, None], + ["Product", "Date", "Received From", "Units Received", "Units Released"], + ["Insulin A", "2024-01-05", "HQ", 100, 5], + ["Insulin B", "2024-01-10", "HQ", 50, None], + ["Patient Recruitment", None, None, None, None], + ["Patient Name", "Patient ID", None, None, None], + ] + for row in rows: + ws.append(row) + + wb.save(tracker_path) + return tracker_path diff --git a/tests/test_cli/test_cli.py b/tests/test_cli/test_cli.py new file mode 100644 index 0000000..21f6f95 --- /dev/null +++ b/tests/test_cli/test_cli.py @@ -0,0 +1,629 @@ +"""Tests for the A4D CLI commands.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import polars as pl +from typer.testing import CliRunner + +from a4d.cli import app +from a4d.errors import DataError +from a4d.pipeline.models import PipelineResult, TrackerResult + +runner = CliRunner(env={"NO_COLOR": "1", "COLUMNS": "200"}) + + +# --------------------------------------------------------------------------- +# Help / invocation smoke tests +# --------------------------------------------------------------------------- + + +class TestHelp: + """Verify every command exposes --help without error.""" + + def test_app_help(self): + result = runner.invoke(app, ["--help"]) + assert result.exit_code == 0 + assert "run" in result.output + + def test_run_patient_help(self): + result = runner.invoke(app, ["run", "patient", "--help"]) + assert result.exit_code == 0 + assert "--file" in result.output + + def test_create_tables_help(self): + result = runner.invoke(app, ["create", "tables", "--help"]) + assert result.exit_code == 0 + assert "--output" in result.output + + def test_upload_tables_help(self): + result = runner.invoke(app, ["upload", "tables", "--help"]) + assert result.exit_code == 0 + assert "--tables-dir" in result.output + + def test_run_pipeline_help(self): + result = runner.invoke(app, ["run", "--help"]) + assert result.exit_code == 0 + assert "--skip-download" in result.output + assert "--skip-upload" in result.output + assert "--skip-patient" in result.output + + +# --------------------------------------------------------------------------- +# Error-path unit tests (no real files needed) +# --------------------------------------------------------------------------- + + +class TestCreateTablesErrors: + """create tables command error handling.""" + + def test_no_cleaned_dirs_exits_nonzero(self, tmp_path): + # Output root exists but has neither patient_data_cleaned/ nor + # product_data_cleaned/ — nothing to build tables from. + result = runner.invoke(app, ["create", "tables", "--output", str(tmp_path)]) + assert result.exit_code == 1 + assert "nothing to build tables from" in result.output + + def test_missing_output_dir_exits_nonzero(self, tmp_path): + missing = tmp_path / "nonexistent" + result = runner.invoke(app, ["create", "tables", "--output", str(missing)]) + assert result.exit_code == 1 + + +class TestUploadTablesErrors: + """upload tables command error handling.""" + + def test_missing_dir_exits_nonzero(self, tmp_path): + missing = tmp_path / "nonexistent_tables" + result = runner.invoke(app, ["upload", "tables", "--tables-dir", str(missing)]) + assert result.exit_code == 1 + assert "not found" in result.output.lower() + + +# --------------------------------------------------------------------------- +# `run` unit test (GCS/BQ mocked) +# --------------------------------------------------------------------------- + + +class TestRunPipeline: + """`run` command with mocked GCP calls.""" + + @patch("a4d.cli.run_product_pipeline") + @patch("a4d.cli.run_patient_pipeline") + @patch("a4d.config.settings") + def test_errors_table_holds_both_arms( + self, mock_settings, mock_run_patient, mock_run_product, tmp_path + ): + """`run` is the production entry point, and until ticket 32 it published + an errors table holding the patient arm only: the patient arm writes the + table from inside its own pipeline, and nothing wrote the product arm's + at all. The source-defect report (ticket 40) derives from this table, so + a silently patient-only table loses every product finding.""" + mock_settings.data_root = tmp_path / "data" + mock_settings.output_root = tmp_path / "output" + mock_settings.project_id = "test-project" + mock_settings.dataset = "test-dataset" + mock_settings.max_workers = 4 + (tmp_path / "data").mkdir() + (tmp_path / "output").mkdir() + + def _result(arm: str) -> PipelineResult: + error = DataError( + file_name=f"{arm}.xlsx", + patient_id="P1", + column="c", + original_value="v", + error_message=f"{arm} problem", + error_code="invalid_value", + script="clean", + function_name="f", + ) + return PipelineResult( + tracker_results=[ + TrackerResult( + tracker_file=Path(f"{arm}.xlsx"), + tracker_name=arm, + success=True, + cleaning_errors=1, + data_errors=[error], + ) + ], + tables={}, + total_trackers=1, + successful_trackers=1, + failed_trackers=0, + success=True, + ) + + mock_run_patient.return_value = _result("patient") + mock_run_product.return_value = _result("product") + + result = runner.invoke( + app, ["run", "--skip-download", "--skip-upload", "--skip-drive-download"] + ) + + assert result.exit_code == 0 + errors = pl.read_parquet(tmp_path / "output" / "tables" / "table_errors.parquet") + assert set(errors["file_name"].to_list()) == {"patient.xlsx", "product.xlsx"} + + @patch("a4d.cli.run_product_pipeline") + @patch("a4d.cli.run_patient_pipeline") + @patch("a4d.config.settings") + def test_skip_upload_calls_pipeline( + self, mock_settings, mock_run_patient, mock_run_product, tmp_path + ): + # Both arms must be mocked, not just patient: `run_product_pipeline` + # binds `settings` at its own module-import time, so patching + # `a4d.config.settings` alone does not stop it discovering trackers + # from the *real* local data_root if left unmocked. + mock_settings.data_root = tmp_path / "data" + mock_settings.output_root = tmp_path / "output" + mock_settings.project_id = "test-project" + mock_settings.dataset = "test-dataset" + mock_settings.max_workers = 4 + + (tmp_path / "data").mkdir() + (tmp_path / "output").mkdir() + + empty_result = PipelineResult( + tracker_results=[], + tables={}, + total_trackers=0, + successful_trackers=0, + failed_trackers=0, + success=True, + ) + mock_run_patient.return_value = empty_result + mock_run_product.return_value = empty_result + + result = runner.invoke( + app, ["run", "--skip-download", "--skip-upload", "--skip-drive-download"] + ) + + mock_run_patient.assert_called_once() + mock_run_product.assert_called_once() + assert result.exit_code == 0 + + @patch("a4d.cli.run_product_pipeline") + @patch("a4d.cli.run_patient_pipeline") + @patch("a4d.config.settings") + def test_patient_failure_soft_fails_and_continues( + self, mock_settings, mock_run_patient, mock_run_product, tmp_path + ): + """A patient tracker failure warns and continues rather than aborting + the run (product/tables/upload still execute) — matches the product + arm's existing soft-fail posture.""" + mock_settings.data_root = tmp_path / "data" + mock_settings.output_root = tmp_path / "output" + mock_settings.project_id = "test-project" + mock_settings.dataset = "test-dataset" + mock_settings.max_workers = 4 + + (tmp_path / "data").mkdir() + (tmp_path / "output").mkdir() + + mock_run_patient.return_value = PipelineResult( + tracker_results=[_tracker_result("bad", success=False, error="Parse error")], + tables={}, + total_trackers=1, + successful_trackers=0, + failed_trackers=1, + success=False, + ) + mock_run_product.return_value = PipelineResult( + tracker_results=[], + tables={}, + total_trackers=0, + successful_trackers=0, + failed_trackers=0, + success=True, + ) + + result = runner.invoke( + app, ["run", "--skip-download", "--skip-upload", "--skip-drive-download"] + ) + + assert result.exit_code == 0, result.output + assert "some patient trackers failed" in result.output.lower() + + @patch("a4d.cli.run_product_pipeline") + @patch("a4d.cli.run_patient_pipeline") + @patch("a4d.config.settings") + def test_skip_patient_runs_product_only( + self, mock_settings, mock_run_patient, mock_run_product, tmp_path + ): + mock_settings.data_root = tmp_path / "data" + mock_settings.output_root = tmp_path / "output" + mock_settings.project_id = "test-project" + mock_settings.dataset = "test-dataset" + mock_settings.max_workers = 4 + + (tmp_path / "data").mkdir() + (tmp_path / "output").mkdir() + + mock_product = MagicMock() + mock_product.success = True + mock_product.total_trackers = 0 + mock_product.successful_trackers = 0 + mock_product.failed_trackers = 0 + mock_product.tracker_results = [] + mock_run_product.return_value = mock_product + + result = runner.invoke( + app, + [ + "run", + "--skip-patient", + "--skip-download", + "--skip-upload", + "--skip-drive-download", + ], + ) + + assert result.exit_code == 0, f"Pipeline failed:\n{result.output}" + mock_run_patient.assert_not_called() + mock_run_product.assert_called_once() + + def test_skip_patient_and_skip_product_mutually_exclusive(self, tmp_path): + result = runner.invoke( + app, + [ + "run", + "--skip-patient", + "--skip-product", + "--skip-download", + "--skip-upload", + "--skip-drive-download", + ], + ) + assert result.exit_code == 1 + assert "mutually exclusive" in result.output.lower() + + +# --------------------------------------------------------------------------- +# Combined patient+product run summary (ticket 11) +# --------------------------------------------------------------------------- + + +def _tracker_result(name, *, success=True, error=None, cleaning_errors=0): + return TrackerResult( + tracker_file=Path(f"{name}.xlsx"), + tracker_name=name, + success=success, + error=error, + cleaning_errors=cleaning_errors, + ) + + +class TestCombinedRunSummary: + """`run`'s combined patient+product summary, only shown when both arms ran.""" + + def _run( + self, mock_settings, mock_run_patient, mock_run_product, tmp_path, patient_trs, product_trs + ): + mock_settings.data_root = tmp_path / "data" + mock_settings.output_root = tmp_path / "output" + mock_settings.project_id = "test-project" + mock_settings.dataset = "test-dataset" + mock_settings.max_workers = 4 + (tmp_path / "data").mkdir() + (tmp_path / "output").mkdir() + + patient_result = PipelineResult( + tracker_results=patient_trs, + tables={}, + total_trackers=len(patient_trs), + successful_trackers=sum(tr.success for tr in patient_trs), + failed_trackers=sum(not tr.success for tr in patient_trs), + success=all(tr.success for tr in patient_trs), + ) + product_result = PipelineResult( + tracker_results=product_trs, + tables={}, + total_trackers=len(product_trs), + successful_trackers=sum(tr.success for tr in product_trs), + failed_trackers=sum(not tr.success for tr in product_trs), + success=all(tr.success for tr in product_trs), + ) + mock_run_patient.return_value = patient_result + mock_run_product.return_value = product_result + + return runner.invoke( + app, + ["run", "--skip-download", "--skip-upload", "--skip-drive-download"], + ) + + @patch("a4d.cli.run_product_pipeline") + @patch("a4d.cli.run_patient_pipeline") + @patch("a4d.config.settings") + def test_both_arms_ok(self, mock_settings, mock_run_patient, mock_run_product, tmp_path): + patient_trs = [_tracker_result("2024_Clinic_A")] + product_trs = [_tracker_result("2024_Clinic_A")] + result = self._run( + mock_settings, mock_run_patient, mock_run_product, tmp_path, patient_trs, product_trs + ) + + assert result.exit_code == 0, result.output + assert "Combined Run Summary" in result.output + assert "Files needing attention" not in result.output + + @patch("a4d.cli.run_product_pipeline") + @patch("a4d.cli.run_patient_pipeline") + @patch("a4d.config.settings") + def test_patient_failed_only(self, mock_settings, mock_run_patient, mock_run_product, tmp_path): + patient_trs = [_tracker_result("2024_Clinic_B", success=False, error="bad header")] + product_trs = [_tracker_result("2024_Clinic_B")] + result = self._run( + mock_settings, mock_run_patient, mock_run_product, tmp_path, patient_trs, product_trs + ) + + assert result.exit_code == 0, result.output + assert "Files needing attention" in result.output + assert "2024_Clinic_B" in result.output + assert "bad header" in result.output + + @patch("a4d.cli.run_product_pipeline") + @patch("a4d.cli.run_patient_pipeline") + @patch("a4d.config.settings") + def test_file_lost_in_both_arms( + self, mock_settings, mock_run_patient, mock_run_product, tmp_path + ): + patient_trs = [_tracker_result("2024_Clinic_C", success=False, error="patient boom")] + product_trs = [_tracker_result("2024_Clinic_C", success=False, error="product boom")] + result = self._run( + mock_settings, mock_run_patient, mock_run_product, tmp_path, patient_trs, product_trs + ) + + assert result.exit_code == 0, result.output + assert "patient boom" in result.output + assert "product boom" in result.output + + @patch("a4d.cli.run_product_pipeline") + @patch("a4d.config.settings") + def test_omitted_when_product_arm_skipped(self, mock_settings, mock_run_product, tmp_path): + mock_settings.data_root = tmp_path / "data" + mock_settings.output_root = tmp_path / "output" + mock_settings.project_id = "test-project" + mock_settings.dataset = "test-dataset" + mock_settings.max_workers = 4 + (tmp_path / "data").mkdir() + (tmp_path / "output").mkdir() + + mock_run_product.return_value = PipelineResult( + tracker_results=[], + tables={}, + total_trackers=0, + successful_trackers=0, + failed_trackers=0, + success=True, + ) + + result = runner.invoke( + app, + [ + "run", + "--skip-patient", + "--skip-download", + "--skip-upload", + "--skip-drive-download", + ], + ) + + assert result.exit_code == 0, result.output + assert "Combined Run Summary" not in result.output + + +# --------------------------------------------------------------------------- +# End-to-end test: run patient with real dummy tracker +# --------------------------------------------------------------------------- + + +class TestProcessPatientE2E: + """End-to-end test for run patient using a synthetic tracker file.""" + + def test_process_single_file_creates_outputs(self, dummy_tracker, tmp_path): + """run patient --file --output should produce parquet outputs.""" + output_dir = tmp_path / "output" + + result = runner.invoke( + app, + [ + "run", + "patient", + "--file", + str(dummy_tracker), + "--output", + str(output_dir), + ], + ) + + assert result.exit_code == 0, f"Pipeline failed:\n{result.output}" + + # Raw parquet should be created + raw_dir = output_dir / "patient_data_raw" + raw_files = list(raw_dir.glob("*_patient_raw.parquet")) + assert len(raw_files) == 1, f"Expected 1 raw parquet, found {len(raw_files)}" + + # Cleaned parquet should be created + cleaned_dir = output_dir / "patient_data_cleaned" + cleaned_files = list(cleaned_dir.glob("*_patient_cleaned.parquet")) + assert len(cleaned_files) == 1, f"Expected 1 cleaned parquet, found {len(cleaned_files)}" + + # Validate cleaned parquet has expected columns and rows + df_cleaned = pl.read_parquet(cleaned_files[0]) + assert "patient_id" in df_cleaned.columns + assert "clinic_id" in df_cleaned.columns + assert "tracker_year" in df_cleaned.columns + assert len(df_cleaned) == 2 # 2 patients in dummy file + + # clinic_id is derived from parent folder name + assert df_cleaned["clinic_id"].unique().to_list() == ["TST"] + assert df_cleaned["tracker_year"].unique().to_list() == [2024] + + def test_process_single_file_creates_tables(self, dummy_tracker, tmp_path): + """Tables (static, monthly, annual) should be created by default.""" + output_dir = tmp_path / "output" + + result = runner.invoke( + app, + [ + "run", + "patient", + "--file", + str(dummy_tracker), + "--output", + str(output_dir), + ], + ) + + assert result.exit_code == 0, f"Pipeline failed:\n{result.output}" + + tables_dir = output_dir / "tables" + assert (tables_dir / "patient_data_monthly.parquet").exists() + assert (tables_dir / "patient_data_static.parquet").exists() + + def test_skip_tables_flag(self, dummy_tracker, tmp_path): + """--skip-tables should skip table creation.""" + output_dir = tmp_path / "output" + + result = runner.invoke( + app, + [ + "run", + "patient", + "--file", + str(dummy_tracker), + "--output", + str(output_dir), + "--skip-tables", + ], + ) + + assert result.exit_code == 0, f"Pipeline failed:\n{result.output}" + + tables_dir = output_dir / "tables" + # Patient tables are skipped, but errors table is always written + skipped_names = [ + "patient_data_static.parquet", + "patient_data_monthly.parquet", + "patient_data_annual.parquet", + ] + for name in skipped_names: + assert not (tables_dir / name).exists(), f"{name} should not exist with --skip-tables" + assert (tables_dir / "table_errors.parquet").exists(), ( + "errors table should always be written" + ) + + def test_process_missing_file_exits_nonzero(self, tmp_path): + """Passing a non-existent file should exit with error.""" + missing = tmp_path / "ghost.xlsx" + output_dir = tmp_path / "output" + + result = runner.invoke( + app, + ["run", "patient", "--file", str(missing), "--output", str(output_dir)], + ) + + assert result.exit_code == 1 + + +class TestProcessProductE2E: + """End-to-end test for run product, mirroring TestProcessPatientE2E. + + Covers the logs/errors table steps added to process_product_cmd to + bring it to parity with process_patient_cmd (previously product never + created a logs or errors table at all). + """ + + def test_process_single_file_creates_outputs(self, dummy_product_tracker, tmp_path): + output_dir = tmp_path / "output" + + result = runner.invoke( + app, + [ + "run", + "product", + "--file", + str(dummy_product_tracker), + "--output", + str(output_dir), + ], + ) + + assert result.exit_code == 0, f"Pipeline failed:\n{result.output}" + + raw_dir = output_dir / "product_data_raw" + assert len(list(raw_dir.glob("*_product_raw.parquet"))) == 1 + + cleaned_dir = output_dir / "product_data_cleaned" + cleaned_files = list(cleaned_dir.glob("*_product_cleaned.parquet")) + assert len(cleaned_files) == 1 + + df_cleaned = pl.read_parquet(cleaned_files[0]) + assert "product" in df_cleaned.columns + assert "clinic_id" in df_cleaned.columns + assert df_cleaned["clinic_id"].unique().to_list() == ["TST"] + + def test_process_single_file_creates_tables(self, dummy_product_tracker, tmp_path): + """Product table, logs table, and errors table should all be created by default.""" + output_dir = tmp_path / "output" + + result = runner.invoke( + app, + [ + "run", + "product", + "--file", + str(dummy_product_tracker), + "--output", + str(output_dir), + ], + ) + + assert result.exit_code == 0, f"Pipeline failed:\n{result.output}" + + tables_dir = output_dir / "tables" + assert (tables_dir / "product_data.parquet").exists() + assert (tables_dir / "table_logs.parquet").exists(), ( + "logs table should be created (parity with run patient)" + ) + assert (tables_dir / "table_errors.parquet").exists(), ( + "errors table should be created (parity with run patient)" + ) + + def test_skip_tables_flag(self, dummy_product_tracker, tmp_path): + """--skip-tables should skip the product table, but errors table is always written.""" + output_dir = tmp_path / "output" + + result = runner.invoke( + app, + [ + "run", + "product", + "--file", + str(dummy_product_tracker), + "--output", + str(output_dir), + "--skip-tables", + ], + ) + + assert result.exit_code == 0, f"Pipeline failed:\n{result.output}" + + tables_dir = output_dir / "tables" + assert not (tables_dir / "product_data.parquet").exists() + assert not (tables_dir / "table_logs.parquet").exists() + assert (tables_dir / "table_errors.parquet").exists(), ( + "errors table should always be written" + ) + + def test_process_missing_file_exits_nonzero(self, tmp_path): + missing = tmp_path / "ghost.xlsx" + output_dir = tmp_path / "output" + + result = runner.invoke( + app, + ["run", "product", "--file", str(missing), "--output", str(output_dir)], + ) + + assert result.exit_code == 1 diff --git a/tests/test_cli/test_force_incremental.py b/tests/test_cli/test_force_incremental.py new file mode 100644 index 0000000..34cb9c2 --- /dev/null +++ b/tests/test_cli/test_force_incremental.py @@ -0,0 +1,207 @@ +"""Tests for --force and --incremental flag interaction across CLI commands.""" + +import hashlib +from pathlib import Path + +from typer.testing import CliRunner + +from a4d.cli import app + +runner = CliRunner(env={"NO_COLOR": "1", "COLUMNS": "200"}) + + +def _hash_dir(dir_path: Path) -> dict[str, str]: + """Return {relative_path: sha256} for the deterministic data outputs. + + Restricted to ``patient_data_cleaned/*.parquet`` and the patient-table + parquets. Skips ``logs/`` (per-tracker JSON contains timestamps) and + ``table_logs.parquet`` (aggregated log timestamps). + """ + out: dict[str, str] = {} + cleaned = dir_path / "patient_data_cleaned" + if cleaned.exists(): + for f in sorted(cleaned.glob("*.parquet")): + out[f"patient_data_cleaned/{f.name}"] = hashlib.sha256(f.read_bytes()).hexdigest() + tables = dir_path / "tables" + if tables.exists(): + for f in sorted(tables.glob("patient_data_*.parquet")): + out[f"tables/{f.name}"] = hashlib.sha256(f.read_bytes()).hexdigest() + return out + + +class TestHelpExposesForce: + """Every command that takes --incremental must also expose --force.""" + + def test_process_patient_help_mentions_force(self): + result = runner.invoke(app, ["run", "patient", "--help"]) + assert result.exit_code == 0 + assert "--force" in result.output + + def test_process_product_help_mentions_force(self): + result = runner.invoke(app, ["run", "product", "--help"]) + assert result.exit_code == 0 + assert "--force" in result.output + + def test_run_pipeline_help_mentions_force(self): + result = runner.invoke(app, ["run", "--help"]) + assert result.exit_code == 0 + assert "--force" in result.output + + +class TestForceFlag: + """--force semantics on run patient (the one CLI command we can drive end-to-end).""" + + def test_force_produces_same_output_as_default(self, dummy_tracker_dir, tmp_path): + """--force is an explicit synonym for the default — outputs must match byte-for-byte.""" + out_default = tmp_path / "out_default" + out_force = tmp_path / "out_force" + + r1 = runner.invoke( + app, + [ + "run", + "patient", + "--data-root", + str(dummy_tracker_dir), + "--output", + str(out_default), + ], + ) + assert r1.exit_code == 0, f"default run failed:\n{r1.output}" + + r2 = runner.invoke( + app, + [ + "run", + "patient", + "--data-root", + str(dummy_tracker_dir), + "--output", + str(out_force), + "--force", + ], + ) + assert r2.exit_code == 0, f"--force run failed:\n{r2.output}" + + assert _hash_dir(out_default) == _hash_dir(out_force) + + def test_force_is_deterministic(self, dummy_tracker_dir, tmp_path): + """Two consecutive --force runs into separate dirs produce identical outputs.""" + out_a = tmp_path / "a" + out_b = tmp_path / "b" + + for out in (out_a, out_b): + r = runner.invoke( + app, + [ + "run", + "patient", + "--data-root", + str(dummy_tracker_dir), + "--output", + str(out), + "--force", + ], + ) + assert r.exit_code == 0, f"run into {out} failed:\n{r.output}" + + assert _hash_dir(out_a) == _hash_dir(out_b) + + def test_force_wipes_existing_outputs(self, dummy_tracker_dir, tmp_path): + """--force must trigger the orchestrator's clean_output=True wipe. + + Pre-seed sentinel files in all four dirs the orchestrator wipes + (patient_data_raw, patient_data_cleaned, tables, logs — see + pipeline/patient.py:176) and confirm they're gone after --force runs. + Without this check the determinism tests would pass even if --force + silently became a no-op (since they hash output equivalence into fresh + empty dirs, not the wipe behavior itself). + """ + out = tmp_path / "out" + wipe_dirs = ("patient_data_raw", "patient_data_cleaned", "tables", "logs") + for subdir in wipe_dirs: + (out / subdir).mkdir(parents=True) + (out / subdir / "_sentinel").write_text("from prior run") + + result = runner.invoke( + app, + [ + "run", + "patient", + "--data-root", + str(dummy_tracker_dir), + "--output", + str(out), + "--force", + ], + ) + assert result.exit_code == 0, f"--force run failed:\n{result.output}" + + for subdir in wipe_dirs: + assert not (out / subdir / "_sentinel").exists(), ( + f"--force did not wipe {subdir}/_sentinel" + ) + + +class TestForceIncrementalConflict: + """--force overrides --incremental with a warning.""" + + def test_warning_printed_when_both_set(self, dummy_tracker_dir, tmp_path): + out = tmp_path / "out" + result = runner.invoke( + app, + [ + "run", + "patient", + "--data-root", + str(dummy_tracker_dir), + "--output", + str(out), + "--force", + "--incremental", + ], + ) + assert result.exit_code == 0, f"combined run failed:\n{result.output}" + assert "--incremental is ignored when --force is set" in result.output + + def test_combined_output_matches_force_alone(self, dummy_tracker_dir, tmp_path): + """--force --incremental should behave exactly like --force (incremental ignored).""" + out_force = tmp_path / "force" + out_both = tmp_path / "both" + + r_force = runner.invoke( + app, + [ + "run", + "patient", + "--data-root", + str(dummy_tracker_dir), + "--output", + str(out_force), + "--force", + ], + ) + assert r_force.exit_code == 0 + + r_both = runner.invoke( + app, + [ + "run", + "patient", + "--data-root", + str(dummy_tracker_dir), + "--output", + str(out_both), + "--force", + "--incremental", + ], + ) + assert r_both.exit_code == 0 + + assert _hash_dir(out_force) == _hash_dir(out_both) + + +# Note: a "second incremental run skips unchanged" test would need to seed the +# tracker_metadata manifest first — run patient does not create it (only +# create tables / the bare run do). That code path is exercised by +# tests/test_state/test_integration.py, so we don't duplicate it here. diff --git a/tests/test_docs_have_no_r_framing.py b/tests/test_docs_have_no_r_framing.py new file mode 100644 index 0000000..2fffa0f --- /dev/null +++ b/tests/test_docs_have_no_r_framing.py @@ -0,0 +1,77 @@ +"""Guard: no module explains itself by what the retired R pipeline did. + +The R pipeline was retired on 2026-08-24 and its source deleted. A comment +saying a rule exists "to match R" no longer explains anything -- it points at +code nobody can read from here -- and it can also be wrong about R: one +docstring claimed a future-date guard "matches R pipeline behavior" when R had +no such guard at all. Documentation should say what the code does now and why, +with the source evidence that made it necessary. + +``src/a4d/migration/`` and its tests are exempt: that package is the archived +R-vs-Python comparison, so R divergence is its actual subject. See its module +docstring. +""" + +import re +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] + +# Exempt: R is the legitimate subject there, not a stand-in for a reason. +EXEMPT = ( + REPO_ROOT / "src" / "a4d" / "migration", + REPO_ROOT / "tests" / "test_migration", + Path(__file__).resolve(), +) + +# Phrases that justify Python by R rather than by evidence, plus the R +# libraries and source files that only ever appear in such a justification. +# Assembled from parts so this file does not trip its own check. +_R = "R" +FORBIDDEN = re.compile( + "|".join( + [ + rf"\b{_R}'s\b", + rf"\b{_R} pipeline\b", + rf"\bmatch(?:es|ing)? {_R}\b", + rf"\blike {_R}\b", + rf"\bmirror(?:s|ing)? {_R}\b", + rf"\bas {_R} does\b", + rf"\b{_R} does\b", + rf"\b{_R}-parity\b", + rf"\b{_R} behaviou?r\b", + r"\bscript\d_[a-z_]+\.R\b", + r"\breadxl\b", + r"\bopenxlsx\b", + r"\bmake\.names\b", + r"\btidyr::", + r"\blubridate\b", + ] + ) +) + + +def _python_files() -> list[Path]: + """Every tracked Python file outside the exempt paths, derived not listed.""" + files = [] + for base in ("src", "tests"): + for path in (REPO_ROOT / base).rglob("*.py"): + if any(path == e or e in path.parents for e in EXEMPT): + continue + files.append(path) + return sorted(files) + + +@pytest.mark.parametrize("path", _python_files(), ids=lambda p: str(p.relative_to(REPO_ROOT))) +def test_module_does_not_explain_itself_by_the_retired_r_pipeline(path: Path) -> None: + offenders = [ + f"{path.relative_to(REPO_ROOT)}:{n}: {line.strip()}" + for n, line in enumerate(path.read_text().splitlines(), start=1) + if FORBIDDEN.search(line) + ] + assert not offenders, ( + "Documentation must explain the current behaviour and its evidence, not " + "defer to the retired R pipeline:\n" + "\n".join(offenders) + ) diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 0000000..84196da --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,167 @@ +"""Tests for error tracking functionality.""" + +import polars as pl + +from a4d.errors import DataError, ErrorCollector + + +def test_data_error_creation(): + """Test creating a DataError instance.""" + error = DataError( + file_name="test.xlsx", + patient_id="XX_YY001", + column="age", + original_value="invalid", + error_message="Could not convert to Int32", + error_code="type_conversion", + function_name="safe_convert_column", + ) + + assert error.file_name == "test.xlsx" + assert error.patient_id == "XX_YY001" + assert error.column == "age" + assert error.error_code == "type_conversion" + assert error.script == "clean" # default value + + +def test_error_collector_add_error(): + """Test adding errors to collector.""" + collector = ErrorCollector() + + assert len(collector) == 0 + assert not collector # __bool__ returns False when empty + + collector.add_error( + file_name="test.xlsx", + patient_id="XX_YY001", + column="age", + original_value="invalid", + error_message="Could not convert", + error_code="type_conversion", + ) + + assert len(collector) == 1 + assert collector # __bool__ returns True when has errors + + +def test_error_collector_add_errors(): + """Test adding multiple errors at once.""" + collector = ErrorCollector() + + errors = [ + DataError( + file_name="test.xlsx", + patient_id="XX_YY001", + column="age", + original_value="invalid", + error_message="Could not convert", + error_code="type_conversion", + ), + DataError( + file_name="test.xlsx", + patient_id="XX_YY002", + column="weight", + original_value="abc", + error_message="Could not convert", + error_code="type_conversion", + ), + ] + + collector.add_errors(errors) + + assert len(collector) == 2 + + +def test_error_collector_to_dataframe(): + """Test converting errors to DataFrame.""" + collector = ErrorCollector() + + collector.add_error( + file_name="test.xlsx", + patient_id="XX_YY001", + column="age", + original_value="invalid", + error_message="Could not convert to Int32", + error_code="type_conversion", + function_name="safe_convert_column", + ) + + df = collector.to_dataframe() + + assert isinstance(df, pl.DataFrame) + assert len(df) == 1 + assert "file_name" in df.columns + assert "patient_id" in df.columns + assert "column" in df.columns + assert "error_code" in df.columns + + # Check categorical columns + assert df.schema["error_code"] == pl.Categorical + assert df.schema["script"] == pl.Categorical + + +def test_error_collector_to_dataframe_empty(): + """Test converting empty collector to DataFrame.""" + collector = ErrorCollector() + df = collector.to_dataframe() + + assert isinstance(df, pl.DataFrame) + assert len(df) == 0 + # Should still have correct schema + assert "file_name" in df.columns + assert "error_code" in df.columns + + +def test_error_collector_get_summary(): + """Test error summary by error_code.""" + collector = ErrorCollector() + + collector.add_error( + file_name="test.xlsx", + patient_id="XX_YY001", + column="age", + original_value="invalid", + error_message="Type error", + error_code="type_conversion", + ) + collector.add_error( + file_name="test.xlsx", + patient_id="XX_YY002", + column="age", + original_value="999", + error_message="Out of range", + error_code="invalid_value", + ) + collector.add_error( + file_name="test.xlsx", + patient_id="XX_YY003", + column="weight", + original_value="abc", + error_message="Type error", + error_code="type_conversion", + ) + + summary = collector.get_error_summary() + + assert summary == {"type_conversion": 2, "invalid_value": 1} + + +def test_error_collector_clear(): + """Test clearing errors from collector.""" + collector = ErrorCollector() + + collector.add_error( + file_name="test.xlsx", + patient_id="XX_YY001", + column="age", + original_value="invalid", + error_message="Error", + error_code="type_conversion", + ) + + assert len(collector) == 1 + + collector.clear() + + assert len(collector) == 0 + assert not collector diff --git a/tests/test_extract/__init__.py b/tests/test_extract/__init__.py new file mode 100644 index 0000000..1690af8 --- /dev/null +++ b/tests/test_extract/__init__.py @@ -0,0 +1 @@ +"""Tests for data extraction modules.""" diff --git a/tests/test_extract/test_common.py b/tests/test_extract/test_common.py new file mode 100644 index 0000000..e3c3608 --- /dev/null +++ b/tests/test_extract/test_common.py @@ -0,0 +1,98 @@ +"""Unit tests for shared tracker-level extraction helpers.""" + +from pathlib import Path + +import polars as pl +import pytest + +from a4d.extract.common import ( + extract_tracker_month, + find_month_sheets, + get_tracker_year, + normalize_patient_id_expr, +) + + +class _StubWorkbook: + """Minimal stand-in for openpyxl.Workbook that exposes ``sheetnames``.""" + + def __init__(self, sheetnames: list[str]): + self.sheetnames = sheetnames + + +def test_get_tracker_year_from_sheet_names(): + year = get_tracker_year(Path("anything.xlsx"), ["Jan24", "Feb24", "Mar24"]) + assert year == 2024 + + +def test_get_tracker_year_falls_back_to_filename(): + year = get_tracker_year(Path("2023_clinic_tracker.xlsx"), ["January", "February"]) + assert year == 2023 + + +def test_get_tracker_year_raises_when_unparseable(): + with pytest.raises(ValueError, match="Could not determine year"): + get_tracker_year(Path("clinic_tracker.xlsx"), ["January", "February"]) + + +def test_find_month_sheets_filters_and_sorts(): + wb = _StubWorkbook(["Cover", "Mar24", "Jan24", "Feb24", "Notes"]) + sheets = find_month_sheets(wb) + assert sheets == ["Jan24", "Feb24", "Mar24"] + + +def test_extract_tracker_month_known_prefix(): + assert extract_tracker_month("Jan24") == 1 + assert extract_tracker_month("Dec23") == 12 + + +def test_extract_tracker_month_unknown_prefix(): + with pytest.raises(ValueError, match="Could not extract month"): + extract_tracker_month("Foo24") + + +def test_re_export_from_patient_module_still_works(): + """Backward-compat re-export shim stays callable from a4d.extract.patient.""" + from a4d.extract.patient import ( + extract_tracker_month as p_extract_tracker_month, + ) + from a4d.extract.patient import ( + find_month_sheets as p_find_month_sheets, + ) + from a4d.extract.patient import ( + get_tracker_year as p_get_tracker_year, + ) + + assert p_extract_tracker_month is extract_tracker_month + assert p_find_month_sheets is find_month_sheets + assert p_get_tracker_year is get_tracker_year + + +def _normalize(ids: list[str | None]) -> list[str | None]: + return ( + pl.DataFrame({"patient_id": ids}, schema={"patient_id": pl.Utf8}) + .select(normalize_patient_id_expr(pl.col("patient_id")).alias("k"))["k"] + .to_list() + ) + + +def test_normalize_patient_id_folds_hyphen_to_underscore(): + """The Mahosot case: month sheets write LA-MH056, the Patient List LA_MH056.""" + assert _normalize(["LA-MH056", "LA_MH056"]) == ["LA_MH056", "LA_MH056"] + + +def test_normalize_patient_id_strips_transfer_clinic_suffix(): + assert _normalize(["MY_SM003_SB", "LA-MH093_LF"]) == ["MY_SM003", "LA_MH093"] + + +def test_normalize_patient_id_leaves_well_formed_ids_alone(): + assert _normalize(["KH_NP026", "TH_ST029"]) == ["KH_NP026", "TH_ST029"] + + +def test_normalize_patient_id_passes_through_ids_without_separator(): + """An ID with no underscore has no country/clinic split to extract.""" + assert _normalize(["ABC123", ""]) == ["ABC123", ""] + + +def test_normalize_patient_id_preserves_null(): + assert _normalize([None, "LA-MH056"]) == [None, "LA_MH056"] diff --git a/tests/test_extract/test_patient.py b/tests/test_extract/test_patient.py new file mode 100644 index 0000000..aab1d04 --- /dev/null +++ b/tests/test_extract/test_patient.py @@ -0,0 +1,701 @@ +"""Tests for patient data extraction.""" + +from pathlib import Path + +import polars as pl +import pytest + +from a4d.extract.patient import ( + extract_patient_data, + extract_tracker_month, + find_month_sheets, + get_tracker_year, + harmonize_patient_data_columns, + merge_duplicate_columns_data, + read_all_patient_sheets, +) + + +def column_letter_to_index(col_letter: str) -> int: + """Convert Excel column letter to 0-based index. + + Examples: + A -> 0, B -> 1, Z -> 25, AA -> 26, AB -> 27, AC -> 28 + """ + result = 0 + for char in col_letter: + result = result * 26 + (ord(char) - ord("A") + 1) + return result - 1 + + +def calculate_expected_columns(start_col: str, end_col: str) -> int: + """Calculate expected number of columns from Excel range. + + Args: + start_col: Starting column letter (e.g., 'B') + end_col: Ending column letter (e.g., 'AC') + + Returns: + Number of columns in the range + + Examples: + B to Z: 25 columns + B to AC: 28 columns + B to AB: 27 columns + """ + start_idx = column_letter_to_index(start_col) + end_idx = column_letter_to_index(end_col) + return end_idx - start_idx + 1 + + +# Test data paths +TRACKER_SBU_2024 = Path( + "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/a4dphase2_upload/" + "Malaysia/SBU/2024_Sibu Hospital A4D Tracker.xlsx" +) +TRACKER_PNG_2019 = Path( + "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/a4dphase2_upload/" + "Malaysia/PNG/2019_Penang General Hospital A4D Tracker_DC.xlsx" +) +TRACKER_PNG_2018 = Path( + "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/a4dphase2_upload/" + "Malaysia/PNG/2018_Penang General Hospital A4D Tracker_DC.xlsx" +) +TRACKER_MHS_2017 = Path( + "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/a4dphase2_upload/" + "Laos/MHS/2017_Mahosot Hospital A4D Tracker.xlsx" +) +TRACKER_MHS_2025 = Path( + "/Volumes/USB SanDisk 3.2Gen1 Media/a4d/a4dphase2_upload/" + "Laos/MHS/2025_06_Mahosot Hospital A4D Tracker.xlsx" +) + + +@pytest.mark.skipif(not TRACKER_SBU_2024.exists(), reason="Tracker file not available") +def test_get_tracker_year_from_sheet_names(): + """Test extracting year from sheet names.""" + year = get_tracker_year(TRACKER_SBU_2024, ["Jan24", "Feb24", "Mar24"]) + assert year == 2024 + + +@pytest.mark.skipif(not TRACKER_SBU_2024.exists(), reason="Tracker file not available") +def test_get_tracker_year_from_filename(): + """Test extracting year from filename as fallback.""" + year = get_tracker_year(TRACKER_SBU_2024, ["January", "February"]) + assert year == 2024 + + +@pytest.mark.skipif(not TRACKER_SBU_2024.exists(), reason="Tracker file not available") +def test_find_month_sheets_2024(): + """Test finding month sheets in 2024 tracker.""" + from openpyxl import load_workbook + + wb = load_workbook(TRACKER_SBU_2024, data_only=True) + month_sheets = find_month_sheets(wb) + + assert len(month_sheets) > 0 + assert any("Jan" in sheet for sheet in month_sheets) + assert any("Dec" in sheet for sheet in month_sheets) + + +# Parameterized test data: (tracker_file, sheet_name, year, expected_patients, expected_cols, notes) +# Note: expected_cols is the actual number after filtering out None header columns +TRACKER_TEST_CASES = [ + # 2024 tracker - optimized single-pass extraction + ( + TRACKER_SBU_2024, + "Jan24", + 2024, + 4, + calculate_expected_columns("B", "AG") - 1, + "Single-pass read-only", + ), + # 2019 tracker - format changes across months! Optimized extraction + ( + TRACKER_PNG_2019, + "Jan19", + 2019, + 10, + calculate_expected_columns("B", "Z"), + "Single-pass read-only", + ), + ( + TRACKER_PNG_2019, + "Feb19", + 2019, + 10, + calculate_expected_columns("B", "AC"), + "Single-pass read-only", + ), + ( + TRACKER_PNG_2019, + "Mar19", + 2019, + 10, + calculate_expected_columns("B", "AB"), + "Single-pass read-only", + ), + ( + TRACKER_PNG_2019, + "Oct19", + 2019, + 11, + calculate_expected_columns("B", "AB"), + "Single-pass read-only", + ), + # 2018 tracker - single-line headers + ( + TRACKER_PNG_2018, + "Dec18", + 2018, + 10, + calculate_expected_columns("B", "T"), + "Single-pass read-only", + ), +] + + +@pytest.mark.skipif( + any(not tf.exists() for tf, _, _, _, _, _ in TRACKER_TEST_CASES), + reason="Tracker files not available", +) +@pytest.mark.parametrize( + ("tracker_file", "sheet_name", "year", "expected_patients", "expected_cols", "notes"), + TRACKER_TEST_CASES, + ids=lambda params: f"{params[1] if isinstance(params, tuple) and len(params) > 1 else params}", +) +def test_extract_patient_data_schema( + tracker_file, sheet_name, year, expected_patients, expected_cols, notes +): + """Test patient data extraction with schema validation across different months. + + This parameterized test validates that: + 1. Correct number of patients are extracted + 2. Correct number of columns match expected (after filtering None headers) + 3. Format changes between months are handled correctly + + The test is critical because tracker formats change even within the same year, + and data quality is inconsistent across different months. + """ + df = extract_patient_data(tracker_file, sheet_name, year) + + # Check dimensions + assert len(df) == expected_patients, ( + f"{sheet_name}: Expected {expected_patients} patients, got {len(df)}" + ) + assert len(df.columns) == expected_cols, ( + f"{sheet_name}: Expected {expected_cols} columns ({notes}), got {len(df.columns)}" + ) + + # Verify we have at least Patient ID column + assert any("patient" in col.lower() and "id" in col.lower() for col in df.columns), ( + f"{sheet_name}: Missing Patient ID column in {df.columns}" + ) + + print(f"\n{sheet_name}: {len(df)} patients × {len(df.columns)} columns ({notes}) ✓") + + +@pytest.mark.skipif(not TRACKER_SBU_2024.exists(), reason="Tracker file not available") +def test_extract_patient_data_2024_detailed(): + """Detailed test for 2024 tracker with patient ID validation.""" + df = extract_patient_data(TRACKER_SBU_2024, "Jan24", 2024) + + # Verify specific patient IDs + patient_ids = df["Patient ID*"].to_list() + assert patient_ids == ["MY_SU001", "MY_SU002", "MY_SU003", "MY_SU004"], ( + f"Expected MY_SU001-004, got {patient_ids}" + ) + + print(f"\n2024 Jan24 - Patient IDs: {patient_ids} ✓") + + +def test_harmonize_patient_data_columns_basic(): + """Test basic column harmonization with known synonyms.""" + raw_df = pl.DataFrame( + { + "Patient ID*": ["MY_SU001", "MY_SU002"], + "Age": [25, 30], + "D.O.B.": ["1998-01-15", "1993-06-20"], + } + ) + + harmonized = harmonize_patient_data_columns(raw_df) + + # Check that columns were renamed to standardized names + assert "patient_id" in harmonized.columns + assert "age" in harmonized.columns + assert "dob" in harmonized.columns + + # Check that data is preserved + assert harmonized["patient_id"].to_list() == ["MY_SU001", "MY_SU002"] + assert harmonized["age"].to_list() == [25, 30] + + +def test_harmonize_patient_data_columns_multiple_synonyms(): + """Test that multiple columns mapping to same name are merged, not dropped. + + The 2023 template's complication screening block splits one canonical column + across B.P./Kidney/Eye/Foot/Lipids sub-columns, each independently populated, + so keeping only the first discards real values -- 2,489 of them across 27 + trackers. Absent sub-columns contribute nothing rather than a literal "NA". + """ + raw_df = pl.DataFrame( + { + "Patient ID": ["P001", None], + "ID": ["P002", None], + "Patient ID*": [None, "P003"], + } + ) + + harmonized = harmonize_patient_data_columns(raw_df) + + assert list(harmonized.columns) == ["patient_id"] + assert harmonized["patient_id"].to_list() == ["P001,P002", "P003"] + + +def test_harmonize_patient_data_columns_unmapped_strict_false(): + """Test that unmapped columns are kept when strict=False (default).""" + raw_df = pl.DataFrame( + { + "Patient ID*": ["MY_SU001"], + "Age": [25], + "UnknownColumn": ["some value"], + } + ) + + harmonized = harmonize_patient_data_columns(raw_df, strict=False) + + # Mapped columns should be renamed + assert "patient_id" in harmonized.columns + assert "age" in harmonized.columns + + # Unmapped column should be kept as-is + assert "UnknownColumn" in harmonized.columns + + +def test_harmonize_patient_data_columns_unmapped_strict_true(): + """Test that unmapped columns raise error when strict=True.""" + raw_df = pl.DataFrame( + { + "Patient ID*": ["MY_SU001"], + "UnknownColumn": ["some value"], + } + ) + + with pytest.raises(ValueError, match="Unmapped columns found"): + harmonize_patient_data_columns(raw_df, strict=True) + + +def test_harmonize_patient_data_columns_empty_dataframe(): + """Test harmonization with empty DataFrame.""" + raw_df = pl.DataFrame() + + harmonized = harmonize_patient_data_columns(raw_df) + + assert len(harmonized) == 0 + assert len(harmonized.columns) == 0 + + +@pytest.mark.skipif(not TRACKER_SBU_2024.exists(), reason="Tracker file not available") +def test_harmonize_real_tracker_data(): + """Test harmonization with real tracker data.""" + # Extract raw data + raw_df = extract_patient_data(TRACKER_SBU_2024, "Jan24", 2024) + + # Harmonize columns + harmonized = harmonize_patient_data_columns(raw_df) + + # Check that key columns were renamed + assert "patient_id" in harmonized.columns + assert "age" in harmonized.columns + + # Check that data is preserved + assert len(harmonized) == len(raw_df) # Same number of rows + assert harmonized["patient_id"].to_list() == ["MY_SU001", "MY_SU002", "MY_SU003", "MY_SU004"] + + +def test_extract_tracker_month(): + """Test extracting month number from sheet name.""" + assert extract_tracker_month("Jan24") == 1 + assert extract_tracker_month("Feb24") == 2 + assert extract_tracker_month("Mar19") == 3 + assert extract_tracker_month("Dec23") == 12 + + # Test with ValueError for invalid sheet names + with pytest.raises(ValueError, match="Could not extract month"): + extract_tracker_month("Sheet1") + + +def test_merge_duplicate_columns_data_no_duplicates(): + """Test that data without duplicate headers is unchanged.""" + headers = ["ID", "Name", "Age", "City"] + data = [["1", "Alice", "25", "NYC"], ["2", "Bob", "30", "LA"]] + + result_headers, result_data = merge_duplicate_columns_data(headers, data) + + assert result_headers == headers + assert result_data == data + + +def test_merge_duplicate_columns_data_with_duplicates(): + """Duplicate headers have their values comma-joined, not reduced to the first.""" + headers = ["ID", "DM Complications", "DM Complications", "DM Complications", "Age"] + data = [["1", "A", "B", "C", "25"], ["2", "X", "Y", "Z", "30"]] + + result_headers, result_data = merge_duplicate_columns_data(headers, data) + + assert result_headers == ["ID", "DM Complications", "Age"] + assert result_data == [["1", "A,B,C", "25"], ["2", "X,Y,Z", "30"]] + + +def test_merge_duplicate_columns_data_with_nulls(): + """Test merging duplicate columns with null values.""" + headers = ["ID", "DM Complications", "DM Complications", "DM Complications", "Age"] + data = [["1", "A", None, "C", "25"], ["2", None, "Y", None, "30"]] + + result_headers, result_data = merge_duplicate_columns_data(headers, data) + + assert result_headers == ["ID", "DM Complications", "Age"] + # Empty values are filtered out before joining + assert result_data == [["1", "A,C", "25"], ["2", "Y", "30"]] + + +def test_merge_duplicate_columns_data_all_nulls(): + """Test merging when all duplicate columns have null values.""" + headers = ["ID", "DM Complications", "DM Complications", "Age"] + data = [["1", None, None, "25"]] + + result_headers, result_data = merge_duplicate_columns_data(headers, data) + + assert result_headers == ["ID", "DM Complications", "Age"] + # All nulls result in None + assert result_data == [["1", None, "25"]] + + +def test_merge_duplicate_columns_data_multiple_groups(): + """Test merging multiple groups of duplicate columns.""" + headers = ["ID", "Status", "Status", "Value", "Value", "Value", "Name"] + data = [["1", "A", "B", "X", "Y", "Z", "Alice"]] + + result_headers, result_data = merge_duplicate_columns_data(headers, data) + + assert result_headers == ["ID", "Status", "Value", "Name"] + assert result_data == [["1", "A,B", "X,Y,Z", "Alice"]] + + +@pytest.mark.skipif(not TRACKER_SBU_2024.exists(), reason="Tracker file not available") +def test_read_all_patient_sheets_2024(): + """Test reading all patient sheets from 2024 tracker with Patient List and Annual.""" + df_all = read_all_patient_sheets(TRACKER_SBU_2024) + + # Check that we have data + assert len(df_all) > 0, "Should have extracted patient data" + + # Check that metadata columns were added + assert "sheet_name" in df_all.columns + assert "tracker_month" in df_all.columns + assert "tracker_year" in df_all.columns + assert "file_name" in df_all.columns + assert "clinic_id" in df_all.columns + + # Check that clinic_id is extracted from parent directory + clinic_ids = df_all["clinic_id"].unique().to_list() + assert len(clinic_ids) == 1 # All rows should have same clinic_id + assert clinic_ids[0] == "SBU" # Parent directory name + + # Check that we have data from multiple months + unique_months = df_all["tracker_month"].unique().to_list() + assert len(unique_months) > 1, "Should have data from multiple months" + + # Check that year is correct + assert all(year == 2024 for year in df_all["tracker_year"].unique().to_list()) + + # Check that patient_id column exists + assert "patient_id" in df_all.columns + + # Check that we filtered out invalid rows (no null patient_ids) + assert df_all["patient_id"].null_count() == 0 + + # Check for baseline HbA1c column from Patient List (should be present after join) + # Note: This may have .static suffix if there were conflicts + hba1c_cols = [col for col in df_all.columns if "hba1c_baseline" in col.lower()] + print(f"\nHbA1c baseline columns: {hba1c_cols}") + + print( + f"\n2024 Tracker: {len(df_all)} total patients from {len(unique_months)} months" + f" (with Patient List & Annual data) ✓" + ) + + +@pytest.mark.skipif(not TRACKER_PNG_2019.exists(), reason="Tracker file not available") +def test_read_all_patient_sheets_2019(): + """Test reading all patient sheets from 2019 tracker (different formats across months).""" + df_all = read_all_patient_sheets(TRACKER_PNG_2019) + + # Check that we have data + assert len(df_all) > 0, "Should have extracted patient data" + + # Check metadata columns + assert "sheet_name" in df_all.columns + assert "tracker_month" in df_all.columns + assert "tracker_year" in df_all.columns + + # Check that year is correct + assert all(year == 2019 for year in df_all["tracker_year"].unique().to_list()) + + # Check that patient_id column exists + assert "patient_id" in df_all.columns + + # Check that we filtered out invalid rows + assert df_all["patient_id"].null_count() == 0 + + # 2019 tracker has format changes across months - verify we handled them + unique_months = df_all["tracker_month"].unique().to_list() + print(f"\n2019 Tracker: {len(df_all)} total patients from {len(unique_months)} months ✓") + + +@pytest.mark.skipif(not TRACKER_SBU_2024.exists(), reason="Tracker file not available") +def test_read_all_patient_sheets_file_name(): + """Test that file_name metadata is correctly added.""" + df_all = read_all_patient_sheets(TRACKER_SBU_2024) + + assert "file_name" in df_all.columns + file_names = df_all["file_name"].unique().to_list() + assert len(file_names) == 1 + assert file_names[0] == TRACKER_SBU_2024.stem + + +@pytest.mark.skipif(not TRACKER_MHS_2017.exists(), reason="Tracker file not available") +def test_read_all_patient_sheets_2017_mhs_complete(): + """ + End-to-end test: 2017 Mahosot Hospital tracker (Laos/MHS). + + Characteristics: + - Year: 2017 + - Sheets: Jan17-Dec17 (March is MISSING) + - NO Patient List or Annual sheets + - clinic_id should be "MHS" + + Expected patient counts per month: + - Jan17: 6, Feb17: 6, Apr17: 6, May17: 8, Jun17: 11, Jul17: 11 + - Aug17: 11, Sep17: 12, Oct17: 12, Nov17: 12, Dec17: 14 + - Total: 109 patients (11 months) + """ + df_all = read_all_patient_sheets(TRACKER_MHS_2017) + + # Basic validation + assert len(df_all) > 0, "Should have extracted patient data" + assert "patient_id" in df_all.columns + assert "tracker_month" in df_all.columns + assert "tracker_year" in df_all.columns + assert "clinic_id" in df_all.columns + + # Check clinic_id + assert df_all["clinic_id"].unique().to_list() == ["MHS"] + + # Check year + assert df_all["tracker_year"].unique().to_list() == [2017] + + # Check we have exactly 11 months (March is missing) + unique_months = sorted(df_all["tracker_month"].unique().to_list()) + expected_months = [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12] # Missing 3 (March) + assert unique_months == expected_months, f"Expected {expected_months}, got {unique_months}" + + # Verify patient counts per month + import calendar + + expected_counts = { + 1: 6, # Jan + 2: 6, # Feb + # 3 is missing (March) + 4: 6, # Apr + 5: 8, # May + 6: 11, # Jun + 7: 11, # Jul + 8: 11, # Aug + 9: 12, # Sep + 10: 12, # Oct + 11: 12, # Nov + 12: 14, # Dec + } + + for month, expected_count in expected_counts.items(): + month_data = df_all.filter(pl.col("tracker_month") == month) + actual_count = len(month_data) + assert actual_count == expected_count, ( + f"Month {month} ({calendar.month_abbr[month]}17): " + f"expected {expected_count} patients, got {actual_count}" + ) + + # Total patient count + total_expected = sum(expected_counts.values()) # 109 + assert len(df_all) == total_expected, ( + f"Total patients: expected {total_expected}, got {len(df_all)}" + ) + + print( + f"\n✓ 2017 MHS Tracker: {len(df_all)} patients from 11 months (March missing as expected)" + ) + + +@pytest.mark.skipif(not TRACKER_MHS_2025.exists(), reason="Tracker file not available") +def test_read_all_patient_sheets_2025_mhs_with_patient_list(): + """ + End-to-end test: 2025 Mahosot Hospital tracker (Laos/MHS). + + Characteristics: + - Year: 2025 + - Sheets: Jan25-Jun25 (6 months) + - HAS Patient List and Annual sheets + - clinic_id should be "MHS" + + Expected patient counts per month: + - Jan25: 95, Feb25: 97, Mar25: 97, Apr25: 97, May25: 98, Jun25: 99 + - Total: 583 patients + """ + df_all = read_all_patient_sheets(TRACKER_MHS_2025) + + # Basic validation + assert len(df_all) > 0, "Should have extracted patient data" + assert "patient_id" in df_all.columns + assert "tracker_month" in df_all.columns + assert "tracker_year" in df_all.columns + assert "clinic_id" in df_all.columns + + # Check clinic_id + assert df_all["clinic_id"].unique().to_list() == ["MHS"] + + # Check year + assert df_all["tracker_year"].unique().to_list() == [2025] + + # Check we have exactly 6 months (Jan-Jun) + unique_months = sorted(df_all["tracker_month"].unique().to_list()) + expected_months = [1, 2, 3, 4, 5, 6] + assert unique_months == expected_months, f"Expected {expected_months}, got {unique_months}" + + # Verify patient counts per month + import calendar + + expected_counts = { + 1: 95, # Jan + 2: 97, # Feb + 3: 97, # Mar + 4: 97, # Apr + 5: 98, # May + 6: 99, # Jun + } + + for month, expected_count in expected_counts.items(): + month_data = df_all.filter(pl.col("tracker_month") == month) + actual_count = len(month_data) + assert actual_count == expected_count, ( + f"Month {month} ({calendar.month_abbr[month]}25): " + f"expected {expected_count} patients, got {actual_count}" + ) + + # Total patient count + total_expected = sum(expected_counts.values()) # 583 + assert len(df_all) == total_expected, ( + f"Total patients: expected {total_expected}, got {len(df_all)}" + ) + + # Check that Patient List data was joined (should have columns from Patient List) + # Note: The exact columns depend on what's in the Patient List sheet + # We verify by checking for potential .static suffix columns + static_cols = [col for col in df_all.columns if ".static" in col] + print(f"\nColumns from Patient List (.static suffix): {len(static_cols)}") + + # Check that Annual data was joined + annual_cols = [col for col in df_all.columns if ".annual" in col] + print(f"Columns from Annual sheet (.annual suffix): {len(annual_cols)}") + + print( + f"\n✓ 2025 MHS Tracker: {len(df_all)} patients from 6 months " + f"(with Patient List & Annual data joined)" + ) + + +def test_export_patient_raw(tmp_path): + """Test exporting patient data to parquet file.""" + from a4d.extract.patient import export_patient_raw, read_all_patient_sheets + + # Use the 2024 SBU tracker as test data + tracker_file = TRACKER_SBU_2024 + if not tracker_file.exists(): + pytest.skip("Tracker file not available") + + # Extract data + df = read_all_patient_sheets(tracker_file) + + # Export to temp directory + output_dir = tmp_path / "patient_data_raw" + output_path = export_patient_raw(df, tracker_file, output_dir) + + # Verify output file exists + assert output_path.exists() + assert output_path.name == "2024_Sibu Hospital A4D Tracker_patient_raw.parquet" + assert output_path.parent == output_dir + + # Verify we can read it back + df_read = pl.read_parquet(output_path) + assert len(df_read) == len(df) + assert df_read.columns == df.columns + + # Verify content matches + assert df_read.equals(df) + + print(f"\n✓ Successfully exported and verified {len(df)} rows to parquet") + + +def _tracker_with_broken_id_formula(tmp_path: Path) -> Path: + """A tracker whose second patient's ID cell holds an Excel #REF! error.""" + import openpyxl + + clinic_dir = tmp_path / "TST" + clinic_dir.mkdir() + tracker_path = clinic_dir / "2024_Test_Clinic.xlsx" + + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "Jan24" + ws.cell(2, 2).value = "Patient ID" + ws.cell(2, 3).value = "Name" + ws.cell(2, 4).value = "Age" + + ws.cell(3, 1).value = 1 + ws.cell(3, 2).value = "TS_TC001" + ws.cell(3, 3).value = "TS_TC001" + ws.cell(3, 4).value = 14 + + ws.cell(4, 1).value = 2 + ws.cell(4, 2).value = "#REF!" + ws.cell(4, 3).value = "#REF!" + ws.cell(4, 4).value = 15 + + wb.save(tracker_path) + return tracker_path + + +def test_excel_error_patient_id_row_is_dropped_and_recorded(tmp_path): + """A row whose ID is a broken formula is dropped, but never silently. + + #REF! is not a malformed identifier a clinic could reconcile -- the cell's + content is gone -- so the row cannot be kept under the Undefined sentinel + the way ticket 47 keeps a misspelled ID: grouping by patient_id would pool + it with every other unidentified patient. It is dropped, and the discard is + reported so the source workbook can be corrected. + """ + from a4d.errors import ErrorCollector + + tracker_path = _tracker_with_broken_id_formula(tmp_path) + collector = ErrorCollector() + + df = read_all_patient_sheets(tracker_path, error_collector=collector) + + assert df["patient_id"].to_list() == ["TS_TC001"] + + errors = collector.to_dataframe() + dropped = errors.filter(pl.col("error_code") == "excel_error_patient_id") + assert len(dropped) == 1 + assert dropped["original_value"][0] == "#REF!" + assert "Jan24" in dropped["error_message"][0] diff --git a/tests/test_extract/test_patient_helpers.py b/tests/test_extract/test_patient_helpers.py new file mode 100644 index 0000000..aeeb300 --- /dev/null +++ b/tests/test_extract/test_patient_helpers.py @@ -0,0 +1,1052 @@ +"""Unit tests for patient extraction helper functions.""" + +import datetime +import random +from unittest.mock import Mock + +import polars as pl +import pytest +from openpyxl import Workbook + +from a4d.extract.patient import ( + filter_valid_columns, + find_data_start_row, + find_dropped_data_columns, + find_layout_changes, + join_static_sheet, + merge_headers, + read_header_rows, + read_patient_rows, + recover_blank_headers, +) + + +def create_mock_mapper(known_columns: set[str]): + """Create a mock ColumnMapper that validates specific column names.""" + mapper = Mock() + mapper.is_known_column = lambda col: col in known_columns + return mapper + + +class TestFindDataStartRow: + """Tests for find_data_start_row() function.""" + + def test_data_starts_at_row_1(self): + """Test when data starts at the very first row.""" + wb = Workbook() + ws = wb.active + ws["A1"] = 1 + ws["A2"] = 2 + + result = find_data_start_row(ws) + assert result == 1 + + wb.close() + + def test_data_starts_after_empty_rows(self): + """Test when there are empty rows before data.""" + wb = Workbook() + ws = wb.active + # Leave rows 1-10 empty + ws["A11"] = 1 + ws["A12"] = 2 + + result = find_data_start_row(ws) + assert result == 11 + + wb.close() + + def test_realistic_tracker_layout(self): + """Test with realistic tracker layout (headers at rows 75-76, data at 77).""" + wb = Workbook() + ws = wb.active + + # Simulate typical tracker: empty rows, then title rows, then headers, then data + # Title area NOT in column A (column A stays empty until headers) + ws["B1"] = "Hospital Name" + ws["C1"] = "General Hospital" + + # Headers at rows 75-76 (typical for real trackers) + ws["B75"] = "Patient" + ws["B76"] = "ID*" + + # Data starts at row 77 + ws["A77"] = 1 + ws["A78"] = 2 + + result = find_data_start_row(ws) + assert result == 77 # First non-None in column A + + wb.close() + + def test_randomized_data_position(self): + """Test with randomized data start position.""" + wb = Workbook() + ws = wb.active + + # Random start position between 10 and 100 + random_start = random.randint(10, 100) + + # Insert first data value at random position (must be numeric) + ws[f"A{random_start}"] = 1 + + result = find_data_start_row(ws) + assert result == random_start + + wb.close() + + def test_column_a_empty_raises_error(self): + """Test that ValueError is raised when column A is empty.""" + wb = Workbook() + ws = wb.active + + # Put data in other columns but not A + ws["B1"] = "Some data" + ws["C5"] = "More data" + + with pytest.raises(ValueError, match="No patient data found in column A"): + find_data_start_row(ws) + + wb.close() + + def test_ignores_none_values(self): + """Test that None/empty cells are skipped correctly.""" + wb = Workbook() + ws = wb.active + + # Explicitly set some cells to None (they start as None anyway) + ws["A1"] = None + ws["A2"] = None + ws["A3"] = None + ws["A4"] = 1 # First numeric data + + result = find_data_start_row(ws) + assert result == 4 + + wb.close() + + def test_blank_string_row_number_directly_above_data_is_data(self): + """A cleared row-number cell must not push the start past its own data row. + + 2022 Children's Hospital 2, Oct22: the first patient's row number was + deleted, leaving a whitespace-only string. Starting one row later reads + that patient's row as a header row, which costs the whole sheet. + """ + wb = Workbook() + ws = wb.active + + ws["B68"] = "Patient ID*" + ws["A70"] = " " # row number cleared, patient data still present + ws["B70"] = "VN_CH001" + ws["A71"] = 2 + ws["B71"] = "VN_CH002" + + result = find_data_start_row(ws) + assert result == 70 + + wb.close() + + def test_blank_string_far_above_data_is_not_data(self): + """A whitespace-only cell separated from the data block stays skipped. + + NOGH 2026 and Phattalung 2021 carry one ~20 rows above the patients; + it is layout residue, not a row number, and the header rows sit between. + """ + wb = Workbook() + ws = wb.active + + ws["A29"] = "" # residue well above the data block + ws["A48"] = 1 + ws["A49"] = 2 + + result = find_data_start_row(ws) + assert result == 48 + + wb.close() + + def test_leading_text_in_column_a_is_not_data(self): + """A stray word in column A row 1 must not be read as the data start. + + 2025/2026 VNCH and 2026 Gensan hold a bare 'm'/'f'/'n' in A1, which is + why the start cannot simply be the first non-empty cell in column A -- + measured across all 254 trackers, that rule would start 14 sheets here. + """ + wb = Workbook() + ws = wb.active + + ws["A1"] = "m" + ws["A84"] = 1 + + result = find_data_start_row(ws) + assert result == 84 + + wb.close() + + def test_scans_read_only_worksheet_in_one_pass(self, tmp_path): + """Must not re-parse the sheet's XML on every row (O(n^2) on read-only sheets). + + ReadOnlyWorksheet.cell() re-parses from row 1 on every call, so a + per-row .cell() loop is quadratic in the row count before data starts. + Asserts _cells_by_row (the underlying XML scan) runs at most once, + which only a single iter_rows() pass over column A can achieve. + """ + from unittest.mock import patch + + from openpyxl import Workbook, load_workbook + from openpyxl.worksheet._read_only import ReadOnlyWorksheet + + path = tmp_path / "dense.xlsx" + wb = Workbook() + ws = wb.active + for row in range(1, 200): + for col in range(1, 40): + ws.cell(row, col, value=f"text{row}_{col}") + ws["A200"] = 1 + wb.save(path) + wb.close() + + wb2 = load_workbook(path, read_only=True) + ws2 = wb2.active + + with patch.object(ReadOnlyWorksheet, "_cells_by_row", wraps=ws2._cells_by_row) as spy: + result = find_data_start_row(ws2) + + assert result == 200 + assert spy.call_count == 1 + + wb2.close() + + +class TestReadPatientRows: + """Tests for read_patient_rows() row-acceptance rules.""" + + @staticmethod + def _sheet(rows): + wb = Workbook() + ws = wb.active + for r, values in enumerate(rows, start=1): + for c, value in enumerate(values, start=1): + if value is not None: + ws.cell(row=r, column=c, value=value) + return wb, ws + + def test_keeps_a_numbered_row(self): + wb, ws = self._sheet([[1, "VN_VC001", "VN_VC001", "Y"]]) + + assert len(read_patient_rows(ws, 1, 4)) == 1 + + wb.close() + + def test_keeps_an_unnumbered_row_that_carries_data(self): + """2024_Mahosot Jun24 row 380: a real patient record whose row number + was never filled in. Bounding the data block by the row-number column + alone would silently discard it, which is why an unnumbered row that + carries actual data is kept.""" + wb, ws = self._sheet([[None, "LA-MH088", "LA-MH088", "Y", "2024-06-19"]]) + + assert len(read_patient_rows(ws, 1, 5)) == 1 + + wb.close() + + def test_drops_an_unnumbered_row_that_only_repeats_its_own_identifier(self): + """2024_Vietnam National Children Jul24 rows 165-188: a bare list of + patient IDs left below the data block, two non-empty cells per row and + nothing else. Treating those as monthly records invented 24 patients.""" + wb, ws = self._sheet([[None, "VN_VC052", "VN_VC052", None, None]]) + + assert read_patient_rows(ws, 1, 5) == [] + + wb.close() + + def test_stops_at_the_first_completely_empty_row(self): + wb, ws = self._sheet( + [[1, "VN_VC001", "VN_VC001"], [None, None, None], [2, "VN_VC002", "VN_VC002"]] + ) + + assert len(read_patient_rows(ws, 1, 3)) == 1 + + wb.close() + + def test_skips_a_row_with_neither_number_nor_identifier(self): + wb, ws = self._sheet([[None, None, "stray note"], [1, "VN_VC001", "VN_VC001"]]) + + rows = read_patient_rows(ws, 1, 3) + + assert [r[1] for r in rows] == ["VN_VC001"] + + wb.close() + + def test_recovers_a_number_typed_into_a_date_formatted_cell(self): + """2025 Hat Yai Annual!H, TH_HY035: the systolic cell carries a + dd-mmm-yyyy format, so Excel stored the typed 120 as 29-Apr-1900 and + openpyxl hands back that datetime. The datum is the number.""" + wb, ws = self._sheet([[1, "TH_HY035", "TH_HY035", None]]) + ws.cell(row=1, column=4, value=datetime.datetime(1900, 4, 29)) + + rows = read_patient_rows(ws, 1, 4) + + assert rows[0][3] == 120 + + wb.close() + + def test_recovers_a_bare_year_typed_into_a_date_formatted_cell(self): + """2019 Yangon Children's Patient List!F, MM_YC005: the D.O.B. cell + holds the bare year 2003 under a date format, so openpyxl resolves + serial 2003 to 25-Jun-1905 and the year is lost before cleaning can + read it (ticket 52). Its 2017, 2018 and 2020 siblings leave the same + cell General-formatted, so only this one file was affected.""" + wb, ws = self._sheet([[1, "MM_YC005", "MM_YC005", None]]) + ws.cell(row=1, column=4, value=datetime.datetime(1905, 6, 25)) + + rows = read_patient_rows(ws, 1, 4) + + assert rows[0][3] == 2003 + + wb.close() + + def test_keeps_a_date_that_could_plausibly_have_been_typed(self): + """The inverse case, already settled for product as + openpyxl_date_typed_stray_cell: a real date landing in the wrong + column stays a date.""" + wb, ws = self._sheet([[1, "TH_CP005", "TH_CP005", None]]) + ws.cell(row=1, column=4, value=datetime.datetime(1956, 8, 1)) + + rows = read_patient_rows(ws, 1, 4) + + assert rows[0][3] == datetime.datetime(1956, 8, 1) + + wb.close() + + +class TestReadHeaderRows: + """Tests for read_header_rows() function.""" + + def test_basic_two_row_headers(self): + """Test reading basic two-row headers.""" + wb = Workbook() + ws = wb.active + + # Data starts at row 5, so headers are at rows 3 and 4 + ws["A3"] = "Patient" + ws["B3"] = "Date" + ws["C3"] = "HbA1c" + + ws["A4"] = "ID*" + ws["B4"] = "(dd-mmm-yyyy)" + ws["C4"] = "%" + + ws["A5"] = "P001" # Data starts here + + header_1, header_2 = read_header_rows(ws, data_start_row=5) + + assert header_1 == ["ID*", "(dd-mmm-yyyy)", "%"] + assert header_2 == ["Patient", "Date", "HbA1c"] + + wb.close() + + def test_trims_to_last_non_none_column(self): + """Test that headers are trimmed to last non-None column.""" + wb = Workbook() + ws = wb.active + + # Data starts at row 10 + ws["A8"] = "Patient" + ws["B8"] = "Name" + ws["C8"] = "Age" + # D8-Z8 remain None + + ws["A9"] = "ID*" + ws["B9"] = None + ws["C9"] = None + + ws["A10"] = "P001" + + header_1, header_2 = read_header_rows(ws, data_start_row=10) + + # Should trim to column C (last non-None) + assert len(header_1) == 3 + assert len(header_2) == 3 + assert header_1 == ["ID*", None, None] + assert header_2 == ["Patient", "Name", "Age"] + + wb.close() + + def test_realistic_tracker_width(self): + """Test with realistic tracker dimensions (31 columns).""" + wb = Workbook() + ws = wb.active + + data_start_row = 77 + + # Create 31 columns of headers + for col_idx in range(1, 32): # 1 to 31 inclusive + ws.cell(row=75, column=col_idx, value=f"H2_Col{col_idx}") + ws.cell(row=76, column=col_idx, value=f"H1_Col{col_idx}") + + # Put data at row 77 + ws.cell(row=77, column=1, value="P001") + + header_1, header_2 = read_header_rows(ws, data_start_row=data_start_row) + + assert len(header_1) == 31 + assert len(header_2) == 31 + assert header_1[0] == "H1_Col1" + assert header_1[30] == "H1_Col31" + assert header_2[0] == "H2_Col1" + assert header_2[30] == "H2_Col31" + + wb.close() + + def test_mixed_none_values_in_headers(self): + """Test headers with mixed None and non-None values.""" + wb = Workbook() + ws = wb.active + + # Header row 2 (further from data) + ws["A3"] = "Patient" + ws["B3"] = None + ws["C3"] = "Updated HbA1c" + ws["D3"] = None # Horizontally merged + ws["E3"] = None + + # Header row 1 (closer to data) + ws["A4"] = "ID*" + ws["B4"] = "Name" + ws["C4"] = "%" + ws["D4"] = "(dd-mmm-yyyy)" + ws["E4"] = None + + ws["A5"] = "P001" # Data + + header_1, header_2 = read_header_rows(ws, data_start_row=5) + + # Should trim to column D (last non-None in header_1) + assert len(header_1) == 4 + assert len(header_2) == 4 + assert header_1 == ["ID*", "Name", "%", "(dd-mmm-yyyy)"] + assert header_2 == ["Patient", None, "Updated HbA1c", None] + + wb.close() + + def test_randomized_header_position(self): + """Test with randomized data start position.""" + wb = Workbook() + ws = wb.active + + # Random data start between rows 20 and 100 + random_data_start = random.randint(20, 100) + header_row_1 = random_data_start - 1 + header_row_2 = random_data_start - 2 + + # Set headers + ws.cell(row=header_row_2, column=1, value="Header2") + ws.cell(row=header_row_1, column=1, value="Header1") + ws.cell(row=random_data_start, column=1, value="Data") + + header_1, header_2 = read_header_rows(ws, data_start_row=random_data_start) + + assert header_1 == ["Header1"] + assert header_2 == ["Header2"] + + wb.close() + + def test_respects_max_cols_parameter(self): + """Test that max_cols parameter limits the read width.""" + wb = Workbook() + ws = wb.active + + # Create 200 columns of data + for col_idx in range(1, 201): + ws.cell(row=3, column=col_idx, value=f"H2_{col_idx}") + ws.cell(row=4, column=col_idx, value=f"H1_{col_idx}") + + ws["A5"] = "Data" + + # Read with max_cols=50 + header_1, header_2 = read_header_rows(ws, data_start_row=5, max_cols=50) + + # Should only read up to column 50 + assert len(header_1) == 50 + assert len(header_2) == 50 + assert header_1[49] == "H1_50" + + wb.close() + + def test_all_none_headers(self): + """Test when both header rows are completely None. + + Note: When no non-None values are found, the function returns + max_cols None values (default behavior). In practice, this edge + case doesn't occur as real trackers always have headers. + """ + wb = Workbook() + ws = wb.active + + # Headers are all None + # (openpyxl cells are None by default) + + ws["A5"] = "Data" + + header_1, header_2 = read_header_rows(ws, data_start_row=5, max_cols=10) + + # Returns max_cols None values when nothing is found + assert len(header_1) == 10 + assert len(header_2) == 10 + assert all(h is None for h in header_1) + assert all(h is None for h in header_2) + + wb.close() + + +class TestMergeHeaders: + """Tests for merge_headers() function.""" + + def test_both_headers_present(self): + """Test merging when both header rows have values.""" + h1 = ["%", "mmol/L", "kg"] + h2 = ["HbA1c", "FBG", "Weight"] + result = merge_headers(h1, h2) + assert result == ["HbA1c %", "FBG mmol/L", "Weight kg"] + + def test_only_h2_present(self): + """Test when only header row 2 has values.""" + h1 = [None, None, None] + h2 = ["Patient ID", "Name", "Age"] + result = merge_headers(h1, h2) + assert result == ["Patient ID", "Name", "Age"] + + def test_only_h1_present(self): + """Test when only header row 1 has values (single-line headers).""" + h1 = ["Patient ID", "Name", "Age"] + h2 = [None, None, None] + result = merge_headers(h1, h2) + assert result == ["Patient ID", "Name", "Age"] + + def test_horizontal_merge_forward_fill(self): + """Test forward-fill with synonym validation. + + Forward-fill happens when mapper validates the combined header. + """ + h1 = ["%", "(dd-mmm-yyyy)", "mmol/L", "(dd-mmm-yyyy)"] + h2 = ["Updated HbA1c", None, "Updated FBG", None] + # Mock mapper that knows these forward-filled patterns + mapper = create_mock_mapper( + { + "Updated HbA1c %", + "Updated HbA1c (dd-mmm-yyyy)", + "Updated FBG mmol/L", + "Updated FBG (dd-mmm-yyyy)", + } + ) + result = merge_headers(h1, h2, mapper) + assert result == [ + "Updated HbA1c %", + "Updated HbA1c (dd-mmm-yyyy)", + "Updated FBG mmol/L", + "Updated FBG (dd-mmm-yyyy)", + ] + + def test_mixed_headers(self): + """Test realistic mix of header patterns. + + Forward-fill happens when mapper validates the combined header. + """ + h1 = ["ID*", "Name", "%", "(date)", None, "kg"] + h2 = ["Patient", None, "HbA1c", None, "Notes", "Weight"] + # Mock mapper that validates these forward-fills + mapper = create_mock_mapper( + { + "Patient ID*", + "Patient Name", + "HbA1c %", + "HbA1c (date)", + } + ) + result = merge_headers(h1, h2, mapper) + assert result == [ + "Patient ID*", + "Patient Name", # Forward-filled and validated + "HbA1c %", + "HbA1c (date)", # Forward-filled and validated + "Notes", + "Weight kg", + ] + + def test_none_values_reset_forward_fill(self): + """Test that None in both headers results in None. + + Forward-fill only happens when h1 exists and mapper validates. + """ + h1 = ["%", "(date)", None, "kg"] + h2 = ["HbA1c", None, None, "Weight"] + # Mock mapper that validates HbA1c forward-fills + mapper = create_mock_mapper( + { + "HbA1c %", + "HbA1c (date)", + } + ) + result = merge_headers(h1, h2, mapper) + assert result == [ + "HbA1c %", + "HbA1c (date)", + None, + "Weight kg", + ] + + def test_whitespace_normalization(self): + """Test that extra whitespace and newlines are normalized.""" + h1 = ["ID\n(format)", " Name "] + h2 = ["Patient\nID", "Full Name"] + result = merge_headers(h1, h2) + assert result == [ + "Patient ID ID (format)", + "Full Name Name", + ] + + def test_updated_year_marker_inherits_previous_subject(self): + """The 2022 template labels an update-date column "Updated 2022". + + The subject it belongs to is only in the column to its left, so + "Updated " alone must not become the column name. Without this, + blood_pressure_updated and edu_occ_updated go unmapped on every 2022 + tracker -- 7,165 values. + """ + h1 = ["mm HG", "Date"] + h2 = ["Blood Pressure ", "Updated\n2022"] + result = merge_headers(h1, h2) + assert result == ["Blood Pressure mm HG", "Blood Pressure Date"] + + def test_updated_year_marker_on_patient_list(self): + """Same marker, different subject: the Patient List's education column.""" + h1 = [None, "Date"] + h2 = ["Level of Education\nOr Occupation", " Updated \n2022"] + result = merge_headers(h1, h2) + assert result == [ + "Level of Education Or Occupation", + "Level of Education Or Occupation Date", + ] + + def test_updated_year_marker_without_previous_subject(self): + """With nothing to inherit, the marker is left alone rather than dropped.""" + h1 = ["Date"] + h2 = ["Updated 2022"] + result = merge_headers(h1, h2) + assert result == ["Updated 2022 Date"] + + def test_updated_without_year_is_not_a_marker(self): + """ "Updated HbA1c" is a real subject, not a continuation marker.""" + h1 = ["%"] + h2 = ["Updated HbA1c"] + result = merge_headers(h1, h2) + assert result == ["Updated HbA1c %"] + + def test_empty_headers(self): + """Test with empty header lists.""" + result = merge_headers([], []) + assert result == [] + + def test_single_column(self): + """Test with single column.""" + h1 = ["ID"] + h2 = ["Patient"] + result = merge_headers(h1, h2) + assert result == ["Patient ID"] + + +class TestMergeHeadersWithMergedSpans: + """A merged upper header names every column its span covers. + + The 2021 Putrajaya layout: "Complication Screening (Current Month Testing)" + is merged across five columns, so the sub-headers "Results" and + "Date (mmm-yy)" further along the block have no upper header of their own. + Forward-fill alone cannot reach them -- it resets at the blank columns + between -- and the bare names map nowhere. + """ + + def test_merged_span_names_a_sub_header_beyond_a_blank_gap(self): + h1 = ["Select for Drop Down", None, None, "Results", "Date (mmm-yy)"] + h2 = ["Complication Screening", None, None, None, None] + mapper = create_mock_mapper( + {"Complication Screening Results", "Complication Screening Date (mmm-yy)"} + ) + + result = merge_headers(h1, h2, mapper=mapper, merged_spans=[(1, 5)]) + + assert result[3] == "Complication Screening Results" + assert result[4] == "Complication Screening Date (mmm-yy)" + + def test_a_column_with_no_sub_header_is_not_named_by_the_title(self): + """A second column under one merged title is not the same field. + + The 2022 template merges "Insulin Regimen" across two columns whose + second holds a near-duplicate of the first ("Basal-bolus MDI (AN/HI)" + against "Basal-bolus (AN/HI)"). Naming both would comma-join them into + one value, which is worse data than the first column alone. + """ + h1 = [None, None] + h2 = ["Insulin Regimen", None] + + result = merge_headers(h1, h2, merged_spans=[(1, 2)]) + + assert result == ["Insulin Regimen", None] + + def test_bare_title_does_not_claim_a_column_another_already_maps_to(self): + """The 2022 complication-screening block, which collided on 290 sheets. + + The block's own column already carries the screening selection; a blank + column inside the same merge would take the bare title, which maps to + the same standard column and silently competes with it. + """ + h1 = ["Drop Down", None, "Results"] + h2 = ["Current Month Complication Screening", None, None] + mapper = create_mock_mapper( + { + "Current Month Complication Screening Drop Down", + "Current Month Complication Screening Results", + } + ) + + result = merge_headers(h1, h2, mapper=mapper, merged_spans=[(1, 3)]) + + assert result == [ + "Current Month Complication Screening Drop Down", + None, + "Current Month Complication Screening Results", + ] + + def test_two_sub_headers_qualifying_alike_do_not_merge(self): + h1 = ["Date", None, "Date"] + h2 = ["Screening", None, None] + mapper = create_mock_mapper({"Screening Date"}) + + result = merge_headers(h1, h2, mapper=mapper, merged_spans=[(1, 3)]) + + assert result == ["Screening Date", None, "Date"] + + def test_without_spans_behaviour_is_unchanged(self): + h1 = ["Select for Drop Down", None, "Results"] + h2 = ["Complication Screening", None, None] + + assert merge_headers(h1, h2) == [ + "Complication Screening Select for Drop Down", + None, + "Results", + ] + + +class TestRecoverBlankHeadersInsideMergedSpan: + """A merged header outranks a sibling sheet's guess. + + Putrajaya's month sheets do not share one layout: Jul21 has "Patient + Observations" at the position Dec21 uses for a complication-screening + selection. Recovering by position files a screening result under + observations. The merged span is the workbook's own statement about what + the column belongs to, so it wins. + """ + + def test_abstains_for_a_position_inside_a_merged_span(self): + headers = ["ID", "Complication Screening", None] + data = [("1", "Kidneys", "Foot Examination (Nerves)")] + siblings = [["ID", "Complication Screening", "Patient Observations"]] + + result = recover_blank_headers(headers, data, siblings, merged_spans=[(2, 3)]) + + assert result == ["ID", "Complication Screening", None] + + def test_recovery_outside_any_span_still_works(self): + """2021 Kantha Bopha's insulin-regimen recovery must survive.""" + headers = ["ID", None] + data = [("1", "Self-mixed BD")] + siblings = [["ID", "Insulin Regime"]] + + result = recover_blank_headers(headers, data, siblings, merged_spans=[(5, 7)]) + + assert result == ["ID", "Insulin Regime"] + + +class TestFilterValidColumns: + """Tests for filter_valid_columns() function.""" + + def test_all_valid_headers(self): + """Test when all headers are valid (no None).""" + headers = ["ID", "Name", "Age"] + data = [("1", "Alice", "30"), ("2", "Bob", "25")] + valid_headers, filtered_data = filter_valid_columns(headers, data) + + assert valid_headers == ["ID", "Name", "Age"] + assert filtered_data == [["1", "Alice", "30"], ["2", "Bob", "25"]] + + def test_some_none_headers(self): + """Test filtering out None headers.""" + headers = ["ID", None, "Name", None, "Age"] + data = [("1", "x", "Alice", "y", "30"), ("2", "x", "Bob", "y", "25")] + valid_headers, filtered_data = filter_valid_columns(headers, data) + + assert valid_headers == ["ID", "Name", "Age"] + assert filtered_data == [["1", "Alice", "30"], ["2", "Bob", "25"]] + + def test_all_none_headers(self): + """Test when all headers are None.""" + headers = [None, None, None] + data = [("1", "2", "3"), ("4", "5", "6")] + valid_headers, filtered_data = filter_valid_columns(headers, data) + + assert valid_headers == [] + assert filtered_data == [] + + def test_empty_data(self): + """Test with empty data.""" + headers = ["ID", "Name"] + data = [] + valid_headers, filtered_data = filter_valid_columns(headers, data) + + assert valid_headers == ["ID", "Name"] + assert filtered_data == [] + + def test_single_valid_column(self): + """Test with single valid column.""" + headers = [None, "ID", None] + data = [("x", "1", "y"), ("x", "2", "y")] + valid_headers, filtered_data = filter_valid_columns(headers, data) + + assert valid_headers == ["ID"] + assert filtered_data == [["1"], ["2"]] + + def test_preserves_order(self): + """Test that column order is preserved.""" + headers = ["A", None, "B", None, "C", "D", None] + data = [(1, 2, 3, 4, 5, 6, 7)] + valid_headers, filtered_data = filter_valid_columns(headers, data) + + assert valid_headers == ["A", "B", "C", "D"] + assert filtered_data == [[1, 3, 5, 6]] + + +class TestFindDroppedDataColumns: + """Tests for find_dropped_data_columns() function.""" + + def test_reports_headerless_column_holding_text(self): + headers = ["ID", None, "Age"] + data = [("1", "Self-mixed BD", "30"), ("2", "Basal-Bolus", "25")] + + assert find_dropped_data_columns(headers, data) == [(1, 2)] + + def test_ignores_headerless_column_that_is_empty(self): + headers = ["ID", None, "Age"] + data = [("1", None, "30"), ("2", "", "25")] + + assert find_dropped_data_columns(headers, data) == [] + + def test_ignores_row_counter(self): + """A headerless all-numeric column is the tracker's own row counter.""" + headers = [None, "ID"] + data = [("1", "KH_PK001"), ("2", "KH_PK002"), ("3.0", "KH_PK003")] + + assert find_dropped_data_columns(headers, data) == [] + + def test_reports_mixed_numeric_and_text_column(self): + """A stray number among real text does not make the column a counter.""" + headers = [None, "ID"] + data = [("7", "KH_JV001"), ("*Needs Meter", "KH_JV002")] + + assert find_dropped_data_columns(headers, data) == [(0, 2)] + + def test_ignores_columns_that_have_headers(self): + headers = ["ID", "Notes"] + data = [("1", "text")] + + assert find_dropped_data_columns(headers, data) == [] + + +class TestRecoverBlankHeaders: + """Tests for recover_blank_headers() function.""" + + def test_recovers_from_the_one_sibling_that_names_the_column(self): + headers = ["ID", None] + data = [("1", "Self-mixed BD")] + siblings = [["ID", "Insulin Regime"]] + + assert recover_blank_headers(headers, data, siblings) == ["ID", "Insulin Regime"] + + def test_abstains_when_siblings_disagree(self): + """Two candidate names is a guess, not a recovery.""" + headers = ["ID", None] + data = [("1", "x")] + siblings = [["ID", "Complication Screening"], ["ID", "Hospitalisation Date"]] + + assert recover_blank_headers(headers, data, siblings) == ["ID", None] + + def test_agreeing_siblings_are_not_a_disagreement(self): + headers = ["ID", None] + data = [("1", "x")] + siblings = [["ID", "Clinic Visit"], ["ID", "Clinic Visit"], ["ID", None]] + + assert recover_blank_headers(headers, data, siblings) == ["ID", "Clinic Visit"] + + def test_abstains_when_no_sibling_names_the_column(self): + """The 2022 template's hidden merged-cell column is blank in every sheet.""" + headers = ["ID", None] + data = [("1", "Self-mixed BD")] + siblings = [["ID", None], ["ID", None]] + + assert recover_blank_headers(headers, data, siblings) == ["ID", None] + + def test_leaves_an_empty_column_alone(self): + """A blank header over no data is a spacer, not a lost column.""" + headers = ["ID", None] + data = [("1", None)] + siblings = [["ID", "Insulin Regime"]] + + assert recover_blank_headers(headers, data, siblings) == ["ID", None] + + def test_ignores_the_row_counter(self): + headers = [None, "ID"] + data = [("1", "KH_PK001"), ("2", "KH_PK002")] + siblings = [["Nr", "ID"]] + + assert recover_blank_headers(headers, data, siblings) == [None, "ID"] + + def test_never_creates_a_duplicate_column_name(self): + """A donor name this sheet already uses would collide on rename.""" + headers = ["ID", "Insulin Regime", None] + data = [("1", "a", "b")] + siblings = [["ID", "Insulin Regime", "Insulin Regime"]] + + assert recover_blank_headers(headers, data, siblings) == ["ID", "Insulin Regime", None] + + def test_tolerates_siblings_of_different_width(self): + headers = ["ID", None] + data = [("1", "x")] + siblings = [["ID"], ["ID", "Clinic Visit"]] + + assert recover_blank_headers(headers, data, siblings) == ["ID", "Clinic Visit"] + + def test_no_siblings_is_a_no_op(self): + headers = ["ID", None] + data = [("1", "x")] + + assert recover_blank_headers(headers, data, []) == ["ID", None] + + +class TestFindLayoutChanges: + """Tests for find_layout_changes() function.""" + + def test_flags_a_position_whose_canonical_column_changes(self): + layouts = { + "Jan20": ["Patient ID", "Baseline FBG (mmol/dL)"], + "Jun20": ["Patient ID", "Baseline FBG (mg/dL)"], + } + mapper = create_mock_mapper(set()) + mapper.get_standard_name = lambda h: { + "Baseline FBG (mmol/dL)": "fbg_baseline_mmol", + "Baseline FBG (mg/dL)": "fbg_baseline_mg", + }.get(h, h) + + changes = find_layout_changes(layouts, mapper) + + assert len(changes) == 1 + assert changes[0].index == 1 + assert changes[0].renames_only is False + assert changes[0].headers == ["Baseline FBG (mg/dL)", "Baseline FBG (mmol/dL)"] + + def test_a_pure_rename_is_flagged_but_marked_harmless(self): + layouts = { + "Jan20": ["Patient ID", "Insulin regime"], + "Mar20": ["Patient ID", "Insulin regimen"], + } + mapper = create_mock_mapper(set()) + mapper.get_standard_name = lambda h: "insulin_regimen" if "nsulin" in h else h + + changes = find_layout_changes(layouts, mapper) + + assert len(changes) == 1 + assert changes[0].renames_only is True + + def test_sheets_that_agree_produce_nothing(self): + layouts = {"Jan20": ["Patient ID", "Age"], "Feb20": ["Patient ID", "Age"]} + mapper = create_mock_mapper(set()) + mapper.get_standard_name = lambda h: h + + assert find_layout_changes(layouts, mapper) == [] + + def test_a_blank_header_is_not_a_change(self): + """One sheet leaving a header empty is recover_blank_headers' business.""" + layouts = {"Jan20": ["Patient ID", "Age"], "Feb20": ["Patient ID", None]} + mapper = create_mock_mapper(set()) + mapper.get_standard_name = lambda h: h + + assert find_layout_changes(layouts, mapper) == [] + + def test_a_single_sheet_cannot_disagree_with_itself(self): + mapper = create_mock_mapper(set()) + mapper.get_standard_name = lambda h: h + + assert find_layout_changes({"Jan20": ["Patient ID"]}, mapper) == [] + + def test_sheets_of_different_width_are_compared_where_they_overlap(self): + layouts = {"Jan20": ["ID", "Insulin Regimen", "BASAL"], "Apr20": ["ID", "BASAL"]} + mapper = create_mock_mapper(set()) + mapper.get_standard_name = lambda h: h + + changes = find_layout_changes(layouts, mapper) + + assert [c.index for c in changes] == [1] + + +class TestJoinStaticSheet: + """The whole-tracker joins key on the normalized ID, not the raw one (ticket 58).""" + + @staticmethod + def _monthly(ids: list[str]) -> pl.DataFrame: + return pl.DataFrame({"patient_id": ids, "fbg_updated_mg": [1.0] * len(ids)}) + + @staticmethod + def _static(ids: list[str]) -> pl.DataFrame: + return pl.DataFrame( + {"patient_id": ids, "dob": [f"2010-01-0{i + 1}" for i in range(len(ids))]} + ) + + @classmethod + def _join(cls, monthly_ids: list[str], static_ids: list[str]) -> pl.DataFrame: + return join_static_sheet( + cls._monthly(monthly_ids), cls._static(static_ids), ".static", "Patient List" + ) + + def test_hyphen_spelled_monthly_id_finds_its_underscore_patient_list_entry(self): + """2023/2024 Mahosot: 672 rows lost every demographic to this mismatch.""" + assert self._join(["LA-MH056"], ["LA_MH056"])["dob"].to_list() == ["2010-01-01"] + + def test_transfer_clinic_suffix_does_not_block_the_match(self): + assert self._join(["MY_SM003_SB"], ["MY_SM003"])["dob"].to_list() == ["2010-01-01"] + + def test_raw_patient_id_column_is_not_rewritten(self): + """The comparison's row-alignment key depends on the raw spelling surviving.""" + assert self._join(["LA-MH056"], ["LA_MH056"])["patient_id"].to_list() == ["LA-MH056"] + + def test_no_join_key_column_leaks_into_the_output(self): + result = self._join(["LA-MH056"], ["LA_MH056"]) + assert result.columns == ["patient_id", "fbg_updated_mg", "dob"] + + def test_unmatched_monthly_id_keeps_its_row_with_null_demographics(self): + """MM_YG013_MG has no Patient List entry at all -- a source defect, not a key bug.""" + result = self._join(["MM_YG013_MG"], ["MM_MG013"]) + assert result.height == 1 + assert result["dob"].to_list() == [None] + + def test_a_row_never_gains_a_second_partner(self): + """Two spellings folding to one key must not duplicate the month row.""" + assert self._join(["TH_ST029"], ["TH-ST029", "TH_ST029"]).height == 1 + + def test_colliding_columns_take_the_suffix_the_caller_asked_for(self): + monthly = pl.DataFrame({"patient_id": ["LA-MH056"], "province": ["monthly"]}) + static = pl.DataFrame({"patient_id": ["LA_MH056"], "province": ["static"]}) + result = join_static_sheet(monthly, static, ".static", "Patient List") + assert result["province"].to_list() == ["monthly"] + assert result["province.static"].to_list() == ["static"] + + def test_the_annual_sheet_join_gets_the_same_key_and_its_own_suffix(self): + """2024 CDA and 2026 Surat Thani lose Annual columns to the same mismatch.""" + annual = pl.DataFrame({"patient_id": ["KH_CD016"], "edu_occ": ["Student"]}) + result = join_static_sheet(self._monthly(["KH-CD016"]), annual, ".annual", "Annual") + assert result["edu_occ"].to_list() == ["Student"] diff --git a/tests/test_extract/test_product.py b/tests/test_extract/test_product.py new file mode 100644 index 0000000..9b10232 --- /dev/null +++ b/tests/test_extract/test_product.py @@ -0,0 +1,401 @@ +"""Unit tests for product data extraction (`a4d.extract.product`).""" + +from pathlib import Path +from unittest.mock import Mock + +import polars as pl +import pytest +from openpyxl import Workbook + +from a4d.errors import ErrorCollector +from a4d.extract.product import ( + ProductSectionNotFoundError, + _count_orphan_released_units, + _harmonize, + add_product_metadata, + extract_product_data, + find_product_section, + read_all_product_sheets, + remove_header_rows, + replace_extra_totals, +) + + +def _make_ws(rows: list[list]): + """Build an in-memory openpyxl worksheet with the given rows. + + Returns the worksheet handle (1-indexed Excel rows). Empty cells use + ``None``. + """ + wb = Workbook() + ws = wb.active + for row in rows: + ws.append(row) + return ws + + +def test_find_product_section_2024_layout(): + """Header row contains product/date/received; patient section follows.""" + ws = _make_ws( + [ + [None, None, None, None, None], + ["Product", "Date", "Units Received", "From", "Released"], + ["Insulin", "2024-06-01", 100, "DKSH", 5], + ["Strips", "2024-06-02", 50, "DKSH", None], + ["Patient Recruitment", None, None, None, None], + ["Patient Name", "Patient ID", None, None, None], + ] + ) + start, end = find_product_section(ws) + assert start == 2 + assert end == 4 + + +def test_find_product_section_raises_on_missing_start(): + """No header row with product/date/received markers triggers the error.""" + ws = _make_ws( + [ + ["Random", "Junk", "Headers"], + ["Patient Name", "Patient ID", None], + ] + ) + with pytest.raises(ProductSectionNotFoundError): + find_product_section(ws) + + +def test_find_product_section_raises_on_missing_end(): + """Header found but no patient-section terminator triggers the error.""" + ws = _make_ws( + [ + ["Product", "Date", "Units Received"], + ["Insulin", "2024-06-01", 100], + ] + ) + with pytest.raises(ProductSectionNotFoundError): + find_product_section(ws) + + +def _make_mapper(known_to_standard: dict[str, str]): + """Build a mock ColumnMapper. + + ``known_to_standard`` maps source column names to the standardized name + they should be renamed to. Anything not in the dict is treated as + unknown (caught by ``is_known_column`` returning False) and dropped. + """ + mapper = Mock() + mapper.synonyms = {standard: [src] for src, standard in known_to_standard.items()} + mapper.is_known_column = lambda col: col in known_to_standard + mapper.rename_columns = lambda df: df.rename(known_to_standard) + return mapper + + +def test_harmonize_renames_known_drops_unknown(): + df = pl.DataFrame( + {"Product Name": ["Insulin"], "Random Junk": ["x"]}, + schema={"Product Name": pl.String, "Random Junk": pl.String}, + ) + mapper = _make_mapper({"Product Name": "product"}) + collector = ErrorCollector() + + out = _harmonize(df, mapper, sheet_name="Jun24", error_collector=collector, file_name="t.xlsx") + + assert out.columns == ["product"] + assert len(collector) == 1 + assert collector.errors[0].column == "Random Junk" + assert collector.errors[0].error_code == "invalid_tracker" + assert collector.errors[0].function_name == "harmonize_input_data_columns" + + +def test_harmonize_no_unknowns_no_log(): + df = pl.DataFrame( + {"Product Name": ["Insulin"]}, + schema={"Product Name": pl.String}, + ) + mapper = _make_mapper({"Product Name": "product"}) + collector = ErrorCollector() + + out = _harmonize(df, mapper, sheet_name="Jun24", error_collector=collector, file_name="t.xlsx") + + assert out.columns == ["product"] + assert len(collector) == 0 + + +def test_replace_extra_totals_masks_after_total_column(): + """If a column to the immediate left of product_units_released contains + 'total' (case-insensitive), the released value is nulled.""" + df = pl.DataFrame( + { + "product": ["A", "B"], + "Total Released": ["Total", None], + "product_units_released": ["10", "20"], + }, + schema={ + "product": pl.String, + "Total Released": pl.String, + "product_units_released": pl.String, + }, + ) + out = replace_extra_totals(df) + assert out["product_units_released"].to_list() == [None, "20"] + + +def test_replace_extra_totals_preserves_clean_rows(): + """No 'total' marker in either of the two preceding columns leaves + product_units_released unchanged.""" + df = pl.DataFrame( + { + "product": ["A", "B"], + "product_received_from": ["DKSH", None], + "product_released_to": ["P1", "P2"], + "product_units_released": ["10", "20"], + }, + schema={ + "product": pl.String, + "product_received_from": pl.String, + "product_released_to": pl.String, + "product_units_released": pl.String, + }, + ) + out = replace_extra_totals(df) + assert out["product_units_released"].to_list() == ["10", "20"] + + +def test_replace_extra_totals_case_insensitive(): + """'total' / 'TOTAL' / 'Total' all trigger masking.""" + df = pl.DataFrame( + { + "product": ["A"], + "TOTAL": ["TOTAL"], + "product_units_released": ["10"], + }, + schema={ + "product": pl.String, + "TOTAL": pl.String, + "product_units_released": pl.String, + }, + ) + out = replace_extra_totals(df) + assert out["product_units_released"].to_list() == [None] + + +def test_extract_product_data_promotes_headers_and_types_strings(): + """First row becomes column names; all values are coerced to pl.String.""" + rows = [ + ["Cover", None, None], # 1: pre-section + ["Product", "Date", "Units"], # 2: header + ["Insulin", "2024-06-01", 100], # 3 + ["Strips", "2024-06-02", 50], # 4 + ] + ws = _make_ws(rows) + + out = extract_product_data(ws, start_row=2, end_row=4) + + assert out.columns == ["Product", "Date", "Units"] + assert out.dtypes == [pl.String, pl.String, pl.String] + assert out["Product"].to_list() == ["Insulin", "Strips"] + assert out["Units"].to_list() == ["100", "50"] + + +def test_extract_product_data_merges_duplicate_headers_with_comma(): + """When two columns share a header, values are joined with ','.""" + rows = [ + ["Product", "Note", "Note"], + ["Insulin", "A", "B"], + ["Strips", None, "C"], + ] + ws = _make_ws(rows) + + out = extract_product_data(ws, start_row=1, end_row=3) + assert "Note" in out.columns + assert out["Note"].to_list() == ["A,B", "C"] + + +def test_extract_product_data_returns_empty_for_short_section(): + rows = [["Product", "Date"]] # only header, no data + ws = _make_ws(rows) + out = extract_product_data(ws, start_row=1, end_row=1) + assert out.height == 0 + assert out.width == 0 + + +def test_add_product_metadata_appends_five_cols(): + df = pl.DataFrame({"product": ["A"]}, schema={"product": pl.String}) + out = add_product_metadata(df, "Jun24", 6, 2024, "tracker.xlsx", "CL001") + assert out["product_table_month"].to_list() == ["06"] + assert out["product_table_year"].to_list() == [2024.0] + assert out["product_sheet_name"].to_list() == ["Jun24"] + assert out["file_name"].to_list() == ["tracker.xlsx"] + assert out["clinic_id"].to_list() == ["CL001"] + + +def test_remove_header_rows_drops_repeated_header_and_empty(): + df = pl.DataFrame( + { + "product": ["Product", "Insulin", None, "Patient Data Summary"], + "product_entry_date": [None, "2024-06", None, None], + }, + schema={"product": pl.String, "product_entry_date": pl.String}, + ) + out = remove_header_rows(df) + assert out["product"].to_list() == ["Insulin"] + + +def test_remove_header_rows_empty_input(): + df = pl.DataFrame({"product": []}, schema={"product": pl.String}) + out = remove_header_rows(df) + assert out.height == 0 + + +def test_remove_header_rows_drops_row_of_empty_strings(): + # A formula-emptied Excel cell surfaces as "" rather than None; such a row + # carries no information and must not reach the output as a movement. + df = pl.DataFrame( + { + "product": ["Insulin", ""], + "product_entry_date": ["2024-06", ""], + }, + schema={"product": pl.String, "product_entry_date": pl.String}, + ) + out = remove_header_rows(df) + assert out["product"].to_list() == ["Insulin"] + + +def test_read_all_product_sheets_end_to_end(tmp_path: Path): + """End-to-end: build a tracker workbook with one month sheet, stub the + mapper, and verify the orchestrator wires extract → harmonize → + metadata → totals correctly.""" + wb = Workbook() + # Remove the default sheet and add a Jun24 sheet. + wb.remove(wb.active) + ws = wb.create_sheet("Jun24") + + rows = [ + [None, None, None, None, None], # 1: pre-section + ["Product", "Date", "Units Received", "From", "Released"], # 2: header + ["Insulin", "2024-06-01", 100, "DKSH", 5], # 3: data + ["Strips", "2024-06-02", 50, "DKSH", None], # 4: data + ["Patient Recruitment", None, None, None, None], # 5: terminator + ["Patient Name", "Patient ID", None, None, None], # 6: end + ] + for row in rows: + ws.append(row) + + tracker_path = tmp_path / "2024_test_tracker.xlsx" + wb.save(tracker_path) + + mapper = _make_mapper( + { + "Product": "product", + "Date": "product_entry_date", + "Units Received": "product_units_received", + "From": "product_received_from", + "Released": "product_units_released", + } + ) + collector = ErrorCollector() + + out = read_all_product_sheets(tracker_path, mapper=mapper, error_collector=collector) + + assert out.height == 2 + assert "product" in out.columns + assert "product_table_year" in out.columns + assert "product_sheet_name" in out.columns + assert "clinic_id" in out.columns + assert out["product_sheet_name"].to_list() == ["Jun24", "Jun24"] + assert out["product_table_year"].to_list() == [2024.0, 2024.0] + assert out["clinic_id"].to_list() == [tracker_path.parent.name] * 2 + # No unknown columns; collector should be empty. + assert len(collector) == 0 + + +def test_read_all_product_sheets_no_month_sheets_raises(tmp_path: Path): + wb = Workbook() + # Default sheet has no month-prefix; remove and add a non-month sheet. + wb.remove(wb.active) + wb.create_sheet("Cover") + tracker_path = tmp_path / "2024_no_months.xlsx" + wb.save(tracker_path) + + mapper = _make_mapper({}) + + with pytest.raises(ValueError, match="No month sheets"): + read_all_product_sheets(tracker_path, mapper=mapper) + + +def test_count_orphan_released_units_logs_per_sheet(): + """3 orphan rows (released_to null while units_released non-null) + produce exactly 1 ErrorCollector entry naming the sheet and count.""" + df = pl.DataFrame( + { + "product_released_to": [None, None, None, "P1", "P2"], + "product_units_released": ["10", "20", "30", "40", None], + }, + schema={ + "product_released_to": pl.String, + "product_units_released": pl.String, + }, + ) + collector = ErrorCollector() + + _count_orphan_released_units( + df, sheet_name="Jul24", file_name="t.xlsx", error_collector=collector + ) + + assert len(collector) == 1 + err = collector.errors[0] + assert err.error_code == "invalid_tracker" + assert err.function_name == "read_product_data_step1" + assert err.column == "product_released_to" + assert "Jul24" in err.error_message + assert "3" in err.error_message + + +def test_count_orphan_released_units_zero_rows_no_log(): + """Clean fixture (all releases have a recipient) emits no log.""" + df = pl.DataFrame( + { + "product_released_to": ["P1", "P2", None], + "product_units_released": ["10", "20", None], + }, + schema={ + "product_released_to": pl.String, + "product_units_released": pl.String, + }, + ) + collector = ErrorCollector() + _count_orphan_released_units(df, "Jul24", "t.xlsx", collector) + assert len(collector) == 0 + + +def test_count_orphan_released_units_treats_whitespace_as_null(): + """Whitespace-only product_released_to (e.g. ' ') counts as orphan.""" + df = pl.DataFrame( + { + "product_released_to": [" ", "\t", "P1"], + "product_units_released": ["10", "20", "30"], + }, + schema={ + "product_released_to": pl.String, + "product_units_released": pl.String, + }, + ) + collector = ErrorCollector() + _count_orphan_released_units(df, "Jul24", "t.xlsx", collector) + assert len(collector) == 1 + assert "2" in collector.errors[0].error_message + + +def test_count_orphan_released_units_no_collector_is_noop(): + """When no collector is passed (e.g. preview path), the helper is a no-op.""" + df = pl.DataFrame( + { + "product_released_to": [None], + "product_units_released": ["10"], + }, + schema={ + "product_released_to": pl.String, + "product_units_released": pl.String, + }, + ) + # Just must not raise. + _count_orphan_released_units(df, "Jul24", "t.xlsx", None) diff --git a/tests/test_extract/test_wide_format.py b/tests/test_extract/test_wide_format.py new file mode 100644 index 0000000..d0bac62 --- /dev/null +++ b/tests/test_extract/test_wide_format.py @@ -0,0 +1,257 @@ +"""Unit tests for Mandalay wide-format tracker reshaping (steps 1.4a/1.4b).""" + +import polars as pl + +from a4d.extract.wide_format import handle_wide_format_cells, handle_wide_format_columns + +# --- handle_wide_format_columns (step 1.4a: 2020-2021 wide columns) --- + +_TOTAL = "Total Units Released" +_PER_PERSON = "Units Released per person" +_RELEASED_TO_WIDE = "Released To (select from drop down list)" + + +def _columns_df(rows: list[dict], extra_cols: list[str] | None = None) -> pl.DataFrame: + # Real tracker layout: Total precedes Released To; the intermediate + # per-recipient columns sit between Released To and Units Released per + # person (that's the span handle_wide_format_columns expands). + cols = [ + "Date", + _TOTAL, + _RELEASED_TO_WIDE, + *(extra_cols or ["Recipient A", "Recipient B"]), + _PER_PERSON, + ] + data = {c: [r.get(c) for r in rows] for c in cols} + return pl.DataFrame(data) + + +def test_columns_gate_missing_required_column_returns_unchanged(): + df = pl.DataFrame({"Date": ["2020-01-01"], _TOTAL: [5]}) + + result = handle_wide_format_columns(df, "2020_Mandalay.xlsx") + + assert result.equals(df) + + +def test_columns_gate_empty_dataframe_returns_unchanged(): + df = _columns_df([]) + + result = handle_wide_format_columns(df, "2020_Mandalay.xlsx") + + assert result.height == 0 + assert result.columns == df.columns + + +def test_columns_no_split_when_total_equals_per_person(): + row = { + "Date": "2020-01-01", + _RELEASED_TO_WIDE: "Clinic X", + "Recipient A": "5", + "Recipient B": None, + _TOTAL: 5, + _PER_PERSON: 5, + } + df = _columns_df([row]) + + result = handle_wide_format_columns(df, "2020_Mandalay.xlsx") + + assert result.height == 1 + + +def test_columns_no_split_when_total_or_per_person_missing(): + row = { + "Date": "2020-01-01", + _RELEASED_TO_WIDE: "Clinic X", + "Recipient A": "5", + "Recipient B": None, + _TOTAL: None, + _PER_PERSON: None, + } + df = _columns_df([row]) + + result = handle_wide_format_columns(df, "2020_Mandalay.xlsx") + + assert result.height == 1 + + +def test_columns_expands_intermediate_cells_into_rows(): + row = { + "Date": "2020-01-01", + _RELEASED_TO_WIDE: "Clinic X", + "Recipient A": "5", + "Recipient B": "3", + _TOTAL: 8, + _PER_PERSON: 4, + } + df = _columns_df([row]) + + result = handle_wide_format_columns(df, "2020_Mandalay.xlsx") + + # Original row + one new row per non-empty intermediate cell. + assert result.height == 3 + expanded = result.tail(2) + assert expanded[_RELEASED_TO_WIDE].to_list() == ["5", "3"] + assert expanded[_PER_PERSON].to_list() == [4, 4] + assert expanded["Date"].to_list() == ["2020-01-01", "2020-01-01"] + + +def test_columns_skips_blank_intermediate_cells(): + row = { + "Date": "2020-01-01", + _RELEASED_TO_WIDE: "Clinic X", + "Recipient A": " ", + "Recipient B": None, + _TOTAL: 8, + _PER_PERSON: 4, + } + df = _columns_df([row]) + + result = handle_wide_format_columns(df, "2020_Mandalay.xlsx") + + assert result.height == 1 + + +def test_columns_start_after_end_returns_unchanged(): + # Recipient columns fall between _RELEASED_TO_WIDE and _PER_PERSON; if + # they're adjacent (no intermediate columns), start > end and nothing splits. + cols = ["Date", _RELEASED_TO_WIDE, _PER_PERSON, _TOTAL] + df = pl.DataFrame(dict(zip(cols, [["2020-01-01"], ["Clinic X"], [4], [8]], strict=True))) + + result = handle_wide_format_columns(df, "2020_Mandalay.xlsx") + + assert result.height == 1 + + +def test_columns_handles_missing_date_column(): + cols = [_TOTAL, _RELEASED_TO_WIDE, "Recipient A", _PER_PERSON] + row_no_split = dict(zip(cols, [5, "Clinic X", "5", 5], strict=True)) + row_splits = dict(zip(cols, [8, "Clinic Y", "3", 4], strict=True)) + df = pl.DataFrame({c: [row_no_split[c], row_splits[c]] for c in cols}) + + result = handle_wide_format_columns(df, "2020_Mandalay.xlsx") + + assert "Date" not in result.columns + # Row 1 unchanged (total == per_person); row 2 unchanged + 1 split row. + assert result.height == 3 + + +# --- handle_wide_format_cells (step 1.4b: 2017-2019 comma-separated cells) --- + + +def _cells_df(rows: list[dict]) -> pl.DataFrame: + cols = ["Date", "Received From", "Released To", "Units Released"] + data = {c: [r.get(c) for r in rows] for c in cols} + return pl.DataFrame(data) + + +def test_cells_gate_filename_not_mandalay_2017_2019_returns_unchanged(): + df = _cells_df([{"Released To": "A - 5, B - 3"}]) + + result = handle_wide_format_cells(df, "2020_Mandalay.xlsx") + + assert result.equals(df) + + +def test_cells_gate_empty_dataframe_returns_unchanged(): + df = _cells_df([]) + + result = handle_wide_format_cells(df, "2019_Mandalay.xlsx") + + assert result.height == 0 + + +def test_cells_gate_missing_required_column_returns_unchanged(): + df = pl.DataFrame({"Date": ["2019-01-01"]}) + + result = handle_wide_format_cells(df, "2019_Mandalay.xlsx") + + assert result.equals(df) + + +def test_cells_gate_no_comma_or_dash_pattern_returns_unchanged(): + df = _cells_df( + [ + { + "Date": "2019-01-01", + "Received From": "HQ", + "Released To": "Clinic X", + "Units Released": 5, + } + ] + ) + + result = handle_wide_format_cells(df, "2019_Mandalay.xlsx") + + assert result.equals(df) + + +def test_cells_splits_comma_separated_fragments(): + df = _cells_df( + [ + { + "Date": "2019-01-01", + "Received From": "HQ", + "Released To": "Clinic A - 5, Clinic B - 3", + "Units Released": None, + } + ] + ) + + result = handle_wide_format_cells(df, "2019_Mandalay.xlsx") + + # Original row (nulled at released_to/units_released) + two split rows. + assert result.height == 3 + split_rows = result.tail(2) + assert split_rows["Released To"].to_list() == ["Clinic A", "Clinic B"] + assert split_rows["Units Released"].to_list() == ["5", "3"] + assert split_rows["Received From"].to_list() == ["HQ", "HQ"] + assert split_rows["Date"].to_list() == ["2019-01-01", "2019-01-01"] + + original_row = result.head(1) + assert original_row["Released To"].to_list() == [None] + assert original_row["Units Released"].to_list() == [None] + + +def test_cells_fragment_without_dash_has_no_quantity(): + df = _cells_df( + [ + { + "Date": "2019-01-01", + "Received From": "HQ", + "Released To": "Clinic A, Clinic B - 3", + "Units Released": None, + } + ] + ) + + result = handle_wide_format_cells(df, "2019_Mandalay.xlsx") + + split_rows = result.tail(2) + assert split_rows["Released To"].to_list() == ["Clinic A", "Clinic B"] + assert split_rows["Units Released"].to_list() == [None, "3"] + + +def test_cells_leaves_non_comma_cells_untouched(): + df = _cells_df( + [ + { + "Date": "2019-01-01", + "Received From": "HQ", + "Released To": "Clinic A - 5", + "Units Released": None, + }, + { + "Date": "2019-01-02", + "Received From": "HQ", + "Released To": "Clinic C, Clinic D", + "Units Released": None, + }, + ] + ) + + result = handle_wide_format_cells(df, "2018_Mandalay.xlsx") + + # First row has no comma -> not split, appended as-is. + first_row = result.head(1) + assert first_row["Released To"].to_list() == ["Clinic A - 5"] diff --git a/tests/test_gcp/__init__.py b/tests/test_gcp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_gcp/test_bigquery.py b/tests/test_gcp/test_bigquery.py new file mode 100644 index 0000000..ea27b34 --- /dev/null +++ b/tests/test_gcp/test_bigquery.py @@ -0,0 +1,173 @@ +"""Tests for BigQuery loading module.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from a4d.gcp.bigquery import ( + PARQUET_TO_TABLE, + TABLE_CONFIGS, + load_pipeline_tables, + load_table, +) + + +def _get_job_config(mock_client): + """Extract job_config from mock client's load_table_from_file call.""" + return mock_client.load_table_from_file.call_args.kwargs["job_config"] + + +class TestTableConfigs: + """Every published table has clustering fields declared.""" + + def test_patient_data_monthly_clustering(self): + assert TABLE_CONFIGS["patient_data_monthly"] == [ + "clinic_id", + "patient_id", + "tracker_date", + ] + + def test_patient_data_annual_clustering(self): + assert TABLE_CONFIGS["patient_data_annual"] == ["patient_id", "tracker_date"] + + def test_patient_data_static_clustering(self): + assert TABLE_CONFIGS["patient_data_static"] == [ + "clinic_id", + "patient_id", + "tracker_date", + ] + + def test_all_pipeline_tables_have_configs(self): + for table_name in PARQUET_TO_TABLE.values(): + assert table_name in TABLE_CONFIGS, f"Missing config for {table_name}" + + +class TestLoadTable: + """Test loading a single parquet file to BigQuery.""" + + def test_raises_file_not_found(self, tmp_path): + missing_file = tmp_path / "missing.parquet" + with pytest.raises(FileNotFoundError, match="Parquet file not found"): + load_table(missing_file, "patient_data_monthly") + + @patch("a4d.gcp.bigquery.get_bigquery_client") + def test_load_table_with_replace(self, mock_get_client, tmp_path): + parquet_file = tmp_path / "test.parquet" + parquet_file.write_bytes(b"fake parquet data") + + mock_client = MagicMock() + mock_job = MagicMock() + mock_job.output_rows = 100 + mock_client.load_table_from_file.return_value = mock_job + mock_get_client.return_value = mock_client + + load_table(parquet_file, "patient_data_monthly", client=mock_client) + + mock_client.load_table_from_file.assert_called_once() + job_config = _get_job_config(mock_client) + assert job_config.clustering_fields == ["clinic_id", "patient_id", "tracker_date"] + mock_job.result.assert_called_once() + + @patch("a4d.gcp.bigquery.get_bigquery_client") + def test_load_table_with_append(self, mock_get_client, tmp_path): + parquet_file = tmp_path / "test.parquet" + parquet_file.write_bytes(b"fake parquet data") + + mock_client = MagicMock() + mock_job = MagicMock() + mock_job.output_rows = 50 + mock_client.load_table_from_file.return_value = mock_job + + load_table(parquet_file, "patient_data_monthly", client=mock_client, replace=False) + + job_config = _get_job_config(mock_client) + assert job_config.write_disposition == "WRITE_APPEND" + + @patch("a4d.gcp.bigquery.get_bigquery_client") + def test_load_table_correct_table_ref(self, mock_get_client, tmp_path): + parquet_file = tmp_path / "test.parquet" + parquet_file.write_bytes(b"fake parquet data") + + mock_client = MagicMock() + mock_job = MagicMock() + mock_job.output_rows = 10 + mock_client.load_table_from_file.return_value = mock_job + + load_table( + parquet_file, + "patient_data_static", + client=mock_client, + dataset="test_dataset", + project_id="test_project", + ) + + table_ref = mock_client.load_table_from_file.call_args.args[1] + assert table_ref == "test_project.test_dataset.patient_data_static" + + +class TestLoadPipelineTables: + """Test loading all pipeline tables.""" + + def test_raises_if_dir_missing(self, tmp_path): + missing_dir = tmp_path / "nonexistent" + with pytest.raises(FileNotFoundError, match="Tables directory not found"): + load_pipeline_tables(missing_dir) + + @patch("a4d.gcp.bigquery.load_table") + @patch("a4d.gcp.bigquery.get_bigquery_client") + def test_loads_existing_tables(self, mock_get_client, mock_load, tmp_path): + tables_dir = tmp_path / "tables" + tables_dir.mkdir() + + # Create some table files + (tables_dir / "patient_data_static.parquet").write_bytes(b"data") + (tables_dir / "patient_data_monthly.parquet").write_bytes(b"data") + + mock_client = MagicMock() + mock_get_client.return_value = mock_client + mock_load.return_value = MagicMock() + + results = load_pipeline_tables(tables_dir, client=mock_client) + + assert mock_load.call_count == 2 + assert "patient_data_static" in results + assert "patient_data_monthly" in results + + @patch("a4d.gcp.bigquery.load_table") + @patch("a4d.gcp.bigquery.get_bigquery_client") + def test_skips_missing_tables(self, mock_get_client, mock_load, tmp_path): + tables_dir = tmp_path / "tables" + tables_dir.mkdir() + + # Only create one table file + (tables_dir / "patient_data_static.parquet").write_bytes(b"data") + + mock_client = MagicMock() + mock_get_client.return_value = mock_client + mock_load.return_value = MagicMock() + + results = load_pipeline_tables(tables_dir, client=mock_client) + + assert mock_load.call_count == 1 + assert "patient_data_static" in results + assert "patient_data_monthly" not in results + + @patch("a4d.gcp.bigquery.load_table") + @patch("a4d.gcp.bigquery.get_bigquery_client") + def test_continues_on_single_table_failure(self, mock_get_client, mock_load, tmp_path): + tables_dir = tmp_path / "tables" + tables_dir.mkdir() + + (tables_dir / "patient_data_static.parquet").write_bytes(b"data") + (tables_dir / "patient_data_monthly.parquet").write_bytes(b"data") + + mock_client = MagicMock() + mock_get_client.return_value = mock_client + + # First call succeeds, second fails + mock_load.side_effect = [MagicMock(), Exception("API error")] + + results = load_pipeline_tables(tables_dir, client=mock_client) + + # Should have one success despite the failure + assert len(results) == 1 diff --git a/tests/test_gcp/test_drive.py b/tests/test_gcp/test_drive.py new file mode 100644 index 0000000..023345f --- /dev/null +++ b/tests/test_gcp/test_drive.py @@ -0,0 +1,89 @@ +"""Tests for Google Drive download module.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from a4d.gcp.drive import CLINIC_DATA_FILE_ID, download_clinic_data + + +class TestDownloadClinicData: + """Tests for download_clinic_data.""" + + @patch("a4d.gcp.drive.google.auth.default") + @patch("a4d.gcp.drive.google.auth.transport.requests.AuthorizedSession") + def test_downloads_to_destination(self, mock_session_cls, mock_auth_default, tmp_path): + mock_auth_default.return_value = (MagicMock(), "project") + mock_session = MagicMock() + mock_session_cls.return_value = mock_session + mock_response = MagicMock() + mock_response.iter_content.return_value = [b"data1", b"data2"] + mock_session.get.return_value = mock_response + + result = download_clinic_data(tmp_path) + + assert result == tmp_path / "clinic_data.xlsx" + assert result.exists() + assert result.read_bytes() == b"data1data2" + + @patch("a4d.gcp.drive.google.auth.default") + @patch("a4d.gcp.drive.google.auth.transport.requests.AuthorizedSession") + def test_uses_correct_file_id(self, mock_session_cls, mock_auth_default, tmp_path): + mock_auth_default.return_value = (MagicMock(), "project") + mock_session = MagicMock() + mock_session_cls.return_value = mock_session + mock_response = MagicMock() + mock_response.iter_content.return_value = [b"xlsx"] + mock_session.get.return_value = mock_response + + download_clinic_data(tmp_path) + + call_url = mock_session.get.call_args[0][0] + assert CLINIC_DATA_FILE_ID in call_url + assert "/export" in call_url + assert "mimeType=" in call_url + + @patch("a4d.gcp.drive.google.auth.default") + @patch("a4d.gcp.drive.google.auth.transport.requests.AuthorizedSession") + def test_uses_drive_readonly_scope(self, mock_session_cls, mock_auth_default, tmp_path): + mock_auth_default.return_value = (MagicMock(), "project") + mock_session = MagicMock() + mock_session_cls.return_value = mock_session + mock_response = MagicMock() + mock_response.iter_content.return_value = [b"xlsx"] + mock_session.get.return_value = mock_response + + download_clinic_data(tmp_path) + + scopes = mock_auth_default.call_args[1]["scopes"] + assert any("drive" in s for s in scopes) + + @patch("a4d.gcp.drive.google.auth.default") + @patch("a4d.gcp.drive.google.auth.transport.requests.AuthorizedSession") + def test_raises_on_http_error(self, mock_session_cls, mock_auth_default, tmp_path): + import requests as req + + mock_auth_default.return_value = (MagicMock(), "project") + mock_session = MagicMock() + mock_session_cls.return_value = mock_session + mock_response = MagicMock() + mock_response.raise_for_status.side_effect = req.HTTPError("403 Forbidden") + mock_session.get.return_value = mock_response + + with pytest.raises(req.HTTPError): + download_clinic_data(tmp_path) + + @patch("a4d.gcp.drive.google.auth.default") + @patch("a4d.gcp.drive.google.auth.transport.requests.AuthorizedSession") + def test_creates_destination_directory(self, mock_session_cls, mock_auth_default, tmp_path): + mock_auth_default.return_value = (MagicMock(), "project") + mock_session = MagicMock() + mock_session_cls.return_value = mock_session + mock_response = MagicMock() + mock_response.iter_content.return_value = [b"xlsx"] + mock_session.get.return_value = mock_response + + dest = tmp_path / "new" / "subdir" + download_clinic_data(dest) + + assert dest.exists() diff --git a/tests/test_gcp/test_select_tracker_metadata.py b/tests/test_gcp/test_select_tracker_metadata.py new file mode 100644 index 0000000..196377f --- /dev/null +++ b/tests/test_gcp/test_select_tracker_metadata.py @@ -0,0 +1,76 @@ +"""Tests for select_tracker_metadata.""" + +from unittest.mock import MagicMock, patch + +from google.api_core.exceptions import GoogleAPIError, NotFound + +from a4d.gcp.bigquery import select_tracker_metadata + + +def _query_result(rows: list[dict]) -> MagicMock: + """Build a MagicMock that mimics client.query(...).result().""" + job = MagicMock() + job.result.return_value = rows + return job + + +def test_happy_path_returns_dataframe(): + rows = [{"file_name": "T1", "clinic_code": "A", "md5": "abc", "complete": True}] + client = MagicMock() + client.query.return_value = _query_result(rows) + + result = select_tracker_metadata(client=client, dataset="tracker", project_id="proj") + + assert result is not None + assert result.height == 1 + assert result["complete"].to_list() == [True] + + +def test_not_found_returns_none(): + client = MagicMock() + client.query.side_effect = NotFound("table not found") + + result = select_tracker_metadata(client=client, dataset="tracker", project_id="proj") + + assert result is None + + +def test_schema_fallback_when_complete_column_missing(): + """Older deployments may have shipped without the `complete` column.""" + client = MagicMock() + # First query (with `complete`) raises a column-missing GoogleAPIError; + # second query (without `complete`) succeeds with the legacy schema. + fallback_rows = [{"file_name": "T1", "clinic_code": "A", "md5": "abc"}] + client.query.side_effect = [ + GoogleAPIError("Unrecognized name: complete at [1:14]"), + _query_result(fallback_rows), + ] + + result = select_tracker_metadata(client=client, dataset="tracker", project_id="proj") + + assert result is not None + assert result["complete"].to_list() == [False], ( + "Schema fallback must synthesise complete=False to force a full reprocess" + ) + assert client.query.call_count == 2 + + +def test_unrelated_google_api_error_returns_none(): + client = MagicMock() + client.query.side_effect = GoogleAPIError("Network unreachable") + + result = select_tracker_metadata(client=client, dataset="tracker", project_id="proj") + + assert result is None + + +def test_auth_failure_during_client_construction_returns_none(): + """When no client is supplied and `get_bigquery_client` raises (e.g. missing + credentials), the function must fall through with a warning, not raise.""" + with patch( + "a4d.gcp.bigquery.get_bigquery_client", + side_effect=Exception("DefaultCredentialsError: no creds"), + ): + result = select_tracker_metadata(dataset="tracker", project_id="proj") + + assert result is None diff --git a/tests/test_gcp/test_storage.py b/tests/test_gcp/test_storage.py new file mode 100644 index 0000000..77ff437 --- /dev/null +++ b/tests/test_gcp/test_storage.py @@ -0,0 +1,114 @@ +"""Tests for Google Cloud Storage module.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from a4d.gcp.storage import download_tracker_files, upload_output + + +class TestDownloadTrackerFiles: + """Test downloading tracker files from GCS.""" + + @patch("a4d.gcp.storage.get_storage_client") + def test_downloads_files(self, mock_get_client, tmp_path): + destination = tmp_path / "trackers" + + mock_client = MagicMock() + mock_get_client.return_value = mock_client + mock_bucket = MagicMock() + mock_client.bucket.return_value = mock_bucket + + # Simulate blobs in bucket + blob1 = MagicMock() + blob1.name = "2024/tracker1.xlsx" + blob2 = MagicMock() + blob2.name = "2024/tracker2.xlsx" + mock_bucket.list_blobs.return_value = [blob1, blob2] + + result = download_tracker_files(destination, client=mock_client) + + assert len(result) == 2 + assert blob1.download_to_filename.called + assert blob2.download_to_filename.called + + @patch("a4d.gcp.storage.get_storage_client") + def test_skips_directory_markers(self, mock_get_client, tmp_path): + destination = tmp_path / "trackers" + + mock_client = MagicMock() + mock_get_client.return_value = mock_client + mock_bucket = MagicMock() + mock_client.bucket.return_value = mock_bucket + + blob_dir = MagicMock() + blob_dir.name = "2024/" + blob_file = MagicMock() + blob_file.name = "2024/tracker.xlsx" + mock_bucket.list_blobs.return_value = [blob_dir, blob_file] + + result = download_tracker_files(destination, client=mock_client) + + assert len(result) == 1 + assert not blob_dir.download_to_filename.called + + @patch("a4d.gcp.storage.get_storage_client") + def test_creates_destination_directory(self, mock_get_client, tmp_path): + destination = tmp_path / "new" / "dir" + + mock_client = MagicMock() + mock_get_client.return_value = mock_client + mock_bucket = MagicMock() + mock_client.bucket.return_value = mock_bucket + mock_bucket.list_blobs.return_value = [] + + download_tracker_files(destination, client=mock_client) + + assert destination.exists() + + +class TestUploadOutput: + """Test uploading output to GCS.""" + + def test_raises_if_source_missing(self, tmp_path): + missing_dir = tmp_path / "nonexistent" + with pytest.raises(FileNotFoundError, match="Source directory not found"): + upload_output(missing_dir) + + @patch("a4d.gcp.storage.get_storage_client") + def test_uploads_files(self, mock_get_client, tmp_path): + source = tmp_path / "output" + source.mkdir() + (source / "tables").mkdir() + (source / "tables" / "data.parquet").write_bytes(b"data") + (source / "logs.txt").write_text("log") + + mock_client = MagicMock() + mock_get_client.return_value = mock_client + mock_bucket = MagicMock() + mock_client.bucket.return_value = mock_bucket + mock_blob = MagicMock() + mock_bucket.blob.return_value = mock_blob + + result = upload_output(source, client=mock_client) + + assert len(result) == 2 + assert mock_blob.upload_from_filename.call_count == 2 + + @patch("a4d.gcp.storage.get_storage_client") + def test_upload_with_prefix(self, mock_get_client, tmp_path): + source = tmp_path / "output" + source.mkdir() + (source / "file.parquet").write_bytes(b"data") + + mock_client = MagicMock() + mock_get_client.return_value = mock_client + mock_bucket = MagicMock() + mock_client.bucket.return_value = mock_bucket + mock_blob = MagicMock() + mock_bucket.blob.return_value = mock_blob + + result = upload_output(source, prefix="2024-01", client=mock_client) + + assert len(result) == 1 + assert result[0] == "2024-01/file.parquet" diff --git a/tests/test_gcp/test_verify.py b/tests/test_gcp/test_verify.py new file mode 100644 index 0000000..1433321 --- /dev/null +++ b/tests/test_gcp/test_verify.py @@ -0,0 +1,122 @@ +"""Tests for post-production-run verification against a backup snapshot.""" + +from unittest.mock import MagicMock + +from a4d.gcp.verify import TableStats, diff_table_stats, fetch_table_stats + + +def _stats(row_count=100, distinct_clinics=10, columns=("clinic_id", "patient_id")): + return TableStats( + row_count=row_count, distinct_clinics=distinct_clinics, columns=frozenset(columns) + ) + + +class TestDiffTableStats: + def test_no_anomalies_when_stats_match(self): + before = {"patient_data_static": _stats()} + after = {"patient_data_static": _stats()} + + assert diff_table_stats(before, after) == [] + + def test_no_anomalies_when_row_count_grows(self): + before = {"product_data": _stats(row_count=100)} + after = {"product_data": _stats(row_count=200)} + + assert diff_table_stats(before, after) == [] + + def test_flags_large_row_count_drop(self): + before = {"product_data": _stats(row_count=100)} + after = {"product_data": _stats(row_count=50)} + + anomalies = diff_table_stats(before, after) + + assert len(anomalies) == 1 + assert anomalies[0].table == "product_data" + assert "row count dropped" in anomalies[0].reason + + def test_ignores_row_count_drop_within_threshold(self): + before = {"product_data": _stats(row_count=100)} + after = {"product_data": _stats(row_count=95)} + + assert diff_table_stats(before, after) == [] + + def test_flags_schema_change(self): + before = {"product_data": _stats(columns=("clinic_id", "patient_id"))} + after = {"product_data": _stats(columns=("clinic_id",))} + + anomalies = diff_table_stats(before, after) + + assert len(anomalies) == 1 + assert "schema changed" in anomalies[0].reason + + def test_flags_distinct_clinic_drop(self): + before = {"product_data": _stats(distinct_clinics=10)} + after = {"product_data": _stats(distinct_clinics=5)} + + anomalies = diff_table_stats(before, after) + + assert len(anomalies) == 1 + assert "distinct clinics dropped" in anomalies[0].reason + + def test_skips_clinic_check_when_table_has_no_clinic_id(self): + before = {"patient_data_annual": _stats(distinct_clinics=None, columns=("patient_id",))} + after = {"patient_data_annual": _stats(distinct_clinics=None, columns=("patient_id",))} + + assert diff_table_stats(before, after) == [] + + def test_flags_table_missing_after_run(self): + before = {"product_data": _stats()} + after = {} + + anomalies = diff_table_stats(before, after) + + assert len(anomalies) == 1 + assert anomalies[0].table == "product_data" + assert "missing" in anomalies[0].reason + + def test_ignores_table_absent_from_before(self): + before = {} + after = {"product_data": _stats()} + + assert diff_table_stats(before, after) == [] + + +class TestFetchTableStats: + def test_queries_row_count_distinct_clinics_and_schema(self): + mock_client = MagicMock() + + mock_field = MagicMock() + mock_field.name = "clinic_id" + mock_table = MagicMock() + mock_table.schema = [mock_field] + mock_client.get_table.return_value = mock_table + + mock_row = MagicMock(row_count=42, distinct_clinics=7) + mock_client.query.return_value.result.return_value = [mock_row] + + stats = fetch_table_stats(mock_client, "product_data") + + assert stats == TableStats( + row_count=42, distinct_clinics=7, columns=frozenset({"clinic_id"}) + ) + mock_client.get_table.assert_called_once_with("a4dphase2.tracker.product_data") + + def test_skips_distinct_clinic_query_when_no_clinic_id_column(self): + mock_client = MagicMock() + + mock_field = MagicMock() + mock_field.name = "patient_id" + mock_table = MagicMock() + mock_table.schema = [mock_field] + mock_client.get_table.return_value = mock_table + + mock_row = MagicMock(row_count=42) + mock_client.query.return_value.result.return_value = [mock_row] + + stats = fetch_table_stats(mock_client, "patient_data_annual") + + assert stats == TableStats( + row_count=42, distinct_clinics=None, columns=frozenset({"patient_id"}) + ) + query = mock_client.query.call_args.args[0] + assert "clinic_id" not in query diff --git a/tests/test_integration/__init__.py b/tests/test_integration/__init__.py new file mode 100644 index 0000000..19172f4 --- /dev/null +++ b/tests/test_integration/__init__.py @@ -0,0 +1,9 @@ +"""Integration tests for A4D pipeline. + +These tests use real tracker files and are marked as 'slow' and 'integration'. +They are skipped by default in CI/CD to keep test runs fast. + +Run them explicitly with: + uv run pytest -m integration + uv run pytest tests/test_integration/ +""" diff --git a/tests/test_integration/conftest.py b/tests/test_integration/conftest.py new file mode 100644 index 0000000..f4ef449 --- /dev/null +++ b/tests/test_integration/conftest.py @@ -0,0 +1,55 @@ +"""Shared fixtures for integration tests.""" + +from pathlib import Path + +import pytest + +# Base path to tracker files +TRACKER_BASE = Path("/Volumes/USB SanDisk 3.2Gen1 Media/a4d/a4dphase2_upload") + + +@pytest.fixture +def tracker_2024_penang(): + """2024 Penang tracker - has Annual + Patient List sheets.""" + return TRACKER_BASE / "Malaysia/PNG/2024_Penang General Hospital A4D Tracker.xlsx" + + +@pytest.fixture +def tracker_2023_sibu(): + """2023 Sibu tracker - has duplicate column mapping edge case.""" + return TRACKER_BASE / "Malaysia/SBU/2023_Sibu Hospital A4D Tracker.xlsx" + + +@pytest.fixture +def tracker_2022_penang(): + """2022 Penang tracker - legacy format without Annual sheet.""" + return TRACKER_BASE / "Malaysia/PNG/2022_Penang General Hospital A4D Tracker.xlsx" + + +@pytest.fixture +def tracker_2024_isdfi(): + """2024 ISDFI Philippines tracker.""" + return TRACKER_BASE / "Philippines/ISD/2024_ISDFI A4D Tracker.xlsx" + + +@pytest.fixture +def tracker_2020_mandalay(): + """2020 Mandalay Children's tracker - triggers wide-format column expansion.""" + return TRACKER_BASE / "Myanmar/MCH/2020_Mandalay Children's Hospital A4D Tracker.xlsx" + + +@pytest.fixture +def tracker_2018_mandalay(): + """2018 Mandalay Children's tracker - triggers comma-separated cell splitting.""" + return TRACKER_BASE / "Myanmar/MCH/2018_Mandalay Children's Hospital A4D Tracker.xlsx" + + +# Expected values for validation +EXPECTED_SCHEMA_COLS = 83 # After cleaning (patient) +EXPECTED_SCHEMA_COLS_PRODUCT = 20 # After cleaning (product) + + +def skip_if_missing(tracker_path: Path): + """Skip test if tracker file is not available.""" + if not tracker_path.exists(): + pytest.skip(f"Tracker file not found: {tracker_path}") diff --git a/tests/test_integration/test_clean_integration.py b/tests/test_integration/test_clean_integration.py new file mode 100644 index 0000000..82207e0 --- /dev/null +++ b/tests/test_integration/test_clean_integration.py @@ -0,0 +1,229 @@ +"""Integration tests for patient data cleaning. + +Tests cleaning on real extracted data, validating: +- Correct schema (83 columns) +- Type conversions work correctly +- Error tracking works +- Derived columns are created +""" + +import pytest + +from a4d.clean.patient import clean_patient_data +from a4d.clean.product import clean_product_data +from a4d.errors import ErrorCollector +from a4d.extract.patient import read_all_patient_sheets +from a4d.extract.product import read_all_product_sheets + +from .conftest import EXPECTED_SCHEMA_COLS, EXPECTED_SCHEMA_COLS_PRODUCT, skip_if_missing + +pytestmark = [pytest.mark.slow, pytest.mark.integration] + + +class TestClean2024Penang: + """Test cleaning on 2024 Penang extracted data.""" + + def test_clean_produces_correct_schema(self, tracker_2024_penang): + """Should produce exactly 83 columns after cleaning.""" + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_patient_sheets(tracker_2024_penang) + collector = ErrorCollector() + df_clean = clean_patient_data(df_raw, collector) + + assert len(df_clean.columns) == EXPECTED_SCHEMA_COLS + + def test_clean_preserves_row_count(self, tracker_2024_penang): + """Should not drop rows during cleaning.""" + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_patient_sheets(tracker_2024_penang) + collector = ErrorCollector() + df_clean = clean_patient_data(df_raw, collector) + + assert len(df_clean) == len(df_raw) + + def test_clean_creates_derived_columns(self, tracker_2024_penang): + """Should create derived columns (insulin_type, insulin_subtype, etc.).""" + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_patient_sheets(tracker_2024_penang) + collector = ErrorCollector() + df_clean = clean_patient_data(df_raw, collector) + + # Check derived columns exist + assert "insulin_type" in df_clean.columns + assert "insulin_subtype" in df_clean.columns + assert "blood_pressure_sys_mmhg" in df_clean.columns + assert "blood_pressure_dias_mmhg" in df_clean.columns + + def test_clean_tracks_errors(self, tracker_2024_penang): + """Should track data quality errors in ErrorCollector.""" + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_patient_sheets(tracker_2024_penang) + collector = ErrorCollector() + clean_patient_data(df_raw, collector) + + # Should have some errors (type conversions, invalid values, etc.) + # Exact count varies, but should be non-zero for this tracker + assert len(collector) >= 0 # May have 0 or more errors + + def test_clean_has_required_columns(self, tracker_2024_penang): + """Should have all required columns in final schema.""" + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_patient_sheets(tracker_2024_penang) + collector = ErrorCollector() + df_clean = clean_patient_data(df_raw, collector) + + # Check key columns exist + required_columns = [ + "patient_id", + "tracker_year", + "tracker_month", + "age", + "hba1c_updated", + "fbg_updated_mg", + "insulin_type", + ] + for col in required_columns: + assert col in df_clean.columns, f"Missing required column: {col}" + + +class TestClean2023Sibu: + """Test cleaning on 2023 Sibu (edge case).""" + + def test_clean_after_duplicate_handling(self, tracker_2023_sibu): + """Should clean successfully after duplicate column handling.""" + skip_if_missing(tracker_2023_sibu) + + df_raw = read_all_patient_sheets(tracker_2023_sibu) + collector = ErrorCollector() + df_clean = clean_patient_data(df_raw, collector) + + assert len(df_clean.columns) == EXPECTED_SCHEMA_COLS + assert len(df_clean) == 14 + + +class TestClean2022PenangLegacy: + """Test cleaning on 2022 Penang (legacy format).""" + + def test_clean_legacy_format(self, tracker_2022_penang): + """Should clean legacy format to same 83-column schema.""" + skip_if_missing(tracker_2022_penang) + + df_raw = read_all_patient_sheets(tracker_2022_penang) + collector = ErrorCollector() + df_clean = clean_patient_data(df_raw, collector) + + # Should produce same schema regardless of input format + assert len(df_clean.columns) == EXPECTED_SCHEMA_COLS + assert len(df_clean) == 156 + + def test_clean_legacy_has_patient_list_data(self, tracker_2022_penang): + """Should preserve Patient List data (dob, province, etc.) after cleaning.""" + skip_if_missing(tracker_2022_penang) + + df_raw = read_all_patient_sheets(tracker_2022_penang) + collector = ErrorCollector() + df_clean = clean_patient_data(df_raw, collector) + + # Patient List columns should be preserved + assert "dob" in df_clean.columns + assert "province" in df_clean.columns + assert "sex" in df_clean.columns + + +class TestCleanProduct2024Penang: + """Test product cleaning on 2024 Penang extracted data.""" + + def test_clean_produces_correct_schema(self, tracker_2024_penang): + """Should produce exactly 20 columns after cleaning (product meta schema).""" + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_product_sheets(tracker_2024_penang) + collector = ErrorCollector() + df_clean = clean_product_data(df_raw, collector) + + assert len(df_clean.columns) == EXPECTED_SCHEMA_COLS_PRODUCT + + def test_clean_drops_uninformative_rows(self, tracker_2024_penang): + """Unlike patient, product cleaning drops blank/uninformative template rows.""" + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_product_sheets(tracker_2024_penang) + collector = ErrorCollector() + df_clean = clean_product_data(df_raw, collector) + + assert len(df_clean) == 244 + assert len(df_clean) < len(df_raw) + + def test_clean_creates_derived_columns(self, tracker_2024_penang): + """Should create derived columns (balance_status, category, unit_capacity).""" + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_product_sheets(tracker_2024_penang) + collector = ErrorCollector() + df_clean = clean_product_data(df_raw, collector) + + assert "product_balance_status" in df_clean.columns + assert "product_category" in df_clean.columns + assert "product_unit_capacity" in df_clean.columns + + def test_clean_tracks_errors(self, tracker_2024_penang): + """Should track data quality errors in ErrorCollector.""" + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_product_sheets(tracker_2024_penang) + collector = ErrorCollector() + clean_product_data(df_raw, collector) + + assert len(collector) >= 0 + + def test_clean_has_required_columns(self, tracker_2024_penang): + """Should have all required columns in final schema.""" + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_product_sheets(tracker_2024_penang) + collector = ErrorCollector() + df_clean = clean_product_data(df_raw, collector) + + required_columns = [ + "product", + "product_table_year", + "product_table_month", + "product_units_released", + "product_units_received", + "product_balance", + ] + for col in required_columns: + assert col in df_clean.columns, f"Missing required column: {col}" + + +class TestCleanProduct2023Sibu: + """Test product cleaning on 2023 Sibu (edge case).""" + + def test_clean_after_duplicate_handling(self, tracker_2023_sibu): + skip_if_missing(tracker_2023_sibu) + + df_raw = read_all_product_sheets(tracker_2023_sibu) + collector = ErrorCollector() + df_clean = clean_product_data(df_raw, collector) + + assert len(df_clean.columns) == EXPECTED_SCHEMA_COLS_PRODUCT + assert len(df_clean) == 67 + + +class TestCleanProduct2022PenangLegacy: + """Test product cleaning on 2022 Penang (legacy format).""" + + def test_clean_legacy_format(self, tracker_2022_penang): + skip_if_missing(tracker_2022_penang) + + df_raw = read_all_product_sheets(tracker_2022_penang) + collector = ErrorCollector() + df_clean = clean_product_data(df_raw, collector) + + assert len(df_clean.columns) == EXPECTED_SCHEMA_COLS_PRODUCT + assert len(df_clean) == 237 diff --git a/tests/test_integration/test_e2e.py b/tests/test_integration/test_e2e.py new file mode 100644 index 0000000..70d4a76 --- /dev/null +++ b/tests/test_integration/test_e2e.py @@ -0,0 +1,250 @@ +"""End-to-end integration tests for the full pipeline (extraction + cleaning). + +Tests the complete workflow on real tracker files, validating: +- Extraction + Cleaning work together correctly +- Final output has correct schema and row counts +- Different tracker formats (2024, 2023, 2022) all produce consistent output +""" + +import pytest + +from a4d.clean.patient import clean_patient_data +from a4d.clean.product import clean_product_data +from a4d.errors import ErrorCollector +from a4d.extract.patient import read_all_patient_sheets +from a4d.extract.product import read_all_product_sheets + +from .conftest import EXPECTED_SCHEMA_COLS, EXPECTED_SCHEMA_COLS_PRODUCT, skip_if_missing + +pytestmark = [pytest.mark.slow, pytest.mark.integration, pytest.mark.e2e] + + +@pytest.mark.parametrize( + ("tracker_fixture", "expected_rows", "expected_year", "description"), + [ + ("tracker_2024_penang", 174, 2024, "2024 Penang - Annual + Patient List"), + ("tracker_2024_isdfi", 70, 2024, "2024 ISDFI Philippines"), + ("tracker_2023_sibu", 14, 2023, "2023 Sibu - duplicate columns edge case"), + ("tracker_2022_penang", 156, 2022, "2022 Penang - legacy format"), + ], +) +def test_e2e_pipeline(tracker_fixture, expected_rows, expected_year, description, request): + """Test full pipeline (extract + clean) on various tracker formats. + + This test validates that: + 1. Extraction works and produces expected row count + 2. Cleaning works and produces 83-column schema + 3. Row count is preserved through the pipeline + 4. Year is extracted correctly + """ + tracker_path = request.getfixturevalue(tracker_fixture) + skip_if_missing(tracker_path) + + # Step 1: Extract + df_raw = read_all_patient_sheets(tracker_path) + assert len(df_raw) == expected_rows, f"Extraction failed for {description}" + + # Step 2: Clean + collector = ErrorCollector() + df_clean = clean_patient_data(df_raw, collector) + + # Validate final output + assert len(df_clean) == expected_rows, f"Cleaning changed row count for {description}" + assert len(df_clean.columns) == EXPECTED_SCHEMA_COLS, f"Schema incorrect for {description}" + assert df_clean["tracker_year"].unique().to_list() == [expected_year], ( + f"Year incorrect for {description}" + ) + + +class TestE2E2024Penang: + """Detailed end-to-end test for 2024 Penang tracker.""" + + def test_e2e_full_pipeline(self, tracker_2024_penang): + """Test complete pipeline with detailed validations.""" + skip_if_missing(tracker_2024_penang) + + # Extract + df_raw = read_all_patient_sheets(tracker_2024_penang) + assert len(df_raw) == 174 + + # Clean + collector = ErrorCollector() + df_clean = clean_patient_data(df_raw, collector) + + # Validate schema + assert len(df_clean.columns) == 83 + assert len(df_clean) == 174 + + # Validate metadata + assert "tracker_year" in df_clean.columns + assert "tracker_month" in df_clean.columns + assert "clinic_id" in df_clean.columns + + # Validate year and months + assert df_clean["tracker_year"].unique().to_list() == [2024] + months = sorted(df_clean["tracker_month"].unique().to_list()) + assert months == list(range(1, 13)) # Should have all 12 months + + # Validate clinic_id + assert df_clean["clinic_id"].unique().to_list() == ["PNG"] + + def test_e2e_critical_columns_populated(self, tracker_2024_penang): + """Validate that critical columns are fully populated after pipeline.""" + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_patient_sheets(tracker_2024_penang) + collector = ErrorCollector() + df_clean = clean_patient_data(df_raw, collector) + + # These columns must be 100% populated for every row + required_full = [ + "patient_id", + "status", + "clinic_id", + "tracker_year", + "tracker_month", + ] + for col in required_full: + null_count = df_clean[col].is_null().sum() + assert null_count == 0, f"{col} has {null_count} null values, expected 0" + + # These columns should have high population (allow some nulls) + required_partial = ["age", "last_clinic_visit_date"] + for col in required_partial: + non_null = df_clean[col].is_not_null().sum() + assert non_null > len(df_clean) * 0.9, f"{col} has <90% population" + + +class TestE2ECrosYearConsistency: + """Test that different years produce consistent schemas.""" + + def test_all_years_produce_same_schema( + self, tracker_2024_penang, tracker_2023_sibu, tracker_2022_penang + ): + """All tracker years should produce the same 83-column schema.""" + trackers = [ + (tracker_2024_penang, "2024_Penang"), + (tracker_2023_sibu, "2023_Sibu"), + (tracker_2022_penang, "2022_Penang"), + ] + + column_names_per_tracker = {} + + for tracker_path, name in trackers: + if not tracker_path.exists(): + pytest.skip(f"Tracker file not found: {tracker_path}") + + # Full pipeline + df_raw = read_all_patient_sheets(tracker_path) + collector = ErrorCollector() + df_clean = clean_patient_data(df_raw, collector) + + # Collect column names + column_names_per_tracker[name] = set(df_clean.columns) + + # All trackers should have same column names + if len(column_names_per_tracker) > 1: + first_columns = list(column_names_per_tracker.values())[0] + for name, columns in column_names_per_tracker.items(): + assert columns == first_columns, f"{name} has different columns than others" + + +@pytest.mark.parametrize( + ("tracker_fixture", "expected_clean_rows", "expected_year", "description"), + [ + ("tracker_2024_penang", 244, 2024, "2024 Penang - Annual + Patient List"), + ("tracker_2024_isdfi", 115, 2024, "2024 ISDFI Philippines"), + ("tracker_2023_sibu", 67, 2023, "2023 Sibu - duplicate columns edge case"), + ("tracker_2022_penang", 237, 2022, "2022 Penang - legacy format"), + ], +) +def test_e2e_pipeline_product( + tracker_fixture, expected_clean_rows, expected_year, description, request +): + """Test full product pipeline (extract + clean) on various tracker formats. + + Unlike patient, product cleaning drops blank/uninformative template rows, + so extract and clean row counts differ (see TestCleanProduct2024Penang). + """ + tracker_path = request.getfixturevalue(tracker_fixture) + skip_if_missing(tracker_path) + + df_raw = read_all_product_sheets(tracker_path) + assert len(df_raw) > 0, f"Extraction failed for {description}" + + collector = ErrorCollector() + df_clean = clean_product_data(df_raw, collector) + + assert len(df_clean) == expected_clean_rows, f"Cleaning row count wrong for {description}" + assert len(df_clean.columns) == EXPECTED_SCHEMA_COLS_PRODUCT, ( + f"Schema incorrect for {description}" + ) + assert df_clean["product_table_year"].unique().to_list() == [expected_year], ( + f"Year incorrect for {description}" + ) + + +class TestE2EProduct2024Penang: + """Detailed end-to-end test for 2024 Penang tracker (product).""" + + def test_e2e_full_pipeline(self, tracker_2024_penang): + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_product_sheets(tracker_2024_penang) + assert len(df_raw) == 696 + + collector = ErrorCollector() + df_clean = clean_product_data(df_raw, collector) + + assert len(df_clean.columns) == EXPECTED_SCHEMA_COLS_PRODUCT + assert len(df_clean) == 244 + + assert "product_table_year" in df_clean.columns + assert "product_table_month" in df_clean.columns + assert "clinic_id" in df_clean.columns + + assert df_clean["product_table_year"].unique().to_list() == [2024] + assert df_clean["clinic_id"].unique().to_list() == ["PNG"] + + def test_e2e_critical_columns_populated(self, tracker_2024_penang): + """Validate that critical columns are fully populated after pipeline.""" + skip_if_missing(tracker_2024_penang) + + df_raw = read_all_product_sheets(tracker_2024_penang) + collector = ErrorCollector() + df_clean = clean_product_data(df_raw, collector) + + required_full = ["product", "product_table_year", "product_table_month", "clinic_id"] + for col in required_full: + null_count = df_clean[col].is_null().sum() + assert null_count == 0, f"{col} has {null_count} null values, expected 0" + + +class TestE2EProductCrossYearConsistency: + """Test that different years produce consistent product schemas.""" + + def test_all_years_produce_same_schema( + self, tracker_2024_penang, tracker_2023_sibu, tracker_2022_penang + ): + trackers = [ + (tracker_2024_penang, "2024_Penang"), + (tracker_2023_sibu, "2023_Sibu"), + (tracker_2022_penang, "2022_Penang"), + ] + + column_names_per_tracker = {} + + for tracker_path, name in trackers: + if not tracker_path.exists(): + pytest.skip(f"Tracker file not found: {tracker_path}") + + df_raw = read_all_product_sheets(tracker_path) + collector = ErrorCollector() + df_clean = clean_product_data(df_raw, collector) + + column_names_per_tracker[name] = set(df_clean.columns) + + if len(column_names_per_tracker) > 1: + first_columns = list(column_names_per_tracker.values())[0] + for name, columns in column_names_per_tracker.items(): + assert columns == first_columns, f"{name} has different columns than others" diff --git a/tests/test_integration/test_extract_integration.py b/tests/test_integration/test_extract_integration.py new file mode 100644 index 0000000..48cc726 --- /dev/null +++ b/tests/test_integration/test_extract_integration.py @@ -0,0 +1,232 @@ +"""Integration tests for patient data extraction. + +Tests extraction on real tracker files, validating: +- Correct number of rows extracted +- Correct number of columns +- Month sheets are processed correctly +- Annual and Patient List sheets are handled (if present) +- Metadata columns are added correctly +""" + +import pytest + +from a4d.extract.patient import read_all_patient_sheets +from a4d.extract.product import read_all_product_sheets + +from .conftest import skip_if_missing + +pytestmark = [pytest.mark.slow, pytest.mark.integration] + + +class TestExtract2024Penang: + """Test extraction on 2024 Penang tracker (has Annual + Patient List).""" + + def test_extract_total_rows(self, tracker_2024_penang): + """Should extract all patient records from all sheets.""" + skip_if_missing(tracker_2024_penang) + + df = read_all_patient_sheets(tracker_2024_penang) + + # 2024 Penang has 12 month sheets + data from Patient List + assert len(df) == 174 + assert len(df.columns) > 0 # Should have columns (exact count varies before cleaning) + + def test_extract_has_metadata_columns(self, tracker_2024_penang): + """Should add metadata columns (tracker_year, tracker_month, sheet_name, file_name).""" + skip_if_missing(tracker_2024_penang) + + df = read_all_patient_sheets(tracker_2024_penang) + + assert "tracker_year" in df.columns + assert "tracker_month" in df.columns + assert "sheet_name" in df.columns + assert "file_name" in df.columns + assert "clinic_id" in df.columns + + def test_extract_year_is_correct(self, tracker_2024_penang): + """Should extract year 2024 from sheet names.""" + skip_if_missing(tracker_2024_penang) + + df = read_all_patient_sheets(tracker_2024_penang) + + # All rows should have year 2024 + assert df["tracker_year"].unique().to_list() == [2024] + + def test_extract_has_12_months(self, tracker_2024_penang): + """Should process 12 month sheets (Jan-Dec 2024).""" + skip_if_missing(tracker_2024_penang) + + df = read_all_patient_sheets(tracker_2024_penang) + + months = sorted(df["tracker_month"].unique().to_list()) + expected_months = list(range(1, 13)) # 1-12 + assert months == expected_months + + def test_extract_clinic_id(self, tracker_2024_penang): + """Should extract clinic_id from parent directory.""" + skip_if_missing(tracker_2024_penang) + + df = read_all_patient_sheets(tracker_2024_penang) + + # Parent directory is PNG + assert df["clinic_id"].unique().to_list() == ["PNG"] + + +class TestExtract2023Sibu: + """Test extraction on 2023 Sibu tracker (edge case with duplicate columns).""" + + def test_extract_handles_duplicates(self, tracker_2023_sibu): + """Should handle duplicate column mappings (complication_screening).""" + skip_if_missing(tracker_2023_sibu) + + # This should not raise DuplicateError + df = read_all_patient_sheets(tracker_2023_sibu) + + assert len(df) == 14 # 2023 Sibu has 14 total records + assert len(df.columns) > 0 + + def test_extract_year_2023(self, tracker_2023_sibu): + """Should extract year 2023.""" + skip_if_missing(tracker_2023_sibu) + + df = read_all_patient_sheets(tracker_2023_sibu) + + assert df["tracker_year"].unique().to_list() == [2023] + + def test_extract_months_sep_to_dec(self, tracker_2023_sibu): + """Should extract months Sep-Dec 2023.""" + skip_if_missing(tracker_2023_sibu) + + df = read_all_patient_sheets(tracker_2023_sibu) + + months = sorted(df["tracker_month"].unique().to_list()) + expected_months = [9, 10, 11, 12] # Sep-Dec + assert months == expected_months + + +class TestExtract2022PenangLegacy: + """Test extraction on 2022 Penang (legacy format without Annual sheet).""" + + def test_extract_legacy_format(self, tracker_2022_penang): + """Should handle legacy format without Annual sheet.""" + skip_if_missing(tracker_2022_penang) + + df = read_all_patient_sheets(tracker_2022_penang) + + assert len(df) == 156 # 2022 Penang has 156 total records + assert len(df.columns) > 0 + + def test_extract_legacy_has_patient_list(self, tracker_2022_penang): + """Should still process Patient List sheet in legacy format.""" + skip_if_missing(tracker_2022_penang) + + df = read_all_patient_sheets(tracker_2022_penang) + + # Should have data from Patient List (static columns like dob, province) + # Check if we have any of the Patient List specific columns + assert "dob" in df.columns or "province" in df.columns + + def test_extract_legacy_year_2022(self, tracker_2022_penang): + """Should extract year 2022.""" + skip_if_missing(tracker_2022_penang) + + df = read_all_patient_sheets(tracker_2022_penang) + + assert df["tracker_year"].unique().to_list() == [2022] + + +class TestExtractProduct2024Penang: + """Test product extraction on 2024 Penang tracker (same file as patient's).""" + + def test_extract_total_rows(self, tracker_2024_penang): + """Should extract all raw product records from all sheets.""" + skip_if_missing(tracker_2024_penang) + + df = read_all_product_sheets(tracker_2024_penang) + + assert len(df) == 696 + assert len(df.columns) > 0 + + def test_extract_has_metadata_columns(self, tracker_2024_penang): + """Should add metadata columns (product_table_year/month, sheet name, file/clinic).""" + skip_if_missing(tracker_2024_penang) + + df = read_all_product_sheets(tracker_2024_penang) + + assert "product_table_year" in df.columns + assert "product_table_month" in df.columns + assert "product_sheet_name" in df.columns + assert "file_name" in df.columns + assert "clinic_id" in df.columns + + def test_extract_year_is_correct(self, tracker_2024_penang): + """Should extract year 2024 from sheet names.""" + skip_if_missing(tracker_2024_penang) + + df = read_all_product_sheets(tracker_2024_penang) + + assert df["product_table_year"].unique().to_list() == [2024.0] + + def test_extract_has_12_months(self, tracker_2024_penang): + """Should process 12 month sheets (Jan-Dec 2024).""" + skip_if_missing(tracker_2024_penang) + + df = read_all_product_sheets(tracker_2024_penang) + + months = sorted(df["product_table_month"].unique().to_list()) + assert months == [f"{m:02d}" for m in range(1, 13)] + + def test_extract_clinic_id(self, tracker_2024_penang): + """Should extract clinic_id from parent directory.""" + skip_if_missing(tracker_2024_penang) + + df = read_all_product_sheets(tracker_2024_penang) + + assert df["clinic_id"].unique().to_list() == ["PNG"] + + +class TestExtractProduct2023Sibu: + """Test product extraction on 2023 Sibu tracker (mirrors patient's edge case).""" + + def test_extract_row_count(self, tracker_2023_sibu): + skip_if_missing(tracker_2023_sibu) + + df = read_all_product_sheets(tracker_2023_sibu) + + assert len(df) == 316 + + def test_extract_months_sep_to_dec(self, tracker_2023_sibu): + skip_if_missing(tracker_2023_sibu) + + df = read_all_product_sheets(tracker_2023_sibu) + + months = sorted(df["product_table_month"].unique().to_list()) + assert months == ["09", "10", "11", "12"] + + +class TestExtractProductWideFormatColumns: + """2020 Mandalay tracker triggers handle_wide_format_columns (step 1.4a).""" + + def test_extract_succeeds_and_expands_recipients(self, tracker_2020_mandalay): + skip_if_missing(tracker_2020_mandalay) + + df = read_all_product_sheets(tracker_2020_mandalay) + + assert len(df) == 1139 + assert "product_released_to" in df.columns + # Recipient names, not the raw "Total"/"per person" totals, should + # populate product_released_to once wide columns are expanded. + released_to = df["product_released_to"].drop_nulls().to_list() + assert any(v.startswith("MM_MD") for v in released_to) + + +class TestExtractProductWideFormatCells: + """2018 Mandalay tracker triggers handle_wide_format_cells (step 1.4b).""" + + def test_extract_succeeds_and_splits_comma_cells(self, tracker_2018_mandalay): + skip_if_missing(tracker_2018_mandalay) + + df = read_all_product_sheets(tracker_2018_mandalay) + + assert len(df) == 575 + assert "product_released_to" in df.columns diff --git a/tests/test_migration/test_compare.py b/tests/test_migration/test_compare.py new file mode 100644 index 0000000..9c6960e --- /dev/null +++ b/tests/test_migration/test_compare.py @@ -0,0 +1,3568 @@ +"""Tests for the R-vs-Python output comparison (ticket 15).""" + +import datetime +import functools + +import polars as pl + +from a4d.clean.schema_product import get_product_data_schema +from a4d.config import settings +from a4d.migration.compare import ( + BUDDHIST_ERA_CONVERSION_CLASSIFIERS, + DERIVED_RUNNING_TOTAL_CLASSIFIERS, + EXCEL_FORMULA_ERROR_CLASSIFIERS, + GROUP_INVARIANT_PRODUCT_COLUMNS, + PATIENT_AGE_FROM_BARE_YEAR_CLASSIFIERS, + PATIENT_BARE_YEAR_CLASSIFIERS, + PATIENT_BEYOND_TRACKER_YEAR_CLASSIFIERS, + PATIENT_BUDDHIST_ERA_CLASSIFIERS, + PATIENT_CLINICAL_NOTE_CLASSIFIERS, + PATIENT_DIAGNOSIS_AGE_CLASSIFIERS, + PATIENT_FBG_TEXT_CLASSIFIERS, + PATIENT_GLUCOSE_CASCADE_CLASSIFIERS, + PATIENT_INSULIN_DRUG_NAME_CLASSIFIERS, + PATIENT_INSULIN_SUBTYPE_CLASSIFIERS, + PATIENT_INSULIN_TOTAL_UNITS_CLASSIFIERS, + PATIENT_JOIN_SUFFIX_COLLISION_CLASSIFIERS, + PATIENT_MERGED_SUBVALUE_TRIM_CLASSIFIERS, + PATIENT_MONTH_NAME_TRUNCATED_CLASSIFIERS, + PATIENT_NA_UNITE_PADDING_CLASSIFIERS, + PATIENT_NON_LATIN_HEADER_CLASSIFIERS, + PATIENT_R_EXTRACTION_GAP_CLASSIFIERS, + PATIENT_RICHTEXT_SPACE_CLASSIFIERS, + PATIENT_SCREENING_SELECTION_CLASSIFIERS, + PATIENT_UNICODE_SANITIZER_CLASSIFIERS, + PATIENT_UNREADABLE_MONTH_CLASSIFIERS, + PATIENT_UNTRIMMED_VALIDATION_CLASSIFIERS, + PATIENT_YMD_FIRST_CLASSIFIERS, + PRODUCT_CATEGORY_CLASSIFIERS, + PRODUCT_ENTRY_DATE_CLASSIFIERS, + PRODUCT_ROW_ORDER_CLASSIFIERS, + PYTHON_CANONICAL_LABEL_CLASSIFIERS, + R_DATE_ERROR_SENTINEL_CLASSIFIERS, + R_NUMERIC_ERROR_SENTINEL_CLASSIFIERS, + ROW_ORDINAL_COL, + SENTINEL_DATE, + STRAY_DATE_CLASSIFIERS, + STRAY_DATE_ZEROED_CLASSIFIERS, + WIDE_FORMAT_FRAGMENT_CLASSIFIERS, + CategoricalOverlap, + CellMismatch, + ColumnsResult, + Delta, + DirectoryComparison, + FileComparison, + IdOverlapResult, + RowKeyOverlap, + ShapeResult, + TotalsMismatch, + _is_python_declines_unreconstructable_date, + _is_python_glucose_unit_corrected, + _is_python_recovers_glucose_r_rejected, + _is_python_rejects_out_of_range_hba1c, + _is_r_age_not_derived_without_dob, + _is_r_static_join_misses_respelled_id, + add_row_ordinal, + align_duplicate_rows, + build_mismatch_rows, + build_summary_rows, + classify, + classify_column_divergence, + compare_categorical_overlap, + compare_cells, + compare_columns, + compare_directory, + compare_id_overlap, + compare_row_key_overlap, + compare_shape, + compare_totals, + compute_deltas, + load_glucose_unit_swaps, + normalize_boolean_literal_column, + normalize_date_column, + normalize_numeric_column, + normalize_whitespace_column, + numeric_normalize_targets, + snapshot_from_summary, + static_join_missed_ids, + string_numeric_normalize_targets, + summarize_directory, + whitespace_normalize_targets, +) + + +class TestNormalizeDateColumn: + def test_parses_excel_serial_string(self): + df = pl.DataFrame({"product_entry_date": ["42872.0"]}) + + result = normalize_date_column(df, "product_entry_date") + + assert result["product_entry_date"].to_list() == [datetime.date(2017, 5, 17)] + + def test_parses_excel_serial_string_without_decimal(self): + df = pl.DataFrame({"product_entry_date": ["46023"]}) + + result = normalize_date_column(df, "product_entry_date") + + assert result["product_entry_date"].to_list() == [datetime.date(2026, 1, 1)] + + def test_parses_iso_datetime_string_to_the_same_date(self): + r_df = pl.DataFrame({"product_entry_date": ["42872.0"]}) + py_df = pl.DataFrame({"product_entry_date": ["2017-05-17 00:00:00"]}) + + r_result = normalize_date_column(r_df, "product_entry_date") + py_result = normalize_date_column(py_df, "product_entry_date") + + assert r_result["product_entry_date"].to_list() == py_result["product_entry_date"].to_list() + + def test_passes_through_null(self): + df = pl.DataFrame({"product_entry_date": [None]}, schema={"product_entry_date": pl.Utf8}) + + result = normalize_date_column(df, "product_entry_date") + + assert result["product_entry_date"].to_list() == [None] + + def test_falls_back_to_sentinel_date_on_unparseable_text(self): + df = pl.DataFrame({"product_entry_date": ["04-May-20026"]}) + + result = normalize_date_column(df, "product_entry_date") + + assert result["product_entry_date"].to_list() == [SENTINEL_DATE] + + def test_is_a_no_op_when_the_column_is_absent(self): + df = pl.DataFrame({"other": [1, 2]}) + + result = normalize_date_column(df, "product_entry_date") + + assert result.columns == ["other"] + + +class TestNormalizeNumericColumn: + def test_parses_numeric_strings_to_float(self): + df = pl.DataFrame({"product_balance": ["9.300000000000001"]}) + + result = normalize_numeric_column(df, "product_balance") + + assert result["product_balance"].to_list() == [9.300000000000001] + + def test_same_float_from_different_string_representations_compare_equal(self): + r_df = pl.DataFrame({"product_balance": ["9.300000000000001"]}) + py_df = pl.DataFrame({"product_balance": ["9.3"]}) + + r_result = normalize_numeric_column(r_df, "product_balance") + py_result = normalize_numeric_column(py_df, "product_balance") + + assert r_result["product_balance"][0] == py_result["product_balance"][0] + + def test_passes_through_null(self): + df = pl.DataFrame({"product_balance": [None]}, schema={"product_balance": pl.Utf8}) + + result = normalize_numeric_column(df, "product_balance") + + assert result["product_balance"].to_list() == [None] + + def test_leaves_non_numeric_text_unchanged(self): + df = pl.DataFrame({"product_units_received": ["START BALANCE"]}) + + result = normalize_numeric_column(df, "product_units_received") + + assert result["product_units_received"].to_list() == ["START BALANCE"] + + def test_is_a_no_op_when_the_column_is_absent(self): + df = pl.DataFrame({"other": [1, 2]}) + + result = normalize_numeric_column(df, "product_balance") + + assert result.columns == ["other"] + + +class TestNumericNormalizeTargets: + def test_includes_a_patient_list_join_suffix_copy(self): + df = pl.DataFrame({"patient_id": ["A"], "fbg_baseline_mg.static": ["93.6"]}) + + assert "fbg_baseline_mg.static" in numeric_normalize_targets(df, exclude=["patient_id"]) + + def test_includes_a_column_the_cleaned_schema_types_as_string(self): + df = pl.DataFrame( + {"patient_id": ["A"], "complication_screening_kidney_test_value": ["9.2"]} + ) + + targets = numeric_normalize_targets(df, exclude=["patient_id"]) + + assert "complication_screening_kidney_test_value" in targets + + def test_excludes_the_row_alignment_key_columns(self): + df = pl.DataFrame({"patient_id": ["A"], "sheet_name": ["Jul24"], "weight": ["31.5"]}) + + targets = numeric_normalize_targets(df, exclude=["patient_id", "sheet_name"]) + + assert targets == ["weight"] + + def test_excludes_synthetic_join_helper_columns(self): + df = pl.DataFrame({"__row_ordinal": [1], "weight": ["31.5"]}) + + assert numeric_normalize_targets(df, exclude=[]) == ["weight"] + + +class TestStringNumericNormalizeTargets: + """Ticket 54: the cleaned stage's own numeric-normalization scope.""" + + def test_includes_a_string_typed_measurement_column(self): + """The cleaned schema types a screening measurement as a string + because the same column can also hold "normal", so cleaning never + casts it and R's float-to-string rounding survives into the output.""" + df = pl.DataFrame( + {"patient_id": ["A"], "complication_screening_kidney_test_value": ["9.2"]} + ) + + targets = string_numeric_normalize_targets(df, exclude=["patient_id"]) + + assert targets == ["complication_screening_kidney_test_value"] + + def test_excludes_an_already_numeric_column(self): + """Both sides hold a real float there, so there is nothing to reparse.""" + df = pl.DataFrame({"patient_id": ["A"], "weight": [31.5]}) + + assert string_numeric_normalize_targets(df, exclude=["patient_id"]) == [] + + def test_excludes_the_row_alignment_key_columns(self): + df = pl.DataFrame({"patient_id": ["A"], "sheet_name": ["Jul24"], "hba1c_updated": ["9.2"]}) + + targets = string_numeric_normalize_targets(df, exclude=["patient_id", "sheet_name"]) + + assert targets == ["hba1c_updated"] + + def test_excludes_synthetic_join_helper_columns(self): + df = pl.DataFrame({"__key_patient_id": ["A"], "hba1c_updated": ["9.2"]}) + + assert string_numeric_normalize_targets(df, exclude=[]) == ["hba1c_updated"] + + +class TestWhitespaceNormalizeTargets: + def test_includes_a_raw_only_column_absent_from_the_cleaned_schema(self): + df = pl.DataFrame({"patient_id": ["A"], "dm_complications": ["Kidney \r\nDamage"]}) + + assert "dm_complications" in whitespace_normalize_targets(df, exclude=["patient_id"]) + + def test_includes_a_column_the_cleaned_schema_types_as_numeric(self): + df = pl.DataFrame({"patient_id": ["A"], "insulin_injections": [" "]}) + + assert "insulin_injections" in whitespace_normalize_targets(df, exclude=["patient_id"]) + + def test_excludes_non_string_columns(self): + df = pl.DataFrame({"weight": [31.5], "edu_occ": ["college "]}) + + assert whitespace_normalize_targets(df, exclude=[]) == ["edu_occ"] + + def test_excludes_synthetic_join_helper_columns(self): + df = pl.DataFrame({"__key_patient_id": ["A"], "edu_occ": ["college "]}) + + assert whitespace_normalize_targets(df, exclude=[]) == ["edu_occ"] + + +class TestNormalizeWhitespaceColumn: + def test_strips_leading_and_trailing_whitespace(self): + df = pl.DataFrame({"product": ["WIZ Twist Lancets (25s)\t"]}) + + result = normalize_whitespace_column(df, "product") + + assert result["product"].to_list() == ["WIZ Twist Lancets (25s)"] + + def test_r_and_python_representations_compare_equal_after_strip(self): + r_df = pl.DataFrame({"product_remarks": ["Remote"]}) + py_df = pl.DataFrame({"product_remarks": ["Remote "]}) + + r_result = normalize_whitespace_column(r_df, "product_remarks") + py_result = normalize_whitespace_column(py_df, "product_remarks") + + assert r_result["product_remarks"].to_list() == py_result["product_remarks"].to_list() + + def test_passes_through_null(self): + df = pl.DataFrame({"product": [None]}, schema={"product": pl.Utf8}) + + result = normalize_whitespace_column(df, "product") + + assert result["product"].to_list() == [None] + + def test_normalizes_crlf_line_endings_to_lf(self): + r_df = pl.DataFrame({"product": ["Accu-Chek Instant Forward \r\nGlucometer Set"]}) + py_df = pl.DataFrame({"product": ["Accu-Chek Instant Forward \nGlucometer Set"]}) + + r_result = normalize_whitespace_column(r_df, "product") + py_result = normalize_whitespace_column(py_df, "product") + + assert r_result["product"].to_list() == py_result["product"].to_list() + + def test_whitespace_only_cell_normalizes_to_null(self): + col = "product_received_from" + r_df = pl.DataFrame({col: [None]}, schema={col: pl.Utf8}) + py_df = pl.DataFrame({col: [" "]}) + + r_result = normalize_whitespace_column(r_df, col) + py_result = normalize_whitespace_column(py_df, col) + + assert r_result[col].to_list() == py_result[col].to_list() + + def test_is_a_no_op_when_the_column_is_absent(self): + df = pl.DataFrame({"other": [1, 2]}) + + result = normalize_whitespace_column(df, "product") + + assert result.columns == ["other"] + + +class TestNormalizeBooleanLiteralColumn: + def test_r_and_python_boolean_spellings_compare_equal(self): + r_df = pl.DataFrame({"clinic_visit": ["FALSE", "TRUE"]}) + py_df = pl.DataFrame({"clinic_visit": ["False", "True"]}) + + r_result = normalize_boolean_literal_column(r_df, "clinic_visit") + py_result = normalize_boolean_literal_column(py_df, "clinic_visit") + + assert r_result["clinic_visit"].to_list() == py_result["clinic_visit"].to_list() + + def test_leaves_ordinary_text_untouched(self): + df = pl.DataFrame({"observations": ["Nil", "false alarm reported"]}) + + result = normalize_boolean_literal_column(df, "observations") + + assert result["observations"].to_list() == ["Nil", "false alarm reported"] + + def test_passes_through_null(self): + df = pl.DataFrame({"clinic_visit": [None]}, schema={"clinic_visit": pl.Utf8}) + + result = normalize_boolean_literal_column(df, "clinic_visit") + + assert result["clinic_visit"].to_list() == [None] + + def test_is_a_no_op_when_the_column_is_absent(self): + df = pl.DataFrame({"other": [1, 2]}) + + assert normalize_boolean_literal_column(df, "clinic_visit").columns == ["other"] + + +class TestAddRowOrdinal: + def test_assigns_sequential_ordinal_within_group(self): + df = pl.DataFrame({"clinic_id": ["A", "A", "A", "B"], "value": [10, 20, 30, 40]}) + + result, key_cols = add_row_ordinal(df, ["clinic_id"]) + + assert key_cols == ["__key_clinic_id", "__row_ordinal"] + assert result["__row_ordinal"].to_list() == [0, 1, 2, 0] + + def test_resets_ordinal_per_distinct_group_combination(self): + df = pl.DataFrame( + { + "clinic_id": ["A", "A", "B", "B", "B"], + "sheet": ["Jan", "Feb", "Jan", "Jan", "Feb"], + } + ) + + result, key_cols = add_row_ordinal(df, ["clinic_id", "sheet"]) + + assert key_cols == ["__key_clinic_id", "__key_sheet", "__row_ordinal"] + assert result["__row_ordinal"].to_list() == [0, 0, 0, 1, 0] + + def test_strips_whitespace_for_the_join_key_but_not_the_original_column(self): + r_df = pl.DataFrame({"sheet": ["May19 "]}) + py_df = pl.DataFrame({"sheet": ["May19"]}) + + r_result, key_cols = add_row_ordinal(r_df, ["sheet"]) + py_result, _ = add_row_ordinal(py_df, ["sheet"]) + + assert r_result["__key_sheet"].to_list() == py_result["__key_sheet"].to_list() + assert r_result["sheet"].to_list() == ["May19 "] + assert py_result["sheet"].to_list() == ["May19"] + + def test_leaves_non_string_group_columns_unstripped(self): + df = pl.DataFrame({"year": [2019, 2019, 2020]}) + + result, key_cols = add_row_ordinal(df, ["year"]) + + assert result["__key_year"].to_list() == [2019, 2019, 2020] + assert result["__row_ordinal"].to_list() == [0, 1, 0] + + def test_keys_as_null_when_group_column_is_missing_entirely(self): + df = pl.DataFrame(schema={}) + + result, key_cols = add_row_ordinal(df, ["clinic_id"]) + + assert key_cols == ["__key_clinic_id", "__row_ordinal"] + assert result.height == 0 + + +class TestPatientOccurrenceOrdinalKey: + """Ticket 45: patient keeps `patient_id` + `sheet_name` as its identity key + and uses `add_row_ordinal` only to break ties within it. + + Product groups by `(clinic_id, product_sheet_name)`, so its ordinal is a + purely positional key. Patient's natural key is sound in all but a handful + of files, so grouping by the identity key itself keeps the identity check + everywhere and disambiguates only where a source sheet lists the same + patient twice. + """ + + PATIENT_KEY = ["patient_id", "sheet_name"] + + def _spliced_sheet(self) -> tuple[pl.DataFrame, pl.DataFrame]: + """`2024_Vietnam National Children`'s `Jul24` shape: two lists spliced + together, so a patient appears twice on one sheet with different data. + """ + rows = { + "patient_id": ["VN001", "VN002", "VN001"], + "sheet_name": ["Jul24", "Jul24", "Jul24"], + "weight": [30.0, 41.0, 55.0], + } + return pl.DataFrame(rows), pl.DataFrame(rows) + + def test_duplicate_identity_key_fans_out_without_the_ordinal(self): + r_df, py_df = self._spliced_sheet() + + mismatches = compare_cells(r_df, py_df, key_cols=self.PATIENT_KEY) + + # VN001's two rows pair every way round; the two cross pairings compare + # unrelated physical rows and report a difference that does not exist. + assert sorted((m.r_value, m.py_value) for m in mismatches) == [(30.0, 55.0), (55.0, 30.0)] + + def test_ordinal_within_the_identity_key_pairs_duplicates_one_to_one(self): + r_df, py_df = self._spliced_sheet() + + r_keyed, key_cols = add_row_ordinal(r_df, self.PATIENT_KEY) + py_keyed, _ = add_row_ordinal(py_df, self.PATIENT_KEY) + + assert key_cols == ["__key_patient_id", "__key_sheet_name", ROW_ORDINAL_COL] + assert compare_row_key_overlap(r_keyed, py_keyed, key_cols) == RowKeyOverlap( + matched=3, r_unmatched=0, py_unmatched=0 + ) + assert compare_cells(r_keyed, py_keyed, key_cols=key_cols) == [] + + def test_unique_keys_are_unaffected_by_the_ordinal(self): + r_df = pl.DataFrame( + {"patient_id": ["A", "B"], "sheet_name": ["Jan24", "Jan24"], "weight": [10.0, 20.0]} + ) + py_df = pl.DataFrame( + {"patient_id": ["B", "A"], "sheet_name": ["Jan24", "Jan24"], "weight": [20.0, 11.0]} + ) + + r_keyed, key_cols = add_row_ordinal(r_df, self.PATIENT_KEY) + py_keyed, _ = add_row_ordinal(py_df, self.PATIENT_KEY) + + # Every ordinal is 0, so pairing is by identity exactly as before -- and + # row order across the two sides still does not matter. + assert r_keyed[ROW_ORDINAL_COL].to_list() == [0, 0] + assert [ + (m.key["__key_patient_id"], m.r_value, m.py_value) + for m in compare_cells(r_keyed, py_keyed, key_cols=key_cols) + ] == [("A", 10.0, 11.0)] + + def test_duplicates_pair_by_content_not_by_position(self): + """Within one patient-and-sheet group the row order carries no meaning, + and the cleaned stage demonstrably reorders: R and Python emit + `VN_VC007`'s two `Jul24` rows the other way round. Pairing by position + there compares the wrong two copies. + """ + r_df = pl.DataFrame( + { + "patient_id": ["VN007", "VN007"], + "sheet_name": ["Jul24", "Jul24"], + "weight": [30.0, 55.0], + "hba1c": [7.0, 9.0], + } + ) + py_df = pl.DataFrame( + { + "patient_id": ["VN007", "VN007"], + "sheet_name": ["Jul24", "Jul24"], + "weight": [55.0, 30.0], + "hba1c": [9.0, 7.1], + } + ) + + r_keyed, py_keyed, key_cols = align_duplicate_rows(r_df, py_df, self.PATIENT_KEY) + + # Only the genuine hba1c difference survives; the swap does not become + # four false mismatches. + assert [ + (m.column, m.r_value, m.py_value) + for m in compare_cells(r_keyed, py_keyed, key_cols=key_cols) + ] == [("hba1c", 7.0, 7.1)] + + def test_position_is_kept_when_no_pairing_beats_it(self): + """The tie-break only moves a row when the alternative is strictly + better, so a group whose rows genuinely both changed is not permuted + into looking like agreement.""" + r_df = pl.DataFrame( + {"patient_id": ["A", "A"], "sheet_name": ["Jan24"] * 2, "weight": [30.0, 40.0]} + ) + py_df = pl.DataFrame( + {"patient_id": ["A", "A"], "sheet_name": ["Jan24"] * 2, "weight": [31.0, 41.0]} + ) + + r_keyed, py_keyed, key_cols = align_duplicate_rows(r_df, py_df, self.PATIENT_KEY) + + assert sorted( + (m.r_value, m.py_value) for m in compare_cells(r_keyed, py_keyed, key_cols=key_cols) + ) == [ + (30.0, 31.0), + (40.0, 41.0), + ] + + def test_unique_keys_and_uneven_groups_fall_back_to_position(self): + r_df = pl.DataFrame( + {"patient_id": ["A", "A", "B"], "sheet_name": ["Jan24"] * 3, "weight": [1.0, 2.0, 3.0]} + ) + py_df = pl.DataFrame( + {"patient_id": ["A", "B"], "sheet_name": ["Jan24"] * 2, "weight": [1.0, 3.0]} + ) + + r_keyed, py_keyed, key_cols = align_duplicate_rows(r_df, py_df, self.PATIENT_KEY) + + overlap = compare_row_key_overlap(r_keyed, py_keyed, key_cols) + + assert (overlap.matched, overlap.r_unmatched, overlap.py_unmatched) == (2, 1, 0) + assert overlap.r_only_keys == ((("A", "Jan24", 1), 1),) + + def test_a_patient_on_another_sheet_never_pairs(self): + """Sheet is the data's monthly granularity, so it stays part of the key: + the same patient's June and July rows must not be compared.""" + r_df = pl.DataFrame({"patient_id": ["A"], "sheet_name": ["Jun24"], "weight": [10.0]}) + py_df = pl.DataFrame({"patient_id": ["A"], "sheet_name": ["Jul24"], "weight": [10.0]}) + + r_keyed, key_cols = add_row_ordinal(r_df, self.PATIENT_KEY) + py_keyed, _ = add_row_ordinal(py_df, self.PATIENT_KEY) + + overlap = compare_row_key_overlap(r_keyed, py_keyed, key_cols) + + assert (overlap.matched, overlap.r_unmatched, overlap.py_unmatched) == (0, 1, 1) + assert overlap.r_only_keys == ((("A", "Jun24", 0), 1),) + assert overlap.py_only_keys == ((("A", "Jul24", 0), 1),) + + +class TestCompareShape: + def test_match_when_row_counts_equal(self): + r_df = pl.DataFrame({"a": [1, 2, 3]}) + py_df = pl.DataFrame({"a": [1, 2, 3]}) + + result = compare_shape(r_df, py_df) + + assert result == ShapeResult(r_rows=3, py_rows=3, match=True) + + def test_no_match_when_row_counts_differ(self): + r_df = pl.DataFrame({"a": [1, 2, 3]}) + py_df = pl.DataFrame({"a": [1, 2]}) + + result = compare_shape(r_df, py_df) + + assert result == ShapeResult(r_rows=3, py_rows=2, match=False) + + +class TestCompareTotals: + def test_no_mismatches_when_sums_match(self): + r_df = pl.DataFrame({"balance": [1.0, 2.0, 3.0]}) + py_df = pl.DataFrame({"balance": [3.0, 2.0, 1.0]}) + + assert compare_totals(r_df, py_df, numeric_cols=["balance"]) == [] + + def test_flags_column_whose_sum_differs(self): + r_df = pl.DataFrame({"balance": [1.0, 2.0, 3.0]}) + py_df = pl.DataFrame({"balance": [1.0, 2.0, 30.0]}) + + result = compare_totals(r_df, py_df, numeric_cols=["balance"]) + + assert result == [TotalsMismatch(column="balance", r_total=6.0, py_total=33.0)] + + def test_ignores_negligible_float_drift(self): + r_df = pl.DataFrame({"balance": [0.1, 0.2]}) + py_df = pl.DataFrame({"balance": [0.1 + 1e-12, 0.2]}) + + assert compare_totals(r_df, py_df, numeric_cols=["balance"]) == [] + + def test_skips_a_column_missing_from_either_side(self): + r_df = pl.DataFrame({"balance": [1.0]}) + py_df = pl.DataFrame({"balance": [1.0]}) + + result = compare_totals(r_df, py_df, numeric_cols=["balance", "not_in_either"]) + + assert result == [] + + def test_skips_a_column_present_only_on_one_side(self): + r_df = pl.DataFrame({"balance": [1.0], "only_r_has_this": [5.0]}) + py_df = pl.DataFrame({"balance": [1.0]}) + + result = compare_totals(r_df, py_df, numeric_cols=["balance", "only_r_has_this"]) + + assert result == [] + + +class TestCompareColumns: + def test_no_diff_when_columns_and_dtypes_match(self): + r_df = pl.DataFrame({"a": [1], "b": ["x"]}) + py_df = pl.DataFrame({"a": [1], "b": ["x"]}) + + result = compare_columns(r_df, py_df) + + assert result == ColumnsResult(only_in_r=[], only_in_py=[], dtype_mismatches=[]) + + def test_flags_columns_present_in_only_one_side(self): + r_df = pl.DataFrame({"a": [1], "b": ["x"]}) + py_df = pl.DataFrame({"a": [1], "c": ["y"]}) + + result = compare_columns(r_df, py_df) + + assert result.only_in_r == ["b"] + assert result.only_in_py == ["c"] + + def test_flags_dtype_mismatch_on_common_column(self): + r_df = pl.DataFrame({"a": [1]}, schema={"a": pl.Int64}) + py_df = pl.DataFrame({"a": [1.0]}, schema={"a": pl.Float64}) + + result = compare_columns(r_df, py_df) + + assert result.dtype_mismatches == [("a", pl.Int64, pl.Float64)] + + +class TestCompareIdOverlap: + def test_no_diff_when_ids_match(self): + r_df = pl.DataFrame({"patient_id": ["a", "b", "c"]}) + py_df = pl.DataFrame({"patient_id": ["c", "b", "a"]}) + + result = compare_id_overlap(r_df, py_df, id_col="patient_id") + + assert result == IdOverlapResult(only_in_r=[], only_in_py=[], common_count=3) + + def test_flags_ids_present_on_only_one_side(self): + r_df = pl.DataFrame({"patient_id": ["a", "b", "dropped"]}) + py_df = pl.DataFrame({"patient_id": ["a", "b", "new"]}) + + result = compare_id_overlap(r_df, py_df, id_col="patient_id") + + assert result == IdOverlapResult(only_in_r=["dropped"], only_in_py=["new"], common_count=2) + + def test_ignores_nulls(self): + r_df = pl.DataFrame({"product": ["a", None]}, schema={"product": pl.Utf8}) + py_df = pl.DataFrame({"product": ["a", None]}, schema={"product": pl.Utf8}) + + result = compare_id_overlap(r_df, py_df, id_col="product") + + assert result == IdOverlapResult(only_in_r=[], only_in_py=[], common_count=1) + + def test_deduplicates_repeated_ids(self): + r_df = pl.DataFrame({"product": ["a", "a", "b"]}) + py_df = pl.DataFrame({"product": ["a"]}) + + result = compare_id_overlap(r_df, py_df, id_col="product") + + assert result == IdOverlapResult(only_in_r=["b"], only_in_py=[], common_count=1) + + +class TestCompareCategoricalOverlap: + def test_no_mismatches_when_all_categorical_columns_agree(self): + r_df = pl.DataFrame({"category": ["a", "b"], "status": ["x", "y"]}) + py_df = pl.DataFrame({"category": ["b", "a"], "status": ["y", "x"]}) + + result = compare_categorical_overlap(r_df, py_df, categorical_cols=["category", "status"]) + + assert result == [] + + def test_flags_only_columns_with_unseen_values(self): + r_df = pl.DataFrame({"category": ["a", "b"], "status": ["x", "y"]}) + py_df = pl.DataFrame({"category": ["a", "typo"], "status": ["x", "y"]}) + + result = compare_categorical_overlap(r_df, py_df, categorical_cols=["category", "status"]) + + assert result == [ + CategoricalOverlap(column="category", only_in_r=["b"], only_in_py=["typo"]) + ] + + def test_ignores_columns_not_listed(self): + r_df = pl.DataFrame({"category": ["a"], "unrelated": ["z"]}) + py_df = pl.DataFrame({"category": ["a"], "unrelated": ["different"]}) + + result = compare_categorical_overlap(r_df, py_df, categorical_cols=["category"]) + + assert result == [] + + def test_skips_a_column_missing_from_either_side(self): + r_df = pl.DataFrame({"category": ["a"]}) + py_df = pl.DataFrame({"category": ["a"]}) + + result = compare_categorical_overlap( + r_df, py_df, categorical_cols=["category", "not_in_either"] + ) + + assert result == [] + + def test_skips_a_column_present_only_on_one_side(self): + r_df = pl.DataFrame({"category": ["a"], "only_r_has_this": ["x"]}) + py_df = pl.DataFrame({"category": ["a"]}) + + result = compare_categorical_overlap( + r_df, py_df, categorical_cols=["category", "only_r_has_this"] + ) + + assert result == [] + + +class TestCompareRowKeyOverlap: + def test_all_rows_matched_when_keys_align_1to1(self): + r_df = pl.DataFrame({"id": [1, 2], "sheet": ["a", "b"]}) + py_df = pl.DataFrame({"id": [2, 1], "sheet": ["b", "a"]}) + + result = compare_row_key_overlap(r_df, py_df, key_cols=["id", "sheet"]) + + assert result == RowKeyOverlap(matched=2, r_unmatched=0, py_unmatched=0) + + def test_flags_rows_whose_key_never_appears_on_the_other_side(self): + r_df = pl.DataFrame({"id": [1, 2]}) + py_df = pl.DataFrame({"id": [1, 3]}) + + result = compare_row_key_overlap(r_df, py_df, key_cols=["id"]) + + assert result == RowKeyOverlap( + matched=1, + r_unmatched=1, + py_unmatched=1, + r_only_keys=(((2,), 1),), + py_only_keys=(((3,), 1),), + ) + + def test_fan_out_excess_counts_as_unmatched_on_the_heavier_side(self): + r_df = pl.DataFrame({"id": [1, 1, 1]}) + py_df = pl.DataFrame({"id": [1]}) + + result = compare_row_key_overlap(r_df, py_df, key_cols=["id"]) + + assert result == RowKeyOverlap( + matched=1, r_unmatched=2, py_unmatched=0, r_only_keys=(((1,), 2),) + ) + + def test_names_the_unmatched_keys_not_just_their_count(self): + """A count alone cannot be triaged -- ticket 47's four recovered rows + were only findable by querying the parquets by hand.""" + r_df = pl.DataFrame({"patient_id": ["KH_NPH02"] * 2, "sheet_name": ["Sep23", "Sep23"]}) + py_df = pl.DataFrame({"patient_id": ["KH_NP026", "KH_NP027"], "sheet_name": ["Sep23"] * 2}) + + result = compare_row_key_overlap(r_df, py_df, key_cols=["patient_id", "sheet_name"]) + + assert result.r_only_keys == ((("KH_NPH02", "Sep23"), 2),) + assert result.py_only_keys == ( + (("KH_NP026", "Sep23"), 1), + (("KH_NP027", "Sep23"), 1), + ) + + def test_unmatched_keys_are_sorted_so_reports_are_stable(self): + r_df = pl.DataFrame({"id": ["c", "a", "b"]}) + py_df = pl.DataFrame({"id": ["z"]}) + + result = compare_row_key_overlap(r_df, py_df, key_cols=["id"]) + + assert [key for key, _ in result.r_only_keys] == [("a",), ("b",), ("c",)] + + +class TestCompareCells: + def test_no_mismatches_when_matched_rows_equal(self): + r_df = pl.DataFrame({"id": [1, 2], "balance": [10.0, 20.0]}) + py_df = pl.DataFrame({"id": [2, 1], "balance": [20.0, 10.0]}) + + assert compare_cells(r_df, py_df, key_cols=["id"]) == [] + + def test_flags_value_mismatch_on_common_column(self): + r_df = pl.DataFrame({"id": [1], "balance": [10.0]}) + py_df = pl.DataFrame({"id": [1], "balance": [99.0]}) + + result = compare_cells(r_df, py_df, key_cols=["id"]) + + assert result == [ + CellMismatch(key={"id": 1}, column="balance", r_value=10.0, py_value=99.0) + ] + + def test_null_status_mismatch_is_flagged(self): + r_df = pl.DataFrame({"id": [1], "note": [None]}, schema={"id": pl.Int64, "note": pl.Utf8}) + py_df = pl.DataFrame({"id": [1], "note": ["x"]}) + + result = compare_cells(r_df, py_df, key_cols=["id"]) + + # r_column_empty_in_file (ticket 63) is True by construction here: the + # fixture's only R row is null, so R populates the column nowhere. + assert result == [ + CellMismatch( + key={"id": 1}, + column="note", + r_value=None, + py_value="x", + r_column_empty_in_file=True, + ) + ] + + def test_ignores_float_drift_within_tolerance(self): + r_df = pl.DataFrame({"id": [1], "balance": [0.1]}) + py_df = pl.DataFrame({"id": [1], "balance": [0.1 + 1e-12]}) + + assert compare_cells(r_df, py_df, key_cols=["id"]) == [] + + def test_only_diffs_rows_matched_on_key(self): + r_df = pl.DataFrame({"id": [1, 2], "balance": [10.0, 999.0]}) + py_df = pl.DataFrame({"id": [1], "balance": [10.0]}) + + assert compare_cells(r_df, py_df, key_cols=["id"]) == [] + + def test_row_order_candidate_false_without_order_group_cols(self): + r_df = pl.DataFrame({"grp": ["a", "a"], "id": [0, 1], "balance": [10.0, 20.0]}) + py_df = pl.DataFrame({"grp": ["a", "a"], "id": [0, 1], "balance": [20.0, 10.0]}) + + result = compare_cells(r_df, py_df, key_cols=["grp", "id"]) + + assert all(m.row_order_candidate is False for m in result) + + def test_row_order_candidate_true_when_value_present_elsewhere_in_group(self): + # R's row 0 (10.0) matches Python's row 1 (10.0) within the same group -- + # a shifted-row pattern, not a genuine content divergence. + r_df = pl.DataFrame({"grp": ["a", "a"], "id": [0, 1], "balance": [10.0, 999.0]}) + py_df = pl.DataFrame({"grp": ["a", "a"], "id": [0, 1], "balance": [20.0, 10.0]}) + + result = compare_cells(r_df, py_df, key_cols=["grp", "id"], order_group_cols=["grp"]) + + by_id = {m.key["id"]: m for m in result} + assert by_id[0].row_order_candidate is True + assert by_id[1].row_order_candidate is False + + def test_era_shift_row_order_candidate_when_the_converted_date_sits_elsewhere(self): + """Ticket 61: converting a Buddhist-era date moves the row, because R + sorts an unconverted 2565 date to the end of its group while Python + sorts the same day chronologically. R's value is absent from Python's + group, so plain value membership cannot see it -- the shifted value + can.""" + r_df = pl.DataFrame( + { + "grp": ["a", "a"], + "id": [0, 1], + "d": [datetime.date(2565, 12, 24), datetime.date(2022, 12, 28)], + } + ) + py_df = pl.DataFrame( + { + "grp": ["a", "a"], + "id": [0, 1], + "d": [datetime.date(2022, 12, 28), datetime.date(2022, 12, 24)], + } + ) + + result = compare_cells(r_df, py_df, key_cols=["grp", "id"], order_group_cols=["grp"]) + + by_id = {m.key["id"]: m for m in result} + assert by_id[0].era_shift_row_order_candidate is True + assert by_id[0].row_order_candidate is False + assert ( + classify(by_id[0], BUDDHIST_ERA_CONVERSION_CLASSIFIERS) + == "buddhist_era_conversion_row_order" + ) + + def test_era_shift_row_order_candidate_false_for_a_gregorian_value(self): + r_df = pl.DataFrame( + { + "grp": ["a", "a"], + "id": [0, 1], + "d": [datetime.date(2022, 12, 24), datetime.date(2022, 12, 28)], + } + ) + py_df = pl.DataFrame( + { + "grp": ["a", "a"], + "id": [0, 1], + "d": [datetime.date(2022, 12, 28), datetime.date(2022, 12, 24)], + } + ) + + result = compare_cells(r_df, py_df, key_cols=["grp", "id"], order_group_cols=["grp"]) + + assert all(m.era_shift_row_order_candidate is False for m in result) + + def test_group_endpoint_matches_when_last_positional_row_agrees(self): + """Ticket 36: a derived running total diverges on every intermediate + row under a re-sort but still lands on the same closing value.""" + r_df = pl.DataFrame( + {"grp": ["a"] * 3, ROW_ORDINAL_COL: [0, 1, 2], "balance": [10.0, 8.0, 5.0]} + ) + py_df = pl.DataFrame( + {"grp": ["a"] * 3, ROW_ORDINAL_COL: [0, 1, 2], "balance": [10.0, 7.0, 5.0]} + ) + + result = compare_cells( + r_df, py_df, key_cols=["grp", ROW_ORDINAL_COL], order_group_cols=["grp"] + ) + + assert [m.group_endpoint_matches for m in result] == [True] + + def test_group_endpoint_matches_false_when_last_positional_row_differs(self): + r_df = pl.DataFrame( + {"grp": ["a"] * 3, ROW_ORDINAL_COL: [0, 1, 2], "balance": [10.0, 8.0, 43572.0]} + ) + py_df = pl.DataFrame( + {"grp": ["a"] * 3, ROW_ORDINAL_COL: [0, 1, 2], "balance": [10.0, 7.0, 5.0]} + ) + + result = compare_cells( + r_df, py_df, key_cols=["grp", ROW_ORDINAL_COL], order_group_cols=["grp"] + ) + + assert all(m.group_endpoint_matches is False for m in result) + + def test_group_endpoint_matches_false_without_a_positional_ordinal(self): + r_df = pl.DataFrame({"grp": ["a"], "id": [0], "balance": [10.0]}) + py_df = pl.DataFrame({"grp": ["a"], "id": [0], "balance": [999.0]}) + + result = compare_cells(r_df, py_df, key_cols=["grp", "id"], order_group_cols=["grp"]) + + assert all(m.group_endpoint_matches is False for m in result) + + def test_row_order_candidate_false_when_value_genuinely_absent_from_group(self): + r_df = pl.DataFrame({"grp": ["a"], "id": [0], "balance": [10.0]}) + py_df = pl.DataFrame({"grp": ["a"], "id": [0], "balance": [999.0]}) + + result = compare_cells(r_df, py_df, key_cols=["grp", "id"], order_group_cols=["grp"]) + + assert result == [ + CellMismatch( + key={"grp": "a", "id": 0}, + column="balance", + r_value=10.0, + py_value=999.0, + row_order_candidate=False, + ) + ] + + def test_row_order_candidate_scoped_to_own_group(self): + r_df = pl.DataFrame({"grp": ["a", "b"], "id": [0, 0], "balance": [10.0, 10.0]}) + py_df = pl.DataFrame({"grp": ["a", "b"], "id": [0, 0], "balance": [999.0, 10.0]}) + + result = compare_cells(r_df, py_df, key_cols=["grp", "id"], order_group_cols=["grp"]) + + assert result == [ + CellMismatch( + key={"grp": "a", "id": 0}, + column="balance", + r_value=10.0, + py_value=999.0, + row_order_candidate=False, + ) + ] + + +def _mismatch(r_value, py_value, column="product_entry_date"): + return CellMismatch(key={"id": 1}, column=column, r_value=r_value, py_value=py_value) + + +class TestClassify: + def test_python_sentinel_r_extraction_gap_when_r_is_null_and_python_is_sentinel(self): + """Ticket 32 traced this to source: Sarawak's 2023 tracker really does + hold December *2024* dates in its Dec23 sheet, so Python's + beyond-tracker-year guard fires while R's own extraction gap leaves + null. Two already-named mechanisms meeting, not a third one.""" + mismatch = _mismatch(r_value=None, py_value=SENTINEL_DATE) + + assert ( + classify(mismatch, PRODUCT_ENTRY_DATE_CLASSIFIERS) == "python_sentinel_r_extraction_gap" + ) + + def test_r_value_missing_when_r_is_null_and_python_parsed_a_real_date(self): + mismatch = _mismatch(r_value=None, py_value=datetime.date(2021, 3, 10)) + + assert classify(mismatch, PRODUCT_ENTRY_DATE_CLASSIFIERS) == "r_value_missing" + + def test_r_value_missing_declines_a_date_outside_the_tracker_window(self): + """r_value_missing claims R lost a value Python read correctly. It may + not say that about a cell where Python itself published 2009 in a 2019 + tracker -- the pipeline logged that one and preserved it deliberately + (ticket 32).""" + mismatch = CellMismatch( + key={"id": 1}, + column="product_entry_date", + r_value=None, + py_value=datetime.date(2009, 12, 4), + tracker_year=2019, + ) + + assert ( + classify(mismatch, PRODUCT_ENTRY_DATE_CLASSIFIERS) + == "python_out_of_window_date_preserved" + ) + + def test_r_value_missing_still_fires_inside_the_tracker_window(self): + mismatch = CellMismatch( + key={"id": 1}, + column="product_entry_date", + r_value=None, + py_value=datetime.date(2019, 12, 4), + tracker_year=2019, + ) + + assert classify(mismatch, PRODUCT_ENTRY_DATE_CLASSIFIERS) == "r_value_missing" + + def test_row_order_divergence_beats_a_coincidental_one_day_gap(self): + """The retired `off_by_one_day` cause claimed a mechanism it never had. + Its ten cleaned-stage rows were consecutive daily entries knocked one + position out of step by R's null-date sort fallback (ticket 32).""" + mismatch = CellMismatch( + key={"id": 1}, + column="product_entry_date", + r_value=datetime.date(2022, 5, 9), + py_value=datetime.date(2022, 5, 10), + row_order_candidate=True, + ) + + registry = PRODUCT_ENTRY_DATE_CLASSIFIERS | PRODUCT_ROW_ORDER_CLASSIFIERS + assert classify(mismatch, registry) == "row_order_divergence" + + def test_excel_1900_leap_serial_for_a_stray_serial_below_sixty_one(self): + """Excel's phantom 1900-02-29: openpyxl reads serial 25 as 1900-01-25, + plain epoch arithmetic as 1900-01-24. Same source integer, and stray + summary-row residue either way.""" + mismatch = _mismatch( + r_value=datetime.date(1900, 1, 24), py_value=datetime.date(1900, 1, 25) + ) + + assert classify(mismatch, PRODUCT_ENTRY_DATE_CLASSIFIERS) == "excel_1900_leap_serial" + + def test_ordinary_one_day_gap_is_now_unclassified(self): + """Without a mechanism behind it, a one-day gap is an open question -- + which is what a real serial-origin bug would look like.""" + mismatch = _mismatch( + r_value=datetime.date(2020, 11, 28), py_value=datetime.date(2020, 11, 29) + ) + + assert classify(mismatch, PRODUCT_ENTRY_DATE_CLASSIFIERS) == "unclassified" + + def test_python_absurd_serial_when_r_holds_the_normalization_sentinel(self): + """The retired `ce_typo` fired on `r_value.year > 2100`, which caught + normalize_date_column's own 9999 sentinel and said nothing about + Python publishing 5567-08-19 from Excel serial 1,339,576.""" + mismatch = _mismatch(r_value=SENTINEL_DATE, py_value=datetime.date(5567, 8, 19)) + + assert classify(mismatch, PRODUCT_ENTRY_DATE_CLASSIFIERS) == "python_absurd_excel_serial" + + def test_unclassified_when_no_classifier_matches(self): + mismatch = _mismatch(r_value=datetime.date(2021, 1, 1), py_value=datetime.date(2021, 6, 1)) + + assert classify(mismatch, PRODUCT_ENTRY_DATE_CLASSIFIERS) == "unclassified" + + def test_r_category_lookup_miss_when_r_is_null_and_python_has_a_category(self): + mismatch = _mismatch(r_value=None, py_value="INSULIN", column="product_category") + + assert classify(mismatch, PRODUCT_CATEGORY_CLASSIFIERS) == "r_category_lookup_miss" + + def test_category_unclassified_when_no_classifier_matches(self): + mismatch = _mismatch(r_value="INSULIN", py_value="TEST STRIPS", column="product_category") + + assert classify(mismatch, PRODUCT_CATEGORY_CLASSIFIERS) == "unclassified" + + def test_row_order_divergence_when_flagged_as_candidate(self): + mismatch = CellMismatch( + key={"id": 1}, + column="product_balance", + r_value=10.0, + py_value=999.0, + row_order_candidate=True, + ) + + assert classify(mismatch, PRODUCT_ROW_ORDER_CLASSIFIERS) == "row_order_divergence" + + def test_row_order_unclassified_when_not_flagged_as_candidate(self): + mismatch = CellMismatch( + key={"id": 1}, + column="product_balance", + r_value=10.0, + py_value=999.0, + row_order_candidate=False, + ) + + assert classify(mismatch, PRODUCT_ROW_ORDER_CLASSIFIERS) == "unclassified" + + def test_r_extraction_gap_when_r_is_null_and_python_has_a_date(self): + mismatch = _mismatch( + r_value=None, py_value=datetime.date(2025, 12, 1), column="recruitment_date" + ) + + assert classify(mismatch, PATIENT_R_EXTRACTION_GAP_CLASSIFIERS) == "r_extraction_gap" + + def test_screening_selection_dropped_when_r_keeps_only_the_first_selection(self): + """Ticket 50, 2021 NPH Dec21 KH_NP006: AD84/AE84 are two "(Select)" + sub-columns of one merged block; R's own raw output parks the second in + ``complicationscreeningselect1`` and maps only the first.""" + mismatch = _mismatch( + r_value="Dilated Eye Examination", + py_value="Dilated Eye Examination,Foot Examination (Nerves)", + column="complication_screening", + ) + + assert ( + classify(mismatch, PATIENT_SCREENING_SELECTION_CLASSIFIERS) + == "r_duplicate_header_selection_dropped" + ) + + def test_screening_selection_unclassified_when_the_kept_selection_differs(self): + """R's value must be Python's own first selection, not merely shorter.""" + mismatch = _mismatch( + r_value="Foot Examination (Nerves)", + py_value="Dilated Eye Examination,Foot Examination (Nerves)", + column="complication_screening", + ) + + assert classify(mismatch, PATIENT_SCREENING_SELECTION_CLASSIFIERS) == "unclassified" + + def test_screening_selection_unclassified_when_python_adds_nothing(self): + mismatch = _mismatch( + r_value="Dilated Eye Examination", + py_value="Dilated Eye Examination", + column="complication_screening", + ) + + assert classify(mismatch, PATIENT_SCREENING_SELECTION_CLASSIFIERS) == "unclassified" + + def test_recruitment_date_unclassified_when_both_sides_null(self): + mismatch = _mismatch(r_value=None, py_value=None, column="recruitment_date") + + assert classify(mismatch, PATIENT_R_EXTRACTION_GAP_CLASSIFIERS) == "unclassified" + + def test_r_validator_rejects_multivalue_when_r_is_undefined(self): + mismatch = _mismatch( + r_value="Undefined", py_value="Rapid-acting,Long-acting", column="insulin_subtype" + ) + + assert ( + classify(mismatch, PATIENT_INSULIN_SUBTYPE_CLASSIFIERS) + == "r_validator_rejects_multivalue" + ) + + def test_insulin_subtype_unclassified_when_r_is_not_the_undefined_sentinel(self): + mismatch = _mismatch( + r_value="Long-acting", py_value="Rapid-acting", column="insulin_subtype" + ) + + assert classify(mismatch, PATIENT_INSULIN_SUBTYPE_CLASSIFIERS) == "unclassified" + + def test_stray_date_typed_cell_when_r_serial_matches_python_full_datetime(self): + mismatch = _mismatch( + r_value="43566", py_value="2019-04-11 00:00:00", column="product_units_received" + ) + + assert classify(mismatch, STRAY_DATE_CLASSIFIERS) == "openpyxl_date_typed_stray_cell" + + def test_stray_date_typed_cell_when_r_serial_hits_the_1900_leap_year_bug(self): + # Excel (and openpyxl, matching it) treats 1900 as a leap year that + # never existed -- serial 59 resolves to 1900-02-28, not 1900-02-27 + # (ticket 24, a real flagged row). + mismatch = _mismatch( + r_value="59", py_value="1900-02-28 00:00:00", column="product_units_received" + ) + + assert classify(mismatch, STRAY_DATE_CLASSIFIERS) == "openpyxl_date_typed_stray_cell" + + def test_stray_date_typed_cell_when_r_serial_matches_python_bare_time(self): + mismatch = _mismatch(r_value="0", py_value="00:00:00", column="product_units_received") + + assert classify(mismatch, STRAY_DATE_CLASSIFIERS) == "openpyxl_date_typed_stray_cell" + + def test_stray_date_typed_cell_unclassified_when_values_are_genuinely_different(self): + mismatch = _mismatch(r_value="5", py_value="7", column="product_units_received") + + assert classify(mismatch, STRAY_DATE_CLASSIFIERS) == "unclassified" + + def test_stray_date_typed_cell_unclassified_when_r_value_is_not_numeric(self): + mismatch = _mismatch( + r_value="START BALANCE", py_value="2019-04-11 00:00:00", column="product_units_received" + ) + + assert classify(mismatch, STRAY_DATE_CLASSIFIERS) == "unclassified" + + def test_derived_running_total_row_order_when_group_endpoint_matches(self): + """Ticket 36: product_balance is recomputed as a running total, so it + cannot travel with its row under a re-sort -- the order-independent + evidence is the group's closing balance, not value membership.""" + mismatch = CellMismatch( + key={"clinic_id": "MHS"}, + column="product_balance", + r_value=310.0, + py_value=306.0, + group_endpoint_matches=True, + ) + + assert ( + classify(mismatch, DERIVED_RUNNING_TOTAL_CLASSIFIERS) + == "derived_running_total_row_order" + ) + + def test_derived_running_total_unclassified_when_group_endpoint_differs(self): + """A group whose closing balance disagrees is a real divergence, not a + re-sort -- it must not be swept up by the same label.""" + mismatch = CellMismatch( + key={"clinic_id": "SBY"}, + column="product_balance", + r_value=43572.0, + py_value=6.0, + group_endpoint_matches=False, + ) + + assert classify(mismatch, DERIVED_RUNNING_TOTAL_CLASSIFIERS) == "unclassified" + + def test_python_future_date_sentinel_when_python_holds_the_error_sentinel(self): + mismatch = _mismatch(r_value=datetime.date(2029, 8, 29), py_value=SENTINEL_DATE) + + assert classify(mismatch, PRODUCT_ENTRY_DATE_CLASSIFIERS) == "python_future_date_sentinel" + + def test_python_future_date_sentinel_unclassified_when_python_holds_a_real_date(self): + mismatch = _mismatch( + r_value=datetime.date(2029, 8, 29), py_value=datetime.date(2023, 8, 29) + ) + + assert classify(mismatch, PRODUCT_ENTRY_DATE_CLASSIFIERS) != "python_future_date_sentinel" + + def test_summary_residue_nulled_when_r_holds_a_tiny_serial_and_python_is_null(self): + mismatch = _mismatch(r_value=datetime.date(1900, 1, 30), py_value=None) + + assert classify(mismatch, PRODUCT_ENTRY_DATE_CLASSIFIERS) == "summary_residue_nulled" + + def test_summary_residue_nulled_unclassified_when_r_holds_a_real_date(self): + mismatch = _mismatch(r_value=datetime.date(2023, 8, 29), py_value=None) + + assert classify(mismatch, PRODUCT_ENTRY_DATE_CLASSIFIERS) == "unclassified" + + def test_stray_date_zeroed_when_r_holds_a_serial_and_python_cleaned_it_to_zero(self): + """Ticket 36: the cleaned-stage face of STRAY_DATE_CLASSIFIERS' cause.""" + mismatch = _mismatch(r_value=43708.0, py_value=0.0, column="product_units_received") + + assert classify(mismatch, STRAY_DATE_ZEROED_CLASSIFIERS) == "stray_date_zeroed" + + def test_stray_date_zeroed_unclassified_when_python_is_a_real_quantity(self): + mismatch = _mismatch(r_value=43708.0, py_value=12.0, column="product_units_received") + + assert classify(mismatch, STRAY_DATE_ZEROED_CLASSIFIERS) == "unclassified" + + def test_stray_date_zeroed_unclassified_when_r_value_is_a_plausible_quantity(self): + """A real unit count must never be mistaken for a date serial.""" + mismatch = _mismatch(r_value=120.0, py_value=0.0, column="product_units_received") + + assert classify(mismatch, STRAY_DATE_ZEROED_CLASSIFIERS) == "unclassified" + + def test_stray_date_zeroed_unclassified_when_r_value_is_not_numeric(self): + mismatch = _mismatch(r_value="START BALANCE", py_value=0.0, column="product_units_received") + + assert classify(mismatch, STRAY_DATE_ZEROED_CLASSIFIERS) == "unclassified" + + def test_wide_format_fragment_truncated_when_python_value_extends_r_value(self): + mismatch = _mismatch( + r_value="2(Error", py_value="2(Error-1)", column="product_units_released" + ) + + assert ( + classify(mismatch, WIDE_FORMAT_FRAGMENT_CLASSIFIERS) == "wide_format_fragment_truncated" + ) + + def test_wide_format_fragment_truncated_unclassified_when_values_are_equal(self): + mismatch = _mismatch(r_value="7", py_value="7", column="product_units_released") + + assert classify(mismatch, WIDE_FORMAT_FRAGMENT_CLASSIFIERS) == "unclassified" + + def test_wide_format_fragment_truncated_unclassified_when_not_a_prefix_relation(self): + mismatch = _mismatch( + r_value="2 MM_MD023", py_value="3 MM_MD099", column="product_units_released" + ) + + assert classify(mismatch, WIDE_FORMAT_FRAGMENT_CLASSIFIERS) == "unclassified" + + def test_excel_formula_error_when_r_holds_the_error_string_and_python_is_null(self): + mismatch = _mismatch(r_value="#DIV/0!", py_value=None, column="bmi") + + assert classify(mismatch, EXCEL_FORMULA_ERROR_CLASSIFIERS) == "excel_formula_error" + + def test_excel_formula_error_when_python_holds_the_error_string_and_r_is_null(self): + """readxl nulls an error cell in a numerically-guessed column; Python keeps it.""" + mismatch = _mismatch(r_value=None, py_value="#DIV/0!", column="bmi") + + assert classify(mismatch, EXCEL_FORMULA_ERROR_CLASSIFIERS) == "excel_formula_error" + + def test_excel_formula_error_matches_every_seeded_excel_error_string(self): + for error_string in ("#DIV/0!", "#VALUE!", "#NUM!", "#N/A", "#REF!", "#NAME?", "#NULL!"): + mismatch = _mismatch(r_value=error_string, py_value=None, column="bmi") + + assert classify(mismatch, EXCEL_FORMULA_ERROR_CLASSIFIERS) == "excel_formula_error" + + def test_excel_formula_error_unclassified_when_both_sides_have_a_value(self): + mismatch = _mismatch(r_value="#DIV/0!", py_value=0.0, column="bmi") + + assert classify(mismatch, EXCEL_FORMULA_ERROR_CLASSIFIERS) == "unclassified" + + def test_excel_formula_error_unclassified_when_neither_side_is_an_error_string(self): + mismatch = _mismatch(r_value="9.3", py_value=None, column="bmi") + + assert classify(mismatch, EXCEL_FORMULA_ERROR_CLASSIFIERS) == "unclassified" + + def test_r_na_unite_padding_when_r_pads_absent_sub_columns_with_na(self): + mismatch = _mismatch( + r_value="JAN,NA,NA,NA,NA", py_value="JAN", column="complication_screening" + ) + + assert classify(mismatch, PATIENT_NA_UNITE_PADDING_CLASSIFIERS) == "r_na_unite_padding" + + def test_r_na_unite_padding_when_every_sub_column_is_absent(self): + mismatch = _mismatch( + r_value="NA,NA,NA,NA,NA", py_value=None, column="complication_screening" + ) + + assert classify(mismatch, PATIENT_NA_UNITE_PADDING_CLASSIFIERS) == "r_na_unite_padding" + + def test_r_na_unite_padding_when_several_sub_columns_are_populated(self): + mismatch = _mismatch( + r_value="NA,JAN,NA,FEB,NA", py_value="JAN,FEB", column="complication_screening" + ) + + assert classify(mismatch, PATIENT_NA_UNITE_PADDING_CLASSIFIERS) == "r_na_unite_padding" + + def test_r_na_unite_padding_when_python_trimmed_a_sub_value_before_merging(self): + """Ticket 50, 2020 Kantha Bopha Dec20 KH_KB040: the padding cause and + ticket 46's verified ``python_trims_merged_subvalue`` stack in one cell, + so neither explained it while the parts were compared untrimmed.""" + mismatch = _mismatch( + r_value="collection in January ,NA", + py_value="collection in January", + column="observations", + ) + + assert classify(mismatch, PATIENT_NA_UNITE_PADDING_CLASSIFIERS) == "r_na_unite_padding" + + def test_r_na_unite_padding_unclassified_when_the_surviving_values_differ(self): + """Stripping R's padding must still leave a real difference unexplained.""" + mismatch = _mismatch(r_value="JAN,NA,NA", py_value="FEB", column="complication_screening") + + assert classify(mismatch, PATIENT_NA_UNITE_PADDING_CLASSIFIERS) == "unclassified" + + def test_r_na_unite_padding_unclassified_when_r_carries_no_na_token(self): + mismatch = _mismatch(r_value="JAN", py_value="JAN,FEB", column="complication_screening") + + assert classify(mismatch, PATIENT_NA_UNITE_PADDING_CLASSIFIERS) == "unclassified" + + def test_r_richtext_space_dropped_on_a_value_split_across_formatting_runs(self): + """2017 Yangon Feb17!L: the cell is a rich-text shared string whose + middle run is the space alone.""" + mismatch = _mismatch( + r_value="8.8(20.9.16)", py_value="8.8 (20.9.16)", column="hba1c_updated" + ) + + assert classify(mismatch, PATIENT_RICHTEXT_SPACE_CLASSIFIERS) == "r_drops_richtext_space" + + def test_r_richtext_space_dropped_in_free_text(self): + mismatch = _mismatch( + r_value="Unable tocontact since June'21", + py_value="Unable to contact since June'21", + column="observations", + ) + + assert classify(mismatch, PATIENT_RICHTEXT_SPACE_CLASSIFIERS) == "r_drops_richtext_space" + + def test_r_richtext_space_unclassified_when_r_is_the_side_with_more_spaces(self): + """The opposite direction is Python's own sub-value trimming, not this.""" + mismatch = _mismatch(r_value="Normal ,No", py_value="Normal,No", column="observations") + + assert classify(mismatch, PATIENT_RICHTEXT_SPACE_CLASSIFIERS) == "unclassified" + + def test_r_richtext_space_unclassified_when_the_text_itself_differs(self): + mismatch = _mismatch( + r_value="8.8(20.9.16)", py_value="9.8 (20.9.16)", column="hba1c_updated" + ) + + assert classify(mismatch, PATIENT_RICHTEXT_SPACE_CLASSIFIERS) == "unclassified" + + def test_python_trims_merged_subvalue_when_r_keeps_the_source_trailing_space(self): + mismatch = _mismatch( + r_value="Normal ,Insulin", py_value="Normal,Insulin", column="observations" + ) + + assert ( + classify(mismatch, PATIENT_MERGED_SUBVALUE_TRIM_CLASSIFIERS) + == "python_trims_merged_subvalue" + ) + + def test_python_trims_merged_subvalue_unclassified_without_a_merge_separator(self): + mismatch = _mismatch(r_value="Normal ", py_value="Normal", column="observations") + + assert classify(mismatch, PATIENT_MERGED_SUBVALUE_TRIM_CLASSIFIERS) == "unclassified" + + def test_buddhist_era_typo_when_r_is_sentinel_and_python_has_an_implausible_be_year(self): + mismatch = _mismatch( + r_value=SENTINEL_DATE, + py_value=datetime.date(2569, 6, 17), + column="hba1c_updated_date", + ) + + assert classify(mismatch, PATIENT_BUDDHIST_ERA_CLASSIFIERS) == "buddhist_era_typo" + + def test_buddhist_era_typo_unclassified_when_python_year_is_a_plausible_gregorian_year(self): + mismatch = _mismatch( + r_value=SENTINEL_DATE, + py_value=datetime.date(2026, 6, 17), + column="hba1c_updated_date", + ) + + assert classify(mismatch, PATIENT_BUDDHIST_ERA_CLASSIFIERS) == "unclassified" + + def test_buddhist_era_typo_when_python_is_the_sentinelling_side(self): + """blood_pressure_updated reverses the direction at the cleaned stage: + R carries the BE year through, Python's future-date guard sentinels it. + """ + mismatch = _mismatch( + r_value=datetime.date(2569, 4, 24), + py_value=SENTINEL_DATE, + column="blood_pressure_updated", + ) + + assert classify(mismatch, PATIENT_BUDDHIST_ERA_CLASSIFIERS) == "buddhist_era_typo" + + def test_python_buddhist_era_converted_when_python_holds_the_gregorian_year(self): + """Ticket 61: the cleaned stage now shifts a BE date to Gregorian, so R + keeps the calendar the clinic wrote and Python publishes the same day + 543 years earlier. A deliberate divergence, not a defect on either + side.""" + mismatch = _mismatch( + r_value=datetime.date(2569, 6, 17), + py_value=datetime.date(2026, 6, 17), + column="hba1c_updated_date", + ) + + assert ( + classify(mismatch, PATIENT_BUDDHIST_ERA_CLASSIFIERS) == "python_buddhist_era_converted" + ) + + def test_python_buddhist_era_converted_unclassified_when_the_day_moves_too(self): + """A shift of exactly 543 years is what identifies the conversion. Any + other difference is a second, unexplained divergence riding along, and + naming it here would hide it.""" + mismatch = _mismatch( + r_value=datetime.date(2569, 6, 17), + py_value=datetime.date(2026, 6, 18), + column="hba1c_updated_date", + ) + + assert classify(mismatch, PATIENT_BUDDHIST_ERA_CLASSIFIERS) == "unclassified" + + def test_python_buddhist_era_converted_unclassified_when_r_year_is_gregorian(self): + mismatch = _mismatch( + r_value=datetime.date(2026, 6, 17), + py_value=datetime.date(1483, 6, 17), + column="hba1c_updated_date", + ) + + assert classify(mismatch, PATIENT_BUDDHIST_ERA_CLASSIFIERS) == "unclassified" + + def test_product_entry_date_carries_the_same_conversion_cause(self): + mismatch = _mismatch( + r_value=datetime.date(2567, 11, 11), + py_value=datetime.date(2024, 11, 11), + column="product_entry_date", + ) + + registry = BUDDHIST_ERA_CONVERSION_CLASSIFIERS | PRODUCT_ENTRY_DATE_CLASSIFIERS + + assert classify(mismatch, registry) == "python_buddhist_era_converted" + + def test_buddhist_era_typo_unclassified_when_r_is_not_the_sentinel(self): + mismatch = _mismatch( + r_value=datetime.date(2026, 6, 17), + py_value=datetime.date(2569, 6, 17), + column="hba1c_updated_date", + ) + + assert classify(mismatch, PATIENT_BUDDHIST_ERA_CLASSIFIERS) == "unclassified" + + +class TestCompareDirectory: + def test_matched_file_gets_a_full_comparison(self): + r_frames = {"a.parquet": pl.DataFrame({"id": [1], "balance": [10.0]})} + py_frames = {"a.parquet": pl.DataFrame({"id": [1], "balance": [10.0]})} + + result = compare_directory(r_frames, py_frames, key_cols=["id"], numeric_cols=["balance"]) + + assert result.files == [ + FileComparison( + file_name="a.parquet", + shape=ShapeResult(r_rows=1, py_rows=1, match=True), + totals=[], + columns=ColumnsResult(only_in_r=[], only_in_py=[], dtype_mismatches=[]), + id_overlap=None, + categorical_overlap=[], + row_key_overlap=RowKeyOverlap(matched=1, r_unmatched=0, py_unmatched=0), + cell_mismatches=[], + ) + ] + assert result.only_in_r == [] + assert result.only_in_py == [] + + def test_computes_categorical_overlap_when_categorical_cols_given(self): + r_frames = {"a.parquet": pl.DataFrame({"id": [1], "category": ["a"]})} + py_frames = {"a.parquet": pl.DataFrame({"id": [1], "category": ["typo"]})} + + result = compare_directory( + r_frames, py_frames, key_cols=["id"], categorical_cols=["category"] + ) + + assert result.files[0].categorical_overlap == [ + CategoricalOverlap(column="category", only_in_r=["a"], only_in_py=["typo"]) + ] + + def test_computes_id_overlap_when_id_col_given(self): + r_frames = {"a.parquet": pl.DataFrame({"id": [1, 2], "balance": [10.0, 20.0]})} + py_frames = {"a.parquet": pl.DataFrame({"id": [1, 3], "balance": [10.0, 30.0]})} + + result = compare_directory(r_frames, py_frames, key_cols=["id"], id_col="id") + + assert result.files[0].id_overlap == IdOverlapResult( + only_in_r=[2], only_in_py=[3], common_count=1 + ) + + def test_id_overlap_is_none_when_id_col_not_given(self): + r_frames = {"a.parquet": pl.DataFrame({"id": [1]})} + py_frames = {"a.parquet": pl.DataFrame({"id": [1]})} + + result = compare_directory(r_frames, py_frames, key_cols=["id"]) + + assert result.files[0].id_overlap is None + + def test_flags_totals_mismatch_within_a_matched_file(self): + r_frames = {"a.parquet": pl.DataFrame({"id": [1], "balance": [10.0]})} + py_frames = {"a.parquet": pl.DataFrame({"id": [1], "balance": [999.0]})} + + result = compare_directory(r_frames, py_frames, key_cols=["id"], numeric_cols=["balance"]) + + assert result.files[0].totals == [ + TotalsMismatch(column="balance", r_total=10.0, py_total=999.0) + ] + + def test_flags_cell_mismatch_within_a_matched_file(self): + r_frames = {"a.parquet": pl.DataFrame({"id": [1], "balance": [10.0]})} + py_frames = {"a.parquet": pl.DataFrame({"id": [1], "balance": [99.0]})} + + result = compare_directory(r_frames, py_frames, key_cols=["id"]) + + assert result.files[0].cell_mismatches == [ + CellMismatch(key={"id": 1}, column="balance", r_value=10.0, py_value=99.0) + ] + + def test_tracks_files_present_on_only_one_side(self): + r_frames = { + "a.parquet": pl.DataFrame({"id": [1]}), + "only_r.parquet": pl.DataFrame({"id": [1]}), + } + py_frames = { + "a.parquet": pl.DataFrame({"id": [1]}), + "only_py.parquet": pl.DataFrame({"id": [1]}), + } + + result = compare_directory(r_frames, py_frames, key_cols=["id"]) + + assert result.only_in_r == ["only_r.parquet"] + assert result.only_in_py == ["only_py.parquet"] + assert [f.file_name for f in result.files] == ["a.parquet"] + + +class TestBuildSummaryRows: + def test_reports_per_column_and_per_cause_counts(self): + comparison = DirectoryComparison( + files=[ + FileComparison( + file_name="a.parquet", + shape=ShapeResult(r_rows=2, py_rows=2, match=True), + totals=[], + columns=ColumnsResult(only_in_r=[], only_in_py=[], dtype_mismatches=[]), + id_overlap=None, + categorical_overlap=[], + row_key_overlap=RowKeyOverlap(matched=2, r_unmatched=0, py_unmatched=0), + cell_mismatches=[ + CellMismatch( + key={"id": 1}, + column="product_entry_date", + r_value=None, + py_value=SENTINEL_DATE, + ), + CellMismatch( + key={"id": 2}, + column="other_col", + r_value=1, + py_value=2, + ), + ], + ) + ], + only_in_r=["missing.parquet"], + only_in_py=[], + ) + + summary = build_summary_rows( + comparison, classifiers_by_column={"product_entry_date": PRODUCT_ENTRY_DATE_CLASSIFIERS} + ) + + assert {"column": "product_entry_date", "mismatches": 1} in summary["per_column"] + assert {"column": "other_col", "mismatches": 1} in summary["per_column"] + assert { + "column": "product_entry_date", + "cause": "python_sentinel_r_extraction_gap", + "mismatches": 1, + } in summary["per_cause"] + assert { + "column": "other_col", + "cause": "unclassified", + "mismatches": 1, + } in summary["per_cause"] + assert summary["only_in_r"] == [{"file": "missing.parquet"}] + assert summary["only_in_py"] == [] + + +class TestSnapshotFromSummary: + def test_reduces_to_column_and_cause_count_dicts(self): + summary = { + "per_column": [ + {"column": "product_entry_date", "mismatches": 559}, + {"column": "product_balance", "mismatches": 480}, + ], + "per_cause": [ + {"column": "product_entry_date", "cause": "r_value_missing", "mismatches": 408}, + ], + "only_in_r": [{"file": "x.parquet"}], + "only_in_py": [], + } + + snapshot = snapshot_from_summary(summary) + + assert snapshot == { + "per_column": {"product_entry_date": 559, "product_balance": 480}, + "per_cause": {"product_entry_date|r_value_missing": 408}, + } + + def test_empty_summary_yields_empty_snapshot(self): + summary = {"per_column": [], "per_cause": [], "only_in_r": [], "only_in_py": []} + + assert snapshot_from_summary(summary) == {"per_column": {}, "per_cause": {}} + + +class TestComputeDeltas: + def test_only_reports_changed_keys(self): + previous = {"a": 10, "b": 5, "c": 3} + current = {"a": 10, "b": 2, "c": 3} + + deltas = compute_deltas(previous, current) + + assert deltas == [Delta(key="b", previous=5, current=2)] + + def test_key_missing_from_current_reads_as_resolved_to_zero(self): + previous = {"product_sheet_name": 201} + current = {} + + deltas = compute_deltas(previous, current) + + assert deltas == [Delta(key="product_sheet_name", previous=201, current=0)] + + def test_key_new_in_current_reads_as_regression_from_zero(self): + previous = {} + current = {"product_category": 214} + + deltas = compute_deltas(previous, current) + + assert deltas == [Delta(key="product_category", previous=0, current=214)] + + def test_no_deltas_when_snapshots_identical(self): + snapshot = {"a": 1, "b": 2} + + assert compute_deltas(snapshot, snapshot) == [] + + def test_deltas_sorted_by_key(self): + previous = {"z": 1, "a": 1} + current = {"z": 2, "a": 2} + + deltas = compute_deltas(previous, current) + + assert [d.key for d in deltas] == ["a", "z"] + + +class TestBuildMismatchRows: + def _comparison(self): + return DirectoryComparison( + files=[ + FileComparison( + file_name="a.parquet", + shape=ShapeResult(r_rows=2, py_rows=2, match=True), + totals=[TotalsMismatch(column="balance", r_total=10.0, py_total=99.0)], + columns=ColumnsResult(only_in_r=[], only_in_py=[], dtype_mismatches=[]), + id_overlap=IdOverlapResult(only_in_r=["p1"], only_in_py=["p2"], common_count=1), + categorical_overlap=[ + CategoricalOverlap(column="status", only_in_r=["x"], only_in_py=["y"]) + ], + row_key_overlap=RowKeyOverlap(matched=1, r_unmatched=0, py_unmatched=0), + cell_mismatches=[ + CellMismatch( + key={"id": 1}, + column="product_entry_date", + r_value=None, + py_value=SENTINEL_DATE, + ) + ], + ) + ], + only_in_r=[], + only_in_py=[], + ) + + def test_id_overlap_rows_have_one_row_per_side_per_value(self): + rows = build_mismatch_rows(self._comparison()) + + assert rows["id_overlap"] == [ + {"file": "a.parquet", "side": "R only", "value": "p1"}, + {"file": "a.parquet", "side": "Python only", "value": "p2"}, + ] + + def test_categorical_overlap_rows_have_one_row_per_side_per_value(self): + rows = build_mismatch_rows(self._comparison()) + + assert rows["categorical_overlap"] == [ + {"file": "a.parquet", "column": "status", "side": "R only", "value": "x"}, + {"file": "a.parquet", "column": "status", "side": "Python only", "value": "y"}, + ] + + def test_row_key_overlap_rows_have_one_row_per_file(self): + comparison = DirectoryComparison( + files=[ + FileComparison( + file_name="a.parquet", + shape=ShapeResult(r_rows=3, py_rows=3, match=True), + totals=[], + columns=ColumnsResult(only_in_r=[], only_in_py=[], dtype_mismatches=[]), + id_overlap=None, + categorical_overlap=[], + row_key_overlap=RowKeyOverlap(matched=1, r_unmatched=2, py_unmatched=0), + cell_mismatches=[], + ) + ], + only_in_r=[], + only_in_py=[], + ) + + rows = build_mismatch_rows(comparison) + + assert rows["row_key_overlap"] == [ + {"file": "a.parquet", "matched": 1, "r_unmatched": 2, "py_unmatched": 0} + ] + + def test_totals_rows_include_the_diff(self): + rows = build_mismatch_rows(self._comparison()) + + assert rows["totals"] == [ + { + "file": "a.parquet", + "column": "balance", + "r_total": 10.0, + "py_total": 99.0, + "diff": 89.0, + } + ] + + def test_cell_mismatch_rows_include_classified_cause(self): + rows = build_mismatch_rows( + self._comparison(), + classifiers_by_column={"product_entry_date": PRODUCT_ENTRY_DATE_CLASSIFIERS}, + ) + + assert rows["cell_mismatches"] == [ + { + "file": "a.parquet", + "key": "id=1", + "column": "product_entry_date", + "r_value": None, + "py_value": SENTINEL_DATE, + "cause": "python_sentinel_r_extraction_gap", + } + ] + + def test_cell_mismatch_cause_is_unclassified_without_a_registry(self): + rows = build_mismatch_rows(self._comparison()) + + assert rows["cell_mismatches"][0]["cause"] == "unclassified" + + def test_column_divergence_rows_cover_existence_and_dtype(self): + comparison = DirectoryComparison( + files=[ + FileComparison( + file_name="a.parquet", + shape=ShapeResult(r_rows=1, py_rows=1, match=True), + totals=[], + columns=ColumnsResult( + only_in_r=["r_only"], + only_in_py=["py_only"], + dtype_mismatches=[("shared", pl.Int64, pl.Float64)], + ), + id_overlap=None, + categorical_overlap=[], + row_key_overlap=RowKeyOverlap(matched=1, r_unmatched=0, py_unmatched=0), + cell_mismatches=[], + ) + ], + only_in_r=[], + only_in_py=[], + ) + + rows = build_mismatch_rows(comparison) + + assert rows["column_divergence"] == [ + { + "file": "a.parquet", + "column": "r_only", + "kind": "only in R", + "r_dtype": "", + "py_dtype": "", + "cause": "unclassified", + }, + { + "file": "a.parquet", + "column": "py_only", + "kind": "only in Python", + "r_dtype": "", + "py_dtype": "", + "cause": "unclassified", + }, + { + "file": "a.parquet", + "column": "shared", + "kind": "dtype mismatch", + "r_dtype": "Int64", + "py_dtype": "Float64", + "cause": "unclassified", + }, + ] + + +class TestSummarizeDirectory: + def _comparison(self): + clean = FileComparison( + file_name="clean.parquet", + shape=ShapeResult(r_rows=2, py_rows=2, match=True), + columns=ColumnsResult(only_in_r=[], only_in_py=[], dtype_mismatches=[]), + totals=[], + cell_mismatches=[], + id_overlap=IdOverlapResult(only_in_r=[], only_in_py=[], common_count=2), + categorical_overlap=[], + row_key_overlap=RowKeyOverlap(matched=2, r_unmatched=0, py_unmatched=0), + ) + dirty = FileComparison( + file_name="dirty.parquet", + shape=ShapeResult(r_rows=3, py_rows=2, match=False), + columns=ColumnsResult( + only_in_r=["a"], only_in_py=["b"], dtype_mismatches=[("c", "Int32", "Float64")] + ), + totals=[TotalsMismatch(column="c", r_total=1.0, py_total=2.0)], + cell_mismatches=[CellMismatch(key={"id": 1}, column="c", r_value=1, py_value=2)], + id_overlap=IdOverlapResult(only_in_r=["X"], only_in_py=["Y", "Z"], common_count=1), + categorical_overlap=[ + CategoricalOverlap(column="s", only_in_r=["p"], only_in_py=[]), + ], + row_key_overlap=RowKeyOverlap(matched=1, r_unmatched=2, py_unmatched=1), + ) + return DirectoryComparison(files=[clean, dirty], only_in_r=["gone.parquet"], only_in_py=[]) + + def test_counts_files_affected_and_totals_per_measure(self): + summary = summarize_directory(self._comparison()) + + assert summary.files_compared == 2 + assert summary.shape_mismatch_files == 1 + assert summary.id_divergence == (1, 3) + assert summary.column_divergence == (1, 3) + assert summary.categorical_divergence == (1, 1) + assert summary.totals_divergence == (1, 1) + assert summary.row_key_divergence == (1, 3) + assert summary.cell_divergence == (1, 1) + assert summary.only_in_r == 1 + assert summary.only_in_py == 0 + + def test_all_clean_directory_reports_zero_everywhere(self): + comparison = self._comparison() + clean_only = DirectoryComparison(files=[comparison.files[0]], only_in_r=[], only_in_py=[]) + + summary = summarize_directory(clean_only) + + assert summary.files_compared == 1 + assert summary.shape_mismatch_files == 0 + assert summary.cell_divergence == (0, 0) + assert summary.row_key_divergence == (0, 0) + + +@functools.cache +def _compare_outputs_module(): + """Cached: each exec of the script defines its own RowAlignment class, so + two loads would make `is` comparisons between their members fail.""" + import importlib.util + import sys + from pathlib import Path + + path = Path(__file__).resolve().parents[2] / "scripts" / "compare_outputs.py" + spec = importlib.util.spec_from_file_location("_compare_outputs_under_test", path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class TestStageWiring: + """Ticket 45: which columns a stage groups its row-alignment ordinal by + decides whether the key still checks identity, so it is named per stage + rather than inferred.""" + + @staticmethod + def _stages(): + return {stage.label: stage for stage in _compare_outputs_module().STAGES} + + def test_patient_stages_align_on_identity_with_the_ordinal_as_tiebreak(self): + alignment = _compare_outputs_module().RowAlignment + for label in ("Patient (raw)", "Patient (cleaned)"): + stage = self._stages()[label] + assert stage.ordinal_group_cols == ["patient_id", "sheet_name"] + assert stage.alignment is alignment.IDENTITY + + def test_product_stages_align_positionally_within_a_sheet(self): + alignment = _compare_outputs_module().RowAlignment + for label in ("Product (raw)", "Product (cleaned)"): + stage = self._stages()[label] + assert stage.ordinal_group_cols == ["clinic_id", "product_sheet_name"] + assert stage.alignment is alignment.POSITIONAL + + def test_row_order_divergence_detection_is_product_only(self): + """`compare_cells`'s order_group_cols diagnostic asks whether a + mismatched value appears elsewhere in its group -- meaningful for + product, whose group is a whole sheet ordered differently by R + (ticket 21), and noise for patient, whose group is one patient on one + sheet.""" + stages = self._stages() + + assert [label for label, stage in stages.items() if stage.detect_row_order_divergence] == [ + "Product (raw)", + "Product (cleaned)", + ] + + +class TestClassifiersByColumnWiring: + """Ticket 25: the script's column -> registry map is the single place a + column can be silently under-classified. + + `product_units_released` carried only WIDE_FORMAT_FRAGMENT_CLASSIFIERS (a + raw-stage-only cause, ticket 24) while every sibling column compared + through the same positional row-alignment key also carried + PRODUCT_ROW_ORDER_CLASSIFIERS -- leaving 2,144 cleaned-stage mismatches + reported as `unclassified` even though the cause was already understood. + """ + + @staticmethod + def _classifiers_by_column(): + return _compare_outputs_module().CLASSIFIERS_BY_COLUMN + + def test_every_positional_key_product_column_carries_row_order_classifier(self): + """Product columns are aligned by ordinal position, so any within-group + sort-order divergence surfaces on all of them -- none may omit it. + + Ticket 36: the column list is derived from the product schema rather + than written out. The hand-written version this replaces named seven + columns and omitted `product_entry_date`, which then reported 169 + cleaned-stage mismatches as `unclassified` for exactly the reason this + test exists to prevent. + """ + registries = self._classifiers_by_column() + positional_columns = [ + column + for column in get_product_data_schema() + if column not in GROUP_INVARIANT_PRODUCT_COLUMNS + ] + + missing = [ + column + for column in positional_columns + if not PRODUCT_ROW_ORDER_CLASSIFIERS.keys() <= registries.get(column, {}).keys() + ] + + assert missing == [] + + def test_units_released_row_order_mismatch_is_classified(self): + registries = self._classifiers_by_column() + mismatch = CellMismatch( + key={"clinic_id": "MHS"}, + column="product_units_released", + r_value=8.0, + py_value=4.0, + row_order_candidate=True, + ) + + assert classify(mismatch, registries["product_units_released"]) == "row_order_divergence" + + def test_entry_date_row_order_mismatch_is_classified(self): + """Both sides hold a real date, so `r_value_missing` cannot fire: R's + readxl nulled the *text*-formatted date cells in a mixed-type column, + which drops R back to input-order sorting (ticket 36).""" + registries = self._classifiers_by_column() + mismatch = CellMismatch( + key={"clinic_id": "VNC"}, + column="product_entry_date", + r_value=datetime.date(2022, 7, 4), + py_value=datetime.date(2022, 4, 20), + row_order_candidate=True, + ) + + assert classify(mismatch, registries["product_entry_date"]) == "row_order_divergence" + + def test_units_received_stray_date_zeroed_is_classified(self): + registries = self._classifiers_by_column() + mismatch = CellMismatch( + key={"clinic_id": "SBY"}, + column="product_units_received", + r_value=43708.0, + py_value=0.0, + row_order_candidate=True, + ) + + assert classify(mismatch, registries["product_units_received"]) == "stray_date_zeroed" + + +class TestUntrimmedValidationClassifier: + """Ticket 36: R sentinels a value for whitespace alone; Python recovers it.""" + + def test_classified_when_r_holds_the_character_sentinel(self): + mismatch = CellMismatch(key={"id": 1}, column="sex", r_value="Undefined", py_value="F") + + assert ( + classify(mismatch, PATIENT_UNTRIMMED_VALIDATION_CLASSIFIERS) + == "r_validator_rejects_untrimmed" + ) + + def test_unclassified_when_both_sides_hold_real_but_different_values(self): + mismatch = CellMismatch(key={"id": 1}, column="sex", r_value="M", py_value="F") + + assert classify(mismatch, PATIENT_UNTRIMMED_VALIDATION_CLASSIFIERS) == "unclassified" + + +class TestPatientInsulinTotalUnitsClassifier: + """Ticket 29: R's insulin-column dedup drops TOTAL Insulin Units entirely.""" + + def test_classified_when_r_is_null_and_python_has_the_value(self): + mismatch = CellMismatch( + key={"id": 1}, column="insulin_total_units", r_value=None, py_value=20.0 + ) + + assert classify(mismatch, PATIENT_INSULIN_TOTAL_UNITS_CLASSIFIERS) == "r_insulin_dedup_drop" + + def test_unclassified_when_r_also_holds_a_value(self): + mismatch = CellMismatch( + key={"id": 1}, column="insulin_total_units", r_value=18.0, py_value=20.0 + ) + + assert classify(mismatch, PATIENT_INSULIN_TOTAL_UNITS_CLASSIFIERS) == "unclassified" + + +class TestPatientJoinSuffixCollisionClassifier: + """Ticket 29: R's join suffixes both sides on a column-name collision.""" + + def test_classified_when_r_is_null_and_python_has_the_value(self): + mismatch = CellMismatch( + key={"id": 1}, column="fbg_baseline_mg", r_value=None, py_value=126.0 + ) + + assert ( + classify(mismatch, PATIENT_JOIN_SUFFIX_COLLISION_CLASSIFIERS) + == "r_join_suffix_collision" + ) + + def test_unclassified_when_both_sides_hold_a_value(self): + mismatch = CellMismatch( + key={"id": 1}, column="fbg_baseline_mg", r_value=100.0, py_value=126.0 + ) + + assert classify(mismatch, PATIENT_JOIN_SUFFIX_COLLISION_CLASSIFIERS) == "unclassified" + + +class TestRNumericErrorSentinelClassifier: + """Ticket 29: R sentinels an unusable source cell where Python nulls it.""" + + def test_classified_when_r_holds_the_numeric_sentinel_and_python_is_null(self): + mismatch = CellMismatch( + key={"id": 1}, column="hba1c_baseline", r_value=999999.0, py_value=None + ) + + assert ( + classify(mismatch, R_NUMERIC_ERROR_SENTINEL_CLASSIFIERS) == "r_numeric_error_sentinel" + ) + + def test_classified_when_the_sentinel_arrives_as_text(self): + mismatch = CellMismatch( + key={"id": 1}, column="hba1c_baseline", r_value="999999", py_value=None + ) + + assert ( + classify(mismatch, R_NUMERIC_ERROR_SENTINEL_CLASSIFIERS) == "r_numeric_error_sentinel" + ) + + def test_unclassified_when_python_also_holds_a_value(self): + mismatch = CellMismatch( + key={"id": 1}, column="hba1c_baseline", r_value=999999.0, py_value=7.5 + ) + + assert classify(mismatch, R_NUMERIC_ERROR_SENTINEL_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_r_holds_an_ordinary_value(self): + mismatch = CellMismatch(key={"id": 1}, column="hba1c_baseline", r_value=8.2, py_value=None) + + assert classify(mismatch, R_NUMERIC_ERROR_SENTINEL_CLASSIFIERS) == "unclassified" + + +class TestPythonCanonicalLabelClassifier: + """Ticket 29: Python collapses a retired spelling to its canonical label.""" + + def test_classified_when_r_holds_a_declared_alias_of_pythons_value(self): + mismatch = CellMismatch( + key={"id": 1}, column="status", r_value="Active - Remote", py_value="Active Remote" + ) + + assert classify(mismatch, PYTHON_CANONICAL_LABEL_CLASSIFIERS) == "python_canonical_label" + + def test_matches_case_insensitively(self): + mismatch = CellMismatch( + key={"id": 1}, column="status", r_value="active-remote", py_value="Active Remote" + ) + + assert classify(mismatch, PYTHON_CANONICAL_LABEL_CLASSIFIERS) == "python_canonical_label" + + def test_unclassified_when_the_two_labels_are_genuinely_different_statuses(self): + mismatch = CellMismatch( + key={"id": 1}, column="status", r_value="Discontinued", py_value="Active" + ) + + assert classify(mismatch, PYTHON_CANONICAL_LABEL_CLASSIFIERS) == "unclassified" + + +class TestRDateErrorSentinelClassifier: + """Ticket 38: R stamps its date sentinel on a cell recording an absence.""" + + def test_classified_when_r_holds_the_date_sentinel_and_python_is_null(self): + mismatch = CellMismatch( + key={"id": 1}, + column="hospitalisation_date", + r_value=datetime.date(9999, 9, 9), + py_value=None, + ) + + assert classify(mismatch, R_DATE_ERROR_SENTINEL_CLASSIFIERS) == "r_date_error_sentinel" + + def test_classified_when_the_sentinel_arrives_as_a_datetime(self): + mismatch = CellMismatch( + key={"id": 1}, + column="hospitalisation_date", + r_value=datetime.datetime(9999, 9, 9), + py_value=None, + ) + + assert classify(mismatch, R_DATE_ERROR_SENTINEL_CLASSIFIERS) == "r_date_error_sentinel" + + def test_unclassified_when_python_also_holds_a_date(self): + mismatch = CellMismatch( + key={"id": 1}, + column="hospitalisation_date", + r_value=datetime.date(9999, 9, 9), + py_value=datetime.date(2020, 3, 1), + ) + + assert classify(mismatch, R_DATE_ERROR_SENTINEL_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_r_holds_a_real_date(self): + mismatch = CellMismatch( + key={"id": 1}, + column="hospitalisation_date", + r_value=datetime.date(2020, 3, 1), + py_value=None, + ) + + assert classify(mismatch, R_DATE_ERROR_SENTINEL_CLASSIFIERS) == "unclassified" + + +class TestClassifyColumnDivergence: + """Ticket 30: name the two structural causes of raw-stage column divergence.""" + + def test_r_blank_header_artifact(self): + """R's make.names turns a blank header cell into the string "NA.".""" + assert ( + classify_column_divergence("only in R", "na", set(), set()) == "r_blank_header_artifact" + ) + assert ( + classify_column_divergence("only in R", "na10064", set(), set()) + == "r_blank_header_artifact" + ) + assert ( + classify_column_divergence("only in R", "na1.static", set(), set()) + == "r_blank_header_artifact" + ) + + def test_real_r_column_name_is_not_a_blank_header_artifact(self): + assert classify_column_divergence("only in R", "name", set(), set()) == "unclassified" + assert ( + classify_column_divergence("only in R", "national_id", set(), set()) == "unclassified" + ) + + def test_name_sanitization_only(self): + """Both sides carry the column; only the name spelling differs.""" + r_only = {"instantmeterreceiveddate"} + py_only = {"INSTANT Meter Received Date"} + + assert ( + classify_column_divergence("only in R", "instantmeterreceiveddate", r_only, py_only) + == "name_sanitization_only" + ) + assert ( + classify_column_divergence( + "only in Python", "INSTANT Meter Received Date", r_only, py_only + ) + == "name_sanitization_only" + ) + + def test_unmatched_column_stays_unclassified(self): + assert ( + classify_column_divergence("only in Python", "BGM A4D", {"na"}, {"BGM A4D"}) + == "unclassified" + ) + + def test_blank_header_artifact_wins_over_a_python_name_that_sanitizes_to_it(self): + """A Python column literally named "NA" must not mask R's artifact.""" + assert ( + classify_column_divergence("only in R", "na", {"na"}, {"N/A"}) + == "r_blank_header_artifact" + ) + + +class TestNonLatinHeaderMiss: + def test_flags_r_null_where_python_read_the_column(self): + m = CellMismatch( + key="__key_patient_id=TH_MK001", + column="last_clinic_visit_date", + r_value=None, + py_value="44609", + ) + + assert PATIENT_NON_LATIN_HEADER_CLASSIFIERS["r_non_latin_header_miss"](m) + + def test_does_not_fire_when_both_sides_read_the_column(self): + m = CellMismatch( + key="__key_patient_id=TH_MK001", + column="last_clinic_visit_date", + r_value="44600", + py_value="44609", + ) + + assert not PATIENT_NON_LATIN_HEADER_CLASSIFIERS["r_non_latin_header_miss"](m) + + +class TestPythonReadsBareYear: + """Ticket 52: a bare four-digit year typed into a date cell.""" + + def test_flags_the_year_r_read_as_a_1905_serial(self): + # Sarawak Patient List!G10 holds 2011. R reads it as Excel serial + # 2011 -> 1905-07-03; Python reads the year and resolves it to 1 Jan. + m = CellMismatch( + key="__key_patient_id=MY_SW001", + column="t1d_diagnosis_date", + r_value=datetime.date(1905, 7, 3), + py_value=datetime.date(2011, 1, 1), + ) + + assert classify(m, PATIENT_BARE_YEAR_CLASSIFIERS) == "python_reads_bare_year" + + def test_flags_a_bare_birth_year(self): + # 2018 Yangon Children's writes MM_YC005's D.O.B. as 2003. + m = CellMismatch( + key="__key_patient_id=MM_YC005", + column="dob", + r_value=datetime.date(1905, 6, 25), + py_value=datetime.date(2003, 1, 1), + ) + + assert classify(m, PATIENT_BARE_YEAR_CLASSIFIERS) == "python_reads_bare_year" + + def test_ignores_a_first_of_january_r_did_not_read_as_that_serial(self): + # Both sides parsed a real date; the serial arithmetic must not match. + m = CellMismatch( + key="__key_patient_id=MY_SW004", + column="t1d_diagnosis_date", + r_value=datetime.date(2012, 6, 16), + py_value=datetime.date(2012, 1, 1), + ) + + assert classify(m, PATIENT_BARE_YEAR_CLASSIFIERS) == "unclassified" + + +class TestPythonAgeFromBareYear: + """Ticket 52: the ages derived from a bare-year date.""" + + def test_flags_an_age_on_a_row_carrying_a_bare_year_date(self): + m = CellMismatch( + key="__key_patient_id=MM_YC005", + column="age", + r_value=999999.0, + py_value=15.0, + row_has_bare_year_date=True, + ) + + assert classify(m, PATIENT_AGE_FROM_BARE_YEAR_CLASSIFIERS) == "python_age_from_bare_year" + + def test_ignores_the_same_shape_without_a_bare_year_date_on_the_row(self): + m = CellMismatch( + key="__key_patient_id=MM_YC099", + column="age", + r_value=999999.0, + py_value=15.0, + ) + + assert classify(m, PATIENT_AGE_FROM_BARE_YEAR_CLASSIFIERS) == "unclassified" + + def test_compare_cells_sets_the_flag_across_the_whole_row(self): + # dob carries the bare year; age is derived from it and must inherit + # the flag even though its own values say nothing about a serial. + r_df = pl.DataFrame( + { + "patient_id": ["MM_YC005"], + "dob": [datetime.date(1905, 6, 25)], + "age": [999999.0], + } + ) + py_df = pl.DataFrame( + { + "patient_id": ["MM_YC005"], + "dob": [datetime.date(2003, 1, 1)], + "age": [15.0], + } + ) + + by_col = {m.column: m for m in compare_cells(r_df, py_df, ["patient_id"])} + + assert by_col["dob"].row_has_bare_year_date + assert by_col["age"].row_has_bare_year_date + + +class TestRYmdFirstMisparse: + """Ticket 51: R's parse_date_string tries "ymd" before "dmy", so a source + string written D.M.YY is read year-first.""" + + def test_flags_the_day_year_swap(self): + # Source "30.1.18": Python reads 2018-01-30, R reads 2030-01-18. + m = CellMismatch( + key="__key_patient_id=MM_YC001", + column="fbg_updated_date", + r_value=datetime.date(2030, 1, 18), + py_value=datetime.date(2018, 1, 30), + ) + + assert classify(m, PATIENT_YMD_FIRST_CLASSIFIERS) == "r_ymd_first_misparse" + + def test_flags_the_swap_when_r_lands_in_the_past(self): + # Source "5.9.17": R reads year 05 -> 2005-09-17, Python 2017-09-05. + m = CellMismatch( + key="__key_patient_id=MM_YC002", + column="hba1c_updated_date", + r_value=datetime.date(2005, 9, 17), + py_value=datetime.date(2017, 9, 5), + ) + + assert classify(m, PATIENT_YMD_FIRST_CLASSIFIERS) == "r_ymd_first_misparse" + + def test_accepts_datetime_as_well_as_date(self): + m = CellMismatch( + key="__key_patient_id=MM_YC001", + column="fbg_updated_date", + r_value=datetime.datetime(2030, 1, 18), + py_value=datetime.datetime(2018, 1, 30), + ) + + assert classify(m, PATIENT_YMD_FIRST_CLASSIFIERS) == "r_ymd_first_misparse" + + def test_unclassified_when_the_month_differs(self): + m = CellMismatch( + key="__key_patient_id=MM_YC001", + column="fbg_updated_date", + r_value=datetime.date(2030, 2, 18), + py_value=datetime.date(2018, 1, 30), + ) + + assert classify(m, PATIENT_YMD_FIRST_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_the_digits_are_not_a_swap(self): + m = CellMismatch( + key="__key_patient_id=MM_YC001", + column="fbg_updated_date", + r_value=datetime.date(2019, 1, 18), + py_value=datetime.date(2018, 1, 30), + ) + + assert classify(m, PATIENT_YMD_FIRST_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_the_dates_agree_on_the_day(self): + """A same-day pair is not a swap even when the years happen to line up + -- an unambiguous D.M.YY date (day 18, year 18) reads the same either + way, so a difference on it has some other cause.""" + m = CellMismatch( + key="__key_patient_id=MM_YC001", + column="fbg_updated_date", + r_value=datetime.date(2018, 1, 18), + py_value=datetime.date(2018, 1, 18), + ) + + assert classify(m, PATIENT_YMD_FIRST_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_a_side_is_the_sentinel(self): + m = CellMismatch( + key="__key_patient_id=MM_YC001", + column="fbg_updated_date", + r_value=SENTINEL_DATE, + py_value=datetime.date(2018, 1, 30), + ) + + assert classify(m, PATIENT_YMD_FIRST_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_a_side_is_null(self): + m = CellMismatch( + key="__key_patient_id=MM_YC001", + column="fbg_updated_date", + r_value=None, + py_value=datetime.date(2018, 1, 30), + ) + + assert classify(m, PATIENT_YMD_FIRST_CLASSIFIERS) == "unclassified" + + +class TestPythonRejectsBeyondTrackerYear: + """Ticket 51: Python's _validate_dates stamps the error sentinel on a date + later than its tracker year; R has no such guard and carries it through.""" + + def test_flags_a_date_later_than_the_sheet_year(self): + m = CellMismatch( + key={"__key_patient_id": "VN_VC002", "__key_sheet_name": "Jan22"}, + column="t1d_diagnosis_date", + r_value=datetime.date(2023, 2, 16), + py_value=SENTINEL_DATE, + ) + + assert classify(m, PATIENT_BEYOND_TRACKER_YEAR_CLASSIFIERS) == ( + "python_rejects_beyond_tracker_year" + ) + + def test_reads_the_year_through_an_apostrophe_in_the_sheet_name(self): + m = CellMismatch( + key={"__key_patient_id": "VN_VC002", "__key_sheet_name": "Oct'22"}, + column="t1d_diagnosis_date", + r_value=datetime.date(2023, 2, 16), + py_value=SENTINEL_DATE, + ) + + assert classify(m, PATIENT_BEYOND_TRACKER_YEAR_CLASSIFIERS) == ( + "python_rejects_beyond_tracker_year" + ) + + def test_unclassified_when_r_date_is_within_the_tracker_year(self): + """The parse-failure half of the same symptom -- Python sentinels a + value it could not read at all, which this cause does not explain.""" + m = CellMismatch( + key={"__key_patient_id": "LA_MH001", "__key_sheet_name": "Jan21"}, + column="fbg_updated_date", + r_value=datetime.date(2020, 9, 1), + py_value=SENTINEL_DATE, + ) + + assert classify(m, PATIENT_BEYOND_TRACKER_YEAR_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_python_is_not_the_sentinelling_side(self): + m = CellMismatch( + key={"__key_patient_id": "VN_VC002", "__key_sheet_name": "Jan22"}, + column="t1d_diagnosis_date", + r_value=SENTINEL_DATE, + py_value=datetime.date(2023, 2, 16), + ) + + assert classify(m, PATIENT_BEYOND_TRACKER_YEAR_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_the_sheet_name_carries_no_year(self): + m = CellMismatch( + key={"__key_patient_id": "VN_VC002", "__key_sheet_name": "Patient List"}, + column="t1d_diagnosis_date", + r_value=datetime.date(2023, 2, 16), + py_value=SENTINEL_DATE, + ) + + assert classify(m, PATIENT_BEYOND_TRACKER_YEAR_CLASSIFIERS) == "unclassified" + + +class TestRUnicodeSanitizerRejectsAccent: + """Ticket 51: R's `[^[:alnum:]]` keeps accented letters, so a misaccented + spelling misses its allowed value; Python's `[^a-z0-9]` folds it home.""" + + def test_flags_an_accented_canonical_value_r_called_undefined(self): + m = CellMismatch( + key={"patient_id": "VN_VC020", "sheet_name": "Jan22"}, + column="province", + r_value="Undefined", + py_value="Thái Nguyên", + ) + + assert classify(m, PATIENT_UNICODE_SANITIZER_CLASSIFIERS) == ( + "r_unicode_sanitizer_rejects_accent" + ) + + def test_unclassified_when_the_canonical_value_is_plain_ascii(self): + """An unaccented canonical value cannot have been rejected for its + accents, so some other cause is at work.""" + m = CellMismatch( + key={"patient_id": "TH_BK001", "sheet_name": "Jan22"}, + column="province", + r_value="Undefined", + py_value="Bangkok", + ) + + assert classify(m, PATIENT_UNICODE_SANITIZER_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_r_did_not_sentinel_the_value(self): + m = CellMismatch( + key={"patient_id": "VN_VC020", "sheet_name": "Jan22"}, + column="province", + r_value="Hà Nội", + py_value="Thái Nguyên", + ) + + assert classify(m, PATIENT_UNICODE_SANITIZER_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_python_has_nothing(self): + m = CellMismatch( + key={"patient_id": "VN_VC020", "sheet_name": "Jan22"}, + column="province", + r_value="Undefined", + py_value=None, + ) + + assert classify(m, PATIENT_UNICODE_SANITIZER_CLASSIFIERS) == "unclassified" + + +class TestRNeverDerivesDiagnosisAge: + """Ticket 54: R's fix_t1d_diagnosis_age call site is commented out, so R + passes the source column through and Python derives the age from dates.""" + + def test_flags_an_age_python_derived_where_the_source_cell_was_blank(self): + m = CellMismatch( + key={"patient_id": "MY_SW001", "sheet_name": "Jan24"}, + column="t1d_diagnosis_age", + r_value=None, + py_value=10, + ) + + assert classify(m, PATIENT_DIAGNOSIS_AGE_CLASSIFIERS) == "r_never_derives_diagnosis_age" + + def test_flags_an_age_python_derived_where_the_source_wrote_it_in_words(self): + """`At birth`, `4mth`, `11yr` -- as.numeric() fails and R sentinels.""" + m = CellMismatch( + key={"patient_id": "MM_MD010", "sheet_name": "May17"}, + column="t1d_diagnosis_age", + r_value=999999.0, + py_value=11, + ) + + assert classify(m, PATIENT_DIAGNOSIS_AGE_CLASSIFIERS) == "r_never_derives_diagnosis_age" + + def test_unclassified_where_both_sides_hold_a_real_age(self): + """A genuine disagreement between two recorded ages is a different + question, and must not be absorbed by this cause.""" + m = CellMismatch( + key={"patient_id": "MM_YG001", "sheet_name": "Jan23"}, + column="t1d_diagnosis_age", + r_value=10.0, + py_value=11, + ) + + assert classify(m, PATIENT_DIAGNOSIS_AGE_CLASSIFIERS) == "unclassified" + + def test_unclassified_where_python_derived_nothing(self): + m = CellMismatch( + key={"patient_id": "TH_CP005", "sheet_name": "Jan23"}, + column="t1d_diagnosis_age", + r_value=None, + py_value=None, + ) + + assert classify(m, PATIENT_DIAGNOSIS_AGE_CLASSIFIERS) == "unclassified" + + +class TestSourceDateInDiagnosisAge: + """Ticket 54: a date typed into the `Age at Diagnosis` column -- R carries + the Excel serial into the age, Python nulls it.""" + + def test_flags_an_impossible_age_r_carried_where_python_nulled(self): + # 20668 is the Excel serial for 1956-08-01, the literal cell content + # of 2023 Chiang Mai's Patient List row for TH_CP005. + m = CellMismatch( + key={"patient_id": "TH_CP005", "sheet_name": "Jan23"}, + column="t1d_diagnosis_age", + r_value=20668.0, + py_value=None, + ) + + assert classify(m, PATIENT_DIAGNOSIS_AGE_CLASSIFIERS) == "source_date_in_diagnosis_age" + + def test_ignores_the_numeric_error_sentinel(self): + """999999 is R's "recorded but invalid" marker, not a date serial -- + r_numeric_error_sentinel owns that shape.""" + m = CellMismatch( + key={"patient_id": "MM_YC003", "sheet_name": "Sep23"}, + column="t1d_diagnosis_age", + r_value=999999.0, + py_value=None, + ) + + assert classify(m, PATIENT_DIAGNOSIS_AGE_CLASSIFIERS) == "unclassified" + + def test_ignores_a_plausible_age_python_happened_to_drop(self): + m = CellMismatch( + key={"patient_id": "MM_YC043", "sheet_name": "Sep23"}, + column="t1d_diagnosis_age", + r_value=12.0, + py_value=None, + ) + + assert classify(m, PATIENT_DIAGNOSIS_AGE_CLASSIFIERS) == "unclassified" + + +class TestFbgTextReading: + """Ticket 55: R's fix_fbg turns non-numeric text into a manufactured mg/dL + reading, and fails on a reading that carries its unit. Python does neither.""" + + def test_flags_r_manufacturing_a_category_value(self): + m = CellMismatch( + key={"patient_id": "MM_YC026", "sheet_name": "Apr17"}, + column="fbg_updated_mg", + r_value=140.0, + py_value=999999.0, + ) + + assert classify(m, PATIENT_FBG_TEXT_CLASSIFIERS) == "r_fbg_text_category_invention" + + def test_flags_r_manufacturing_the_high_value(self): + m = CellMismatch( + key={"patient_id": "MM_YC031", "sheet_name": "Jun17"}, + column="fbg_updated_mg", + r_value=200.0, + py_value=999999.0, + ) + + assert classify(m, PATIENT_FBG_TEXT_CLASSIFIERS) == "r_fbg_text_category_invention" + + def test_ignores_a_category_value_python_also_produced(self): + """Both sides landing on 200 is agreement, not this cause.""" + m = CellMismatch( + key={"patient_id": "MM_YC031", "sheet_name": "Jun17"}, + column="fbg_updated_mg", + r_value=200.0, + py_value=201.0, + ) + + assert classify(m, PATIENT_FBG_TEXT_CLASSIFIERS) == "unclassified" + + def test_flags_python_reading_a_unit_suffixed_value_r_could_not(self): + m = CellMismatch( + key={"patient_id": "MM_MD007", "sheet_name": "Mar18"}, + column="fbg_updated_mg", + r_value=999999.0, + py_value=148.0, + ) + + assert classify(m, PATIENT_FBG_TEXT_CLASSIFIERS) == "r_unit_suffix_not_stripped" + + def test_ignores_a_python_sentinel_on_both_sides(self): + m = CellMismatch( + key={"patient_id": "MM_MD007", "sheet_name": "Mar18"}, + column="fbg_updated_mg", + r_value=999999.0, + py_value=999999.0, + ) + + assert classify(m, PATIENT_FBG_TEXT_CLASSIFIERS) == "unclassified" + + +class TestInsulinSubtypeFromDrugName: + """Ticket 55: 2024 Sarawak ticks the insulin columns by writing the drug's + name, which R never reads.""" + + def test_flags_a_subtype_only_python_recovers(self): + m = CellMismatch( + key={"patient_id": "MY_SW001", "sheet_name": "Jan24"}, + column="insulin_subtype", + r_value=None, + py_value="Rapid-acting,Long-acting", + ) + + assert classify(m, PATIENT_INSULIN_DRUG_NAME_CLASSIFIERS) == "r_drops_drug_name_tick" + + def test_ignores_the_undefined_sentinel(self): + """An unticked row is a different question, still open.""" + m = CellMismatch( + key={"patient_id": "PH_CD001", "sheet_name": "Jan25"}, + column="insulin_subtype", + r_value=None, + py_value="Undefined", + ) + + assert classify(m, PATIENT_INSULIN_DRUG_NAME_CLASSIFIERS) == "unclassified" + + +class TestRMonthNameTruncatedToYear: + """Ticket 56: R deletes the fourth letter of a spelled-out month, then falls + through its order list to "y" and keeps only the year.""" + + def test_flags_a_month_year_cell_r_read_as_january(self): + # Source "April-17": R yields 2017-01-01, Python 2017-04-01. + m = CellMismatch( + key={"patient_id": "TH_MK001", "sheet_name": "Jan21"}, + column="recruitment_date", + r_value=datetime.date(2017, 1, 1), + py_value=datetime.date(2017, 4, 1), + ) + + assert ( + classify(m, PATIENT_MONTH_NAME_TRUNCATED_CLASSIFIERS) + == "r_month_name_truncated_to_year" + ) + + def test_unclassified_when_python_also_reads_january(self): + """Both on 1 January is the bare-year cause, not this one.""" + m = CellMismatch( + key={"patient_id": "TH_MK001", "sheet_name": "Jan21"}, + column="recruitment_date", + r_value=datetime.date(2017, 1, 1), + py_value=datetime.date(2017, 1, 1), + ) + + assert classify(m, PATIENT_MONTH_NAME_TRUNCATED_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_python_carries_a_day_of_month(self): + """A month-year cell resolves to the first; anything else is a different + source shape.""" + m = CellMismatch( + key={"patient_id": "TH_MK001", "sheet_name": "Jan21"}, + column="recruitment_date", + r_value=datetime.date(2017, 1, 1), + py_value=datetime.date(2017, 4, 23), + ) + + assert classify(m, PATIENT_MONTH_NAME_TRUNCATED_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_the_years_differ(self): + m = CellMismatch( + key={"patient_id": "TH_MK001", "sheet_name": "Jan21"}, + column="recruitment_date", + r_value=datetime.date(2016, 1, 1), + py_value=datetime.date(2017, 4, 1), + ) + + assert classify(m, PATIENT_MONTH_NAME_TRUNCATED_CLASSIFIERS) == "unclassified" + + +class TestRReadsDayAsMonth: + """Ticket 56: where the month name is one R cannot read, R ignores it and + reads the remaining digits as month and year.""" + + def test_flags_the_day_promoted_to_month(self): + # Source "9-Dce-20": R yields 2020-09-01, Python 2020-12-09. + m = CellMismatch( + key={"patient_id": "LA_MH001", "sheet_name": "Dec20"}, + column="bmi_date", + r_value=datetime.date(2020, 9, 1), + py_value=datetime.date(2020, 12, 9), + ) + + assert classify(m, PATIENT_UNREADABLE_MONTH_CLASSIFIERS) == "r_reads_day_as_month" + + def test_flags_it_when_day_and_month_number_coincide(self): + # Source "5-Mei-2023": R yields 2023-05-01, Python 2023-05-05. + m = CellMismatch( + key={"patient_id": "MY_LK001", "sheet_name": "May23"}, + column="hba1c_updated_date", + r_value=datetime.date(2023, 5, 1), + py_value=datetime.date(2023, 5, 5), + ) + + assert classify(m, PATIENT_UNREADABLE_MONTH_CLASSIFIERS) == "r_reads_day_as_month" + + def test_unclassified_when_r_is_not_on_the_first(self): + m = CellMismatch( + key={"patient_id": "LA_MH001", "sheet_name": "Dec20"}, + column="bmi_date", + r_value=datetime.date(2020, 9, 3), + py_value=datetime.date(2020, 12, 9), + ) + + assert classify(m, PATIENT_UNREADABLE_MONTH_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_pythons_day_is_not_rs_month(self): + m = CellMismatch( + key={"patient_id": "LA_MH001", "sheet_name": "Dec20"}, + column="bmi_date", + r_value=datetime.date(2020, 9, 1), + py_value=datetime.date(2020, 12, 11), + ) + + assert classify(m, PATIENT_UNREADABLE_MONTH_CLASSIFIERS) == "unclassified" + + +class TestRParseOrderCannotReadCell: + """Ticket 56: where R's fixed order list has no reading for the cell it + sentinels, while Python reads the date the clinic wrote.""" + + def test_flags_a_day_r_cannot_promote_to_a_month(self): + # Source "25-Mach-20": R's fallback needs month 25, so it yields NA. + m = CellMismatch( + key={"patient_id": "LA_MH009", "sheet_name": "Apr20"}, + column="bmi_date", + r_value=datetime.date(9999, 9, 9), + py_value=datetime.date(2020, 3, 25), + ) + + assert classify(m, PATIENT_UNREADABLE_MONTH_CLASSIFIERS) == "r_parse_order_cannot_read_cell" + + def test_unclassified_when_python_holds_an_early_day(self): + """A day of 12 or less is one R's positional fallback can read as a + month, so an R sentinel there has some other cause.""" + m = CellMismatch( + key={"patient_id": "LA_MH009", "sheet_name": "Apr20"}, + column="bmi_date", + r_value=datetime.date(9999, 9, 9), + py_value=datetime.date(2020, 3, 4), + ) + + assert classify(m, PATIENT_UNREADABLE_MONTH_CLASSIFIERS) == "unclassified" + + def test_unclassified_when_python_has_nothing_either(self): + m = CellMismatch( + key={"patient_id": "LA_MH009", "sheet_name": "Apr20"}, + column="bmi_date", + r_value=datetime.date(9999, 9, 9), + py_value=None, + ) + + assert classify(m, PATIENT_UNREADABLE_MONTH_CLASSIFIERS) == "unclassified" + + +class TestPythonDeclinesUnreconstructableDate: + """Round 10's largest surviving group: R publishes a date, Python sentinels. + + R's parse chain cannot fail loudly (round 9, executed against R 4.5), so a + source token damaged past reading still yields R *a* date. Python's parser + refuses. Every cell behind this is a source defect for ticket 40. + """ + + @staticmethod + def _m(r_value, py_value): + return CellMismatch(key={"id": 1}, column="lost_date", r_value=r_value, py_value=py_value) + + def test_r_date_against_python_sentinel(self): + assert _is_python_declines_unreconstructable_date( + self._m(datetime.date(2025, 4, 25), SENTINEL_DATE) + ) + + def test_both_real_dates_are_not_this_cause(self): + assert not _is_python_declines_unreconstructable_date( + self._m(datetime.date(2025, 4, 25), datetime.date(2025, 3, 25)) + ) + + def test_the_reverse_direction_is_a_different_cause(self): + assert not _is_python_declines_unreconstructable_date( + self._m(SENTINEL_DATE, datetime.date(2025, 4, 25)) + ) + + def test_r_null_is_not_this_cause(self): + assert not _is_python_declines_unreconstructable_date(self._m(None, SENTINEL_DATE)) + + +class TestPythonRejectsOutOfRangeHba1c: + """A glucose reading typed into the HbA1c column: R passes 240 through, + Python fails it against the schema's HbA1c bound and sentinels.""" + + @staticmethod + def _m(r_value, py_value, column="hba1c_updated"): + return CellMismatch(key={"id": 1}, column=column, r_value=r_value, py_value=py_value) + + def test_reading_above_the_declared_maximum(self): + assert _is_python_rejects_out_of_range_hba1c(self._m(240.0, settings.error_val_numeric)) + + def test_reading_inside_the_range_is_a_different_cause(self): + assert not _is_python_rejects_out_of_range_hba1c(self._m(9.4, settings.error_val_numeric)) + + def test_python_must_hold_the_sentinel(self): + assert not _is_python_rejects_out_of_range_hba1c(self._m(240.0, 240.0)) + + def test_only_the_hba1c_columns(self): + assert not _is_python_rejects_out_of_range_hba1c( + self._m(240.0, settings.error_val_numeric, column="weight") + ) + + +class TestGlucoseUnitSwapSurvivors: + """The two shapes the ticket-42 unit swap leaves that the original + classifier could not see, both measured on the real 254-tracker set.""" + + @staticmethod + def _m(r_value, py_value, column="fbg_baseline_mg"): + return CellMismatch(key={"id": 1}, column=column, r_value=r_value, py_value=py_value) + + def test_a_swapped_columns_outlier_is_out_of_this_classifiers_reach(self): + """2020 Kantha Bopha: 90.1% of the column is sub-30, so it is read as + mmol/L and 47.8 is rejected against the mmol ceiling -- but 47.8 is a + fine mg/dL reading, and whether the column was swapped is not visible + from one cell. These six stay unclassified deliberately (ticket 57).""" + assert not _is_python_glucose_unit_corrected(self._m(47.8, settings.error_val_numeric)) + + def test_a_reading_outside_the_headers_unit_is_still_caught(self): + assert _is_python_glucose_unit_corrected(self._m(900.0, settings.error_val_numeric)) + + def test_swap_recovers_a_reading_r_rejected(self): + """2022 Penang: the mmol cell reads "-" (R sentinels it) while the mg + cell holds 9, which the swap moves across.""" + assert _is_python_recovers_glucose_r_rejected( + self._m(settings.error_val_numeric, 9.0, column="fbg_baseline_mmol") + ) + + def test_an_implausible_recovery_is_not_claimed(self): + assert not _is_python_recovers_glucose_r_rejected( + self._m(settings.error_val_numeric, 999.0, column="fbg_baseline_mmol") + ) + + +class TestSwappedColumnIsJudgedInTheUnitItHolds: + """Ticket 44: the file-level fact a per-cell classifier could not see.""" + + @staticmethod + def _m(r_value, py_value, column="fbg_baseline_mg", swapped=False): + return CellMismatch( + key={"id": 1}, + column=column, + r_value=r_value, + py_value=py_value, + column_unit_swapped=swapped, + ) + + def test_an_outlier_in_a_swapped_column_is_judged_against_the_mmol_ceiling(self): + """2020 Kantha Bopha's 47.8 and 53.8: the column is 90.1% sub-30, so + Python reads it as mmol/L and rejects both against the 45 mmol/L + ceiling. Round 10 could only write these six out in prose.""" + assert _is_python_glucose_unit_corrected( + self._m(47.8, settings.error_val_numeric, swapped=True) + ) + + def test_the_same_reading_in_an_unswapped_column_is_not_claimed(self): + assert not _is_python_glucose_unit_corrected(self._m(47.8, settings.error_val_numeric)) + + def test_a_manufactured_constant_stays_with_the_cause_that_explains_it(self): + """140 exceeds the mmol ceiling, but it is not a reading -- fix_fbg + made it out of "Lost follow up", which r_fbg_text_category_invention + already says.""" + assert not _is_python_glucose_unit_corrected( + self._m(140.0, settings.error_val_numeric, swapped=True) + ) + + def test_a_reading_the_mmol_ceiling_accepts_is_not_claimed(self): + assert not _is_python_glucose_unit_corrected( + self._m(12.0, settings.error_val_numeric, swapped=True) + ) + + +class TestMmolCascadesFromItsMgSibling: + """Ticket 44: the mmol column is derived from the mg one, so an mg cell's + divergence restates itself next door. All three shapes are real rows from + the 254-tracker run.""" + + @staticmethod + def _m(r_value, py_value, mg_sibling, column="fbg_updated_mmol"): + return CellMismatch( + key={"id": 1}, + column=column, + r_value=r_value, + py_value=py_value, + mg_sibling=mg_sibling, + ) + + def test_the_swap_moves_a_reading_r_never_had_in_mmol(self): + """2025 Kantha Bopha II KH_KB097: the source writes 14.4 under the + mg/dL header, so R keeps mg=14.4 with no mmol at all while Python + moves it across and rescales mg to 259.2.""" + assert ( + classify( + self._m(None, 14.4, mg_sibling=(14.4, 259.2)), + PATIENT_GLUCOSE_CASCADE_CLASSIFIERS, + ) + == "mmol_derived_from_mg_sibling" + ) + + def test_r_derives_mmol_from_an_mg_reading_python_refused(self): + """2018 Yangon: fix_fbg invents 140 mg/dL out of "Lost follow up", and + R divides it by 18 to publish 7.78 mmol/L as a real measurement.""" + assert ( + classify( + self._m(140 / 18, None, mg_sibling=(140.0, settings.error_val_numeric)), + PATIENT_GLUCOSE_CASCADE_CLASSIFIERS, + ) + == "mmol_derived_from_mg_sibling" + ) + + def test_python_derives_mmol_from_an_mg_reading_r_refused(self): + """2018 CDA: the source writes "148 mg/dl (Mar-18)". R's as.numeric + fails on the whole string and sentinels it; Python reads 148 and + derives 8.22 mmol/L.""" + assert ( + classify( + self._m(None, 148 / 18, mg_sibling=(settings.error_val_numeric, 148.0)), + PATIENT_GLUCOSE_CASCADE_CLASSIFIERS, + ) + == "mmol_derived_from_mg_sibling" + ) + + def test_a_sentinel_on_one_side_still_carries_no_reading_of_its_own(self): + """2019 Kantha Bopha: R publishes 140/18 while Python sentinels the + mmol cell too, because the mg cell it derives from was unreadable.""" + assert ( + classify( + self._m( + 140 / 18, + settings.error_val_numeric, + mg_sibling=(140.0, settings.error_val_numeric), + ), + PATIENT_GLUCOSE_CASCADE_CLASSIFIERS, + ) + == "mmol_derived_from_mg_sibling" + ) + + def test_an_mmol_value_that_is_not_its_own_mg_cells_image_is_not_claimed(self): + """The bound that keeps this from swallowing a genuine mmol + divergence: 9.0 is not 148/18, so the cell carries a reading of its + own and the cascade has not explained it.""" + assert ( + classify( + self._m(None, 9.0, mg_sibling=(settings.error_val_numeric, 148.0)), + PATIENT_GLUCOSE_CASCADE_CLASSIFIERS, + ) + == "unclassified" + ) + + def test_an_agreeing_mg_sibling_leaves_the_mmol_divergence_unexplained(self): + """Nothing cascaded: both sides read the same mg cell, so a difference + in the mmol cell is its own.""" + assert ( + classify( + self._m(None, 148 / 18, mg_sibling=(148.0, 148.0)), + PATIENT_GLUCOSE_CASCADE_CLASSIFIERS, + ) + == "unclassified" + ) + + def test_both_sides_absent_is_not_a_cascade(self): + assert ( + classify( + self._m(None, settings.error_val_numeric, mg_sibling=(140.0, None)), + PATIENT_GLUCOSE_CASCADE_CLASSIFIERS, + ) + == "unclassified" + ) + + def test_the_mg_column_itself_is_not_in_scope(self): + assert ( + classify( + self._m(None, 14.4, mg_sibling=(14.4, 259.2), column="fbg_updated_mg"), + PATIENT_GLUCOSE_CASCADE_CLASSIFIERS, + ) + == "unclassified" + ) + + +class TestCompareCellsSuppliesGlucoseContext: + def test_the_mg_sibling_travels_with_the_mmol_mismatch(self): + r_df = pl.DataFrame({"id": [1], "fbg_updated_mg": [14.4], "fbg_updated_mmol": [None]}) + py_df = pl.DataFrame({"id": [1], "fbg_updated_mg": [259.2], "fbg_updated_mmol": [14.4]}) + + mismatches = compare_cells(r_df, py_df, key_cols=["id"]) + + mmol = next(m for m in mismatches if m.column == "fbg_updated_mmol") + assert mmol.mg_sibling == (14.4, 259.2) + + def test_a_swapped_column_is_flagged_on_both_halves_of_its_pair(self): + r_df = pl.DataFrame({"id": [1], "fbg_updated_mg": [14.4], "fbg_updated_mmol": [None]}) + py_df = pl.DataFrame({"id": [1], "fbg_updated_mg": [259.2], "fbg_updated_mmol": [14.4]}) + + mismatches = compare_cells( + r_df, py_df, key_cols=["id"], unit_swapped_columns={"fbg_updated_mg"} + ) + + assert {m.column for m in mismatches if m.column_unit_swapped} == { + "fbg_updated_mg", + "fbg_updated_mmol", + } + + def test_an_unswapped_run_flags_nothing(self): + r_df = pl.DataFrame({"id": [1], "fbg_updated_mg": [14.4], "fbg_updated_mmol": [None]}) + py_df = pl.DataFrame({"id": [1], "fbg_updated_mg": [259.2], "fbg_updated_mmol": [14.4]}) + + mismatches = compare_cells(r_df, py_df, key_cols=["id"]) + + assert not any(m.column_unit_swapped for m in mismatches) + + +class TestGlucoseUnitSwapsFromErrorRecords: + """The swap list is read back from the pipeline's own error table rather + than restated here -- ``resolve_glucose_units`` already writes one row per + swapped file-column.""" + + def test_swaps_are_grouped_by_tracker(self): + errors = pl.DataFrame( + { + "file_name": ["2020_KB", "2020_KB", "2018_CDA", "2020_KB"], + "column": [ + "fbg_baseline_mg", + "fbg_updated_mg", + "fbg_updated_mg", + "fbg_updated_mg", + ], + "error_code": [ + "glucose_unit_swapped", + "glucose_unit_swapped", + "glucose_unit_suspect", + "glucose_unit_swapped", + ], + } + ) + + assert load_glucose_unit_swaps(errors) == {"2020_KB": {"fbg_baseline_mg", "fbg_updated_mg"}} + + def test_a_run_with_no_swaps_yields_nothing(self): + errors = pl.DataFrame( + {"file_name": ["2018_CDA"], "column": ["age"], "error_code": ["type_conversion"]} + ) + + assert load_glucose_unit_swaps(errors) == {} + + +class TestClinicalNoteClassifiers: + """Ticket 39: hospitalisation_date's residual, whose whole population round + 6 measured as clinical notes rather than dates. + """ + + def test_python_reads_the_date_r_could_not_read_out_of_a_note(self): + mismatch = _mismatch( + r_value=SENTINEL_DATE, + py_value=datetime.date(2020, 10, 23), + column="hospitalisation_date", + ) + + assert ( + classify(mismatch, PATIENT_CLINICAL_NOTE_CLASSIFIERS) + == "python_reads_date_in_clinical_note" + ) + + def test_both_pipelines_find_a_date_in_the_note_and_disagree(self): + # "DKA: admitted 6-12 Nov 2020": R reads the range's two numbers as a + # day and a month; Python takes the admission day. + mismatch = _mismatch( + r_value=datetime.date(2020, 12, 6), + py_value=datetime.date(2020, 11, 6), + column="hospitalisation_date", + ) + + assert ( + classify(mismatch, PATIENT_CLINICAL_NOTE_CLASSIFIERS) == "note_dates_read_differently" + ) + + def test_r_turns_a_bare_year_in_the_note_into_the_first_of_january(self): + # "DKA 2020: June, Aug, Nov" -- R's answer contradicts the note's own + # months, so Python's refusal is the correct side. + mismatch = _mismatch( + r_value=datetime.date(2020, 1, 1), + py_value=SENTINEL_DATE, + column="hospitalisation_date", + ) + + assert ( + classify(mismatch, PATIENT_CLINICAL_NOTE_CLASSIFIERS) + == "r_invents_january_from_bare_year" + ) + + def test_a_date_python_refuses_that_r_dates_mid_year_is_not_this_cause(self): + """Only 1 January carries the bare-year signature; any other R date + means R read something the note actually said, which is a different + question. + """ + mismatch = _mismatch( + r_value=datetime.date(2020, 6, 15), + py_value=SENTINEL_DATE, + column="hospitalisation_date", + ) + + assert classify(mismatch, PATIENT_CLINICAL_NOTE_CLASSIFIERS) == "unclassified" + + def test_both_sentinels_is_not_a_mismatch_this_registry_claims(self): + mismatch = _mismatch( + r_value=SENTINEL_DATE, + py_value=SENTINEL_DATE, + column="hospitalisation_date", + ) + + assert classify(mismatch, PATIENT_CLINICAL_NOTE_CLASSIFIERS) == "unclassified" + + +class TestBuddhistEraTypoBand: + """Ticket 60: ``buddhist_era_typo`` tested only ``year >= 2400``, so it + claimed any far-future date as a Buddhist-era typo. Since ticket 61 the + cleaned stage *converts* genuine BE dates, so a residual year above the + threshold is by construction not a recoverable BE date -- and the two real + populations it was claiming (3035-03-01 in a 2025 CDA tracker, 5025-05-19 + in a 2025 Surat Thani tracker) are ordinary year typos in the source + workbook, verified against the cells themselves. + """ + + def test_buddhist_era_typo_when_the_year_sits_in_the_sheets_own_be_band(self): + mismatch = CellMismatch( + key={"__key_sheet_name": "Jun26"}, + column="fbg_updated_date", + r_value=SENTINEL_DATE, + py_value=datetime.date(2569, 6, 17), + ) + + assert classify(mismatch, PATIENT_BUDDHIST_ERA_CLASSIFIERS) == "buddhist_era_typo" + + def test_buddhist_era_typo_declines_a_year_far_outside_the_be_band(self): + mismatch = CellMismatch( + key={"__key_sheet_name": "Mar25"}, + column="fbg_updated_date", + r_value=SENTINEL_DATE, + py_value=datetime.date(3035, 3, 1), + ) + + assert classify(mismatch, PATIENT_BUDDHIST_ERA_CLASSIFIERS) == "unclassified" + + def test_buddhist_era_typo_declines_the_surat_thani_year_typo(self): + mismatch = CellMismatch( + key={"__key_sheet_name": "May25"}, + column="fbg_updated_date", + r_value=SENTINEL_DATE, + py_value=datetime.date(5025, 5, 19), + ) + + assert classify(mismatch, PATIENT_BUDDHIST_ERA_CLASSIFIERS) == "unclassified" + + def test_buddhist_era_typo_still_fires_when_the_sheet_year_is_unknown(self): + """Without a sheet year the band cannot be applied, so the cause keeps + its old behaviour rather than silently dropping a real BE date.""" + mismatch = CellMismatch( + key={"id": 1}, + column="fbg_updated_date", + r_value=SENTINEL_DATE, + py_value=datetime.date(2569, 6, 17), + ) + + assert classify(mismatch, PATIENT_BUDDHIST_ERA_CLASSIFIERS) == "buddhist_era_typo" + + +class TestAbsurdExcelSerialOnPatientDates: + """Ticket 60: bounding ``buddhist_era_typo`` exposed five 2025 Surat Thani + raw cells whose R side is the Excel serial ``1141523`` and whose Python + side is ``5025-05-19``. They fell through to + ``r_parse_order_cannot_read_cell``, which is false about R -- R did not + fail its parse orders, it carried the serial through as a string, and the + comparison's own ``normalize_date_column`` is what produced the sentinel. + ``python_absurd_excel_serial`` (ticket 32) already names this shape; it was + only ever wired to the product entry-date registry. + """ + + def test_absurd_serial_claims_the_surat_thani_raw_cell(self): + mismatch = CellMismatch( + key={"__key_sheet_name": "May25"}, + column="fbg_updated_date", + r_value=SENTINEL_DATE, + py_value=datetime.date(5025, 5, 19), + ) + + registry = _compare_outputs_module().CLASSIFIERS_BY_COLUMN["fbg_updated_date"] + + assert classify(mismatch, registry) == "python_absurd_excel_serial" + + def test_a_genuine_be_year_still_reaches_buddhist_era_typo(self): + """The absurd-serial cause must not out-rank a real Buddhist-era date, + which in the raw stage is not yet converted and so also sits above the + 2100 threshold.""" + mismatch = CellMismatch( + key={"__key_sheet_name": "Jun26"}, + column="fbg_updated_date", + r_value=SENTINEL_DATE, + py_value=datetime.date(2569, 6, 17), + ) + + registry = _compare_outputs_module().CLASSIFIERS_BY_COLUMN["fbg_updated_date"] + + assert classify(mismatch, registry) == "buddhist_era_typo" + + def test_a_day_past_12_with_a_plausible_year_still_reaches_parse_order(self): + mismatch = CellMismatch( + key={"__key_sheet_name": "May25"}, + column="fbg_updated_date", + r_value=SENTINEL_DATE, + py_value=datetime.date(2025, 5, 19), + ) + + registry = _compare_outputs_module().CLASSIFIERS_BY_COLUMN["fbg_updated_date"] + + assert classify(mismatch, registry) == "r_parse_order_cannot_read_cell" + + +class TestRStaticJoinMissesRespelledId: + """Ticket 58: R attaches the static sheets on the raw ID, so a respelled one misses.""" + + @staticmethod + def _mismatch(patient_id: str, r_value=None, py_value="Vientiane Capital"): + return CellMismatch( + key={"__key_patient_id": patient_id, "__key_sheet_name": "Jun24"}, + column="province", + r_value=r_value, + py_value=py_value, + ) + + def test_fires_on_a_hyphen_spelled_id_where_r_is_null(self): + """2023/2024 Mahosot write LA-MH056 against a Patient List entry of LA_MH056.""" + assert _is_r_static_join_misses_respelled_id(self._mismatch("LA-MH056")) + + def test_fires_on_a_transfer_clinic_suffix(self): + assert _is_r_static_join_misses_respelled_id(self._mismatch("MY_SM003_SB")) + + def test_declines_a_well_formed_id(self): + """recruitment_date's 28,009 xml:space rows are all well-formed IDs (ticket 62).""" + assert not _is_r_static_join_misses_respelled_id(self._mismatch("KH_CD001")) + + def test_declines_when_r_has_a_value_of_its_own(self): + assert not _is_r_static_join_misses_respelled_id( + self._mismatch("LA-MH056", r_value="Khammouane") + ) + + def test_declines_when_python_has_nothing_to_offer(self): + assert not _is_r_static_join_misses_respelled_id(self._mismatch("LA-MH056", py_value=None)) + + def test_declines_when_the_key_carries_no_patient_id(self): + """The cleaned stage normalizes patient_id on both sides, so it cannot discriminate.""" + assert not _is_r_static_join_misses_respelled_id( + CellMismatch(key={"id": 1}, column="province", r_value=None, py_value="x") + ) + + def test_fires_on_the_prefixed_key_add_row_ordinal_actually_produces(self): + """A bare 'patient_id' lookup silently finds nothing -- the _sheet_year trap.""" + assert _is_r_static_join_misses_respelled_id( + CellMismatch( + key={"__key_patient_id": "LA-MH056", "__key_sheet_name": "Jun24"}, + column="province", + r_value=None, + py_value="Vientiane Capital", + ) + ) + + +class TestStaticJoinMissedIdsOnTheCleanedStage: + """Ticket 63: the same cause, told apart by a fact the cleaned stage cannot hold.""" + + @staticmethod + def _mismatch(*, flagged: bool, r_value=None, py_value="Vientiane Capital"): + return CellMismatch( + key={"__key_patient_id": "LA_MH056", "__key_sheet_name": "Jun24"}, + column="province", + r_value=r_value, + py_value=py_value, + static_join_missed_id=flagged, + ) + + def test_fires_on_a_normalized_id_carrying_the_flag(self): + """By the cleaned stage both sides spell it LA_MH056; only the flag remembers.""" + assert _is_r_static_join_misses_respelled_id(self._mismatch(flagged=True)) + + def test_declines_a_normalized_id_without_the_flag(self): + """This is the raw-stage predicate's behaviour, and it must not change.""" + assert not _is_r_static_join_misses_respelled_id(self._mismatch(flagged=False)) + + def test_declines_a_flagged_row_where_r_has_a_value_of_its_own(self): + """The flag is a property of the patient, not of the cell -- R-null still gates it.""" + assert not _is_r_static_join_misses_respelled_id( + self._mismatch(flagged=True, r_value="Khammouane") + ) + + def test_declines_a_flagged_row_where_python_has_nothing_to_offer(self): + assert not _is_r_static_join_misses_respelled_id( + self._mismatch(flagged=True, py_value=None) + ) + + def test_declines_a_column_r_lost_across_the_whole_file(self): + """2024 Preah Kossamak: R reads zero recruitment_date in 863 rows. + + That is r_extraction_gap's xml:space header defect (ticket 62), which + nulls the column for every patient at once -- the respelled ID is + incidental, and claiming the cell would carry the wrong reason. + """ + mismatch = CellMismatch( + key={"__key_patient_id": "KH_PK001", "__key_sheet_name": "Jun24"}, + column="recruitment_date", + r_value=None, + py_value=datetime.date(2021, 6, 22), + static_join_missed_id=True, + r_column_empty_in_file=True, + ) + + assert not _is_r_static_join_misses_respelled_id(mismatch) + + def test_still_fires_where_r_reads_the_column_for_other_patients(self): + """2024 Mahosot: R reads 672 recruitment_dates of 1,068 rows, so its + nulls really are per-patient.""" + assert _is_r_static_join_misses_respelled_id( + self._mismatch(flagged=True, py_value="Vientiane Capital") + ) + + +class TestStaticJoinMissedIds: + """Ticket 63: which identities in a raw frame were spelled two ways.""" + + def test_returns_the_normalized_spelling_of_a_respelled_id(self): + """The set is matched against the cleaned key, which is already normalized.""" + raw = pl.DataFrame({"patient_id": ["LA-MH056", "LA_MH001"]}) + + assert static_join_missed_ids(raw) == {"LA_MH056"} + + def test_includes_a_transfer_clinic_suffix(self): + raw = pl.DataFrame({"patient_id": ["MY_SM003_SB"]}) + + assert static_join_missed_ids(raw) == {"MY_SM003"} + + def test_is_empty_when_every_id_is_well_formed(self): + raw = pl.DataFrame({"patient_id": ["KH_CD001", "KH_CD002"]}) + + assert static_join_missed_ids(raw) == set() + + def test_tolerates_a_frame_without_a_patient_id_column(self): + """Raw output is not schema-normalized -- a 2017 tracker may lack the column.""" + assert static_join_missed_ids(pl.DataFrame({"sheet_name": ["Jan24"]})) == set() + + def test_ignores_a_null_id(self): + assert static_join_missed_ids(pl.DataFrame({"patient_id": [None]})) == set() + + +class TestCompareCellsStaticJoinMissedIds: + """Ticket 63: the per-file identity set reaches the mismatch it explains.""" + + @staticmethod + def _frames(): + r_df = pl.DataFrame( + {"patient_id": ["LA_MH056", "LA_MH001"], "province": [None, None]}, + ) + py_df = pl.DataFrame( + {"patient_id": ["LA_MH056", "LA_MH001"], "province": ["Vientiane Capital", "Bolikham"]}, + ) + return r_df, py_df + + def test_flags_only_the_rows_whose_identity_missed_the_join(self): + r_df, py_df = self._frames() + + mismatches = compare_cells(r_df, py_df, ["patient_id"], static_join_missed_ids={"LA_MH056"}) + + flagged = {m.key["patient_id"]: m.static_join_missed_id for m in mismatches} + assert flagged == {"LA_MH056": True, "LA_MH001": False} + + def test_flags_nothing_when_the_set_is_not_supplied(self): + """The raw stage passes none, and its own per-cell discriminator still works.""" + r_df, py_df = self._frames() + + mismatches = compare_cells(r_df, py_df, ["patient_id"]) + + assert not any(m.static_join_missed_id for m in mismatches) + + +class TestRAgeNotDerivedWithoutDob: + """Ticket 63: age is derived, so a recovered D.O.B. moves it on Python's side only.""" + + @staticmethod + def _mismatch(*, dob_recovered: bool, r_value="13", py_value="14"): + return CellMismatch( + key={"__key_patient_id": "LA_MH060", "__key_sheet_name": "Jan23"}, + column="age", + r_value=r_value, + py_value=py_value, + row_dob_recovered=dob_recovered, + ) + + def test_fires_where_r_kept_the_typed_age_and_python_recomputed(self): + """LA_MH060 turned 14 on 2009-01-16, so Jan23 is 14; R's month sheet says 13.""" + assert _is_r_age_not_derived_without_dob(self._mismatch(dob_recovered=True)) + + def test_fires_where_r_has_no_age_at_all(self): + """60 of the 113 have an empty month-sheet Age cell and no D.O.B. to fall back on.""" + assert _is_r_age_not_derived_without_dob( + self._mismatch(dob_recovered=True, r_value=None, py_value="15") + ) + + def test_declines_where_r_had_the_dob_too(self): + """Both sides derived from the same birth date, so a difference is something else.""" + assert not _is_r_age_not_derived_without_dob(self._mismatch(dob_recovered=False)) + + def test_declines_where_python_has_nothing_to_offer(self): + assert not _is_r_age_not_derived_without_dob( + self._mismatch(dob_recovered=True, py_value=None) + ) + + +class TestCompareCellsRowDobRecovered: + """Ticket 63: the per-row fact the age cascade needs, tighter than the file-level set.""" + + def test_flags_a_row_where_only_python_has_the_birth_date(self): + r_df = pl.DataFrame({"patient_id": ["A"], "dob": [None], "age": ["13"]}) + py_df = pl.DataFrame({"patient_id": ["A"], "dob": ["2009-01-16"], "age": ["14"]}) + + mismatches = compare_cells(r_df, py_df, ["patient_id"]) + + assert all(m.row_dob_recovered for m in mismatches) + + def test_does_not_flag_a_row_where_both_sides_have_it(self): + r_df = pl.DataFrame({"patient_id": ["A"], "dob": ["2009-01-16"], "age": ["13"]}) + py_df = pl.DataFrame({"patient_id": ["A"], "dob": ["2009-01-16"], "age": ["14"]}) + + mismatches = compare_cells(r_df, py_df, ["patient_id"]) + + assert not any(m.row_dob_recovered for m in mismatches) + + def test_does_not_flag_a_frame_without_a_dob_column(self): + """The raw stage's frames vary by tracker year, so the column may be absent.""" + r_df = pl.DataFrame({"patient_id": ["A"], "age": ["13"]}) + py_df = pl.DataFrame({"patient_id": ["A"], "age": ["14"]}) + + mismatches = compare_cells(r_df, py_df, ["patient_id"]) + + assert not any(m.row_dob_recovered for m in mismatches) + + +class TestCompareCellsRColumnEmptyInFile: + """Ticket 63: a column R lost file-wide is a different mechanism to a per-row miss.""" + + def test_flags_a_column_r_populates_nowhere(self): + r_df = pl.DataFrame({"patient_id": ["A", "B"], "recruitment_date": [None, None]}) + py_df = pl.DataFrame( + {"patient_id": ["A", "B"], "recruitment_date": ["2021-06-22", "2020-01-05"]} + ) + + mismatches = compare_cells(r_df, py_df, ["patient_id"]) + + assert all(m.r_column_empty_in_file for m in mismatches) + + def test_does_not_flag_a_column_r_populates_for_someone(self): + """One surviving value proves R can read the column, so a null is per-row.""" + r_df = pl.DataFrame({"patient_id": ["A", "B"], "recruitment_date": [None, "2020-01-05"]}) + py_df = pl.DataFrame( + {"patient_id": ["A", "B"], "recruitment_date": ["2021-06-22", "2020-01-05"]} + ) + + mismatches = compare_cells(r_df, py_df, ["patient_id"]) + + assert [m.r_column_empty_in_file for m in mismatches] == [False] diff --git a/tests/test_pipeline/__init__.py b/tests/test_pipeline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_pipeline/test_product.py b/tests/test_pipeline/test_product.py new file mode 100644 index 0000000..72f5447 --- /dev/null +++ b/tests/test_pipeline/test_product.py @@ -0,0 +1,186 @@ +"""Tests for product pipeline orchestration (`pipeline/product.py`). + +Previously untested at the orchestration level (only per-tracker extract/clean +functions had unit tests) -- process_tracker_product is monkeypatched here so +these tests don't need real Excel trackers. +""" + +from datetime import date +from pathlib import Path + +import polars as pl + +from a4d.clean.schema_product import get_product_data_schema +from a4d.pipeline import product as product_pipeline +from a4d.pipeline.models import TrackerResult +from a4d.pipeline.product import process_product_tables, run_product_pipeline + + +def _write_cleaned_product_parquet(path: Path, released_to: str = "MY_SU001") -> Path: + schema = get_product_data_schema() + row = { + "product": "Insulin A", + "product_units_notes": None, + "product_entry_date": date(2024, 1, 15), + "product_units_released": 5.0, + "product_released_to": released_to, + "product_units_received": None, + "product_received_from": "HQ", + "product_balance": 10.0, + "product_units_returned": None, + "product_returned_by": None, + "product_table_month": 1, + "product_table_year": 2024, + "product_sheet_name": "Jan24", + "file_name": path.name, + "product_balance_status": "ok", + "product_category": "Insulin", + "orig_product_released_to": None, + "product_unit_capacity": 1, + "product_remarks": None, + "clinic_id": "SU", + } + pl.DataFrame([row], schema=schema).write_parquet(path) + return path + + +class TestProcessProductTables: + def test_no_cleaned_files_returns_empty(self, tmp_path: Path): + cleaned_dir = tmp_path / "product_data_cleaned" + cleaned_dir.mkdir() + output_dir = tmp_path / "tables" + + result = process_product_tables(cleaned_dir, output_dir) + + assert result == {} + + def test_creates_product_data_table(self, tmp_path: Path): + cleaned_dir = tmp_path / "product_data_cleaned" + cleaned_dir.mkdir() + _write_cleaned_product_parquet(cleaned_dir / "tracker_a_product_cleaned.parquet") + output_dir = tmp_path / "tables" + + result = process_product_tables(cleaned_dir, output_dir) + + assert "product_data" in result + assert result["product_data"].exists() + + +class TestRunProductPipeline: + def test_no_tracker_files_returns_empty_result(self, tmp_path: Path): + result = run_product_pipeline(tracker_files=[], output_root=tmp_path) + + assert result.total_trackers == 0 + assert result.success is True + + def test_sequential_processes_trackers_successfully(self, tmp_path: Path, monkeypatch): + tracker_file = tmp_path / "tracker_a.xlsx" + tracker_file.write_bytes(b"fake") + + def fake_process(tracker_file, output_root, mapper=None): + return TrackerResult( + tracker_file=tracker_file, + tracker_name=tracker_file.stem, + raw_output=None, + cleaned_output=None, + success=True, + cleaning_errors=0, + ) + + monkeypatch.setattr(product_pipeline, "process_tracker_product", fake_process) + + result = run_product_pipeline( + tracker_files=[tracker_file], + output_root=tmp_path, + skip_tables=True, + max_workers=1, + ) + + assert result.total_trackers == 1 + assert result.successful_trackers == 1 + assert result.failed_trackers == 0 + assert result.success is True + + def test_sequential_records_failed_tracker(self, tmp_path: Path, monkeypatch): + tracker_file = tmp_path / "tracker_bad.xlsx" + tracker_file.write_bytes(b"fake") + + def fake_process(tracker_file, output_root, mapper=None): + return TrackerResult( + tracker_file=tracker_file, + tracker_name=tracker_file.stem, + success=False, + error="boom", + ) + + monkeypatch.setattr(product_pipeline, "process_tracker_product", fake_process) + + result = run_product_pipeline( + tracker_files=[tracker_file], + output_root=tmp_path, + skip_tables=True, + max_workers=1, + ) + + assert result.failed_trackers == 1 + assert result.success is False + + def test_creates_tables_when_not_skipped(self, tmp_path: Path, monkeypatch): + tracker_file = tmp_path / "tracker_a.xlsx" + tracker_file.write_bytes(b"fake") + + def fake_process(tracker_file, output_root, mapper=None): + cleaned_dir = output_root / "product_data_cleaned" + cleaned_dir.mkdir(parents=True, exist_ok=True) + cleaned_output = cleaned_dir / f"{tracker_file.stem}_product_cleaned.parquet" + _write_cleaned_product_parquet(cleaned_output) + return TrackerResult( + tracker_file=tracker_file, + tracker_name=tracker_file.stem, + cleaned_output=cleaned_output, + success=True, + ) + + monkeypatch.setattr(product_pipeline, "process_tracker_product", fake_process) + + result = run_product_pipeline( + tracker_files=[tracker_file], + output_root=tmp_path, + skip_tables=False, + max_workers=1, + ) + + assert "product_data" in result.tables + assert result.tables["product_data"].exists() + + def test_clean_output_removes_existing_output_dirs(self, tmp_path: Path, monkeypatch): + raw_dir = tmp_path / "product_data_raw" + cleaned_dir = tmp_path / "product_data_cleaned" + raw_dir.mkdir() + cleaned_dir.mkdir() + (raw_dir / "stale_raw.parquet").write_bytes(b"stale") + (cleaned_dir / "stale_cleaned.parquet").write_bytes(b"stale") + + tables_dir = tmp_path / "tables" + tables_dir.mkdir() + stale_table = tables_dir / "product_data.parquet" + stale_table.write_bytes(b"stale") + + monkeypatch.setattr( + product_pipeline, + "process_tracker_product", + lambda tracker_file, output_root, mapper=None: TrackerResult( + tracker_file=tracker_file, tracker_name=tracker_file.stem, success=True + ), + ) + + run_product_pipeline( + tracker_files=[], + output_root=tmp_path, + clean_output=True, + skip_tables=True, + ) + + assert not raw_dir.exists() + assert not cleaned_dir.exists() + assert not stale_table.exists() diff --git a/tests/test_reference/__init__.py b/tests/test_reference/__init__.py new file mode 100644 index 0000000..54f1221 --- /dev/null +++ b/tests/test_reference/__init__.py @@ -0,0 +1 @@ +"""Tests for reference data loaders and validators.""" diff --git a/tests/test_reference/test_provinces.py b/tests/test_reference/test_provinces.py new file mode 100644 index 0000000..61eb58d --- /dev/null +++ b/tests/test_reference/test_provinces.py @@ -0,0 +1,248 @@ +"""Tests for province validation.""" + +from a4d.reference import ( + get_country_for_province, + is_valid_province, + load_allowed_provinces, + load_provinces_by_country, +) + + +class TestLoadAllowedProvinces: + """Tests for load_allowed_provinces function.""" + + def test_loads_provinces_from_yaml(self): + """Test that provinces are loaded from YAML file.""" + provinces = load_allowed_provinces() + + assert isinstance(provinces, list) + assert len(provinces) > 0 + assert all(isinstance(p, str) for p in provinces) + + def test_provinces_are_lowercased(self): + """Test that all provinces are lowercased for case-insensitive matching.""" + provinces = load_allowed_provinces() + + # All should be lowercase + assert all(p == p.lower() for p in provinces) + + def test_includes_known_provinces_lowercased(self): + """Test that known provinces are included (lowercased).""" + provinces = load_allowed_provinces() + + # Test samples from each country in the YAML (lowercased) + assert "bangkok" in provinces # Thailand + assert "vientiane" in provinces # Laos + assert "hà nội*" in provinces # Vietnam (note the asterisk) + assert "phnom penh" in provinces # Cambodia + assert "yangon region" in provinces # Myanmar + assert "kuala lumpur*" in provinces # Malaysia + + def test_returns_flattened_list(self): + """Test that provinces from all countries are in single list.""" + provinces = load_allowed_provinces() + provinces_by_country = load_provinces_by_country() + + # Count should match flattened version + expected_count = sum(len(provs) for provs in provinces_by_country.values()) + assert len(provinces) == expected_count + + def test_no_duplicates(self): + """Test that there are no duplicate provinces in the list.""" + provinces = load_allowed_provinces() + + assert len(provinces) == len(set(provinces)) + + +class TestLoadProvincesByCountry: + """Tests for load_provinces_by_country function.""" + + def test_loads_provinces_by_country(self): + """Test that provinces are organized by country.""" + provinces_by_country = load_provinces_by_country() + + assert isinstance(provinces_by_country, dict) + assert len(provinces_by_country) > 0 + + def test_provinces_are_lowercased(self): + """Test that all provinces are lowercased.""" + provinces_by_country = load_provinces_by_country() + + for _country, provinces in provinces_by_country.items(): + assert all(p == p.lower() for p in provinces) + + def test_includes_expected_countries(self): + """Test that expected countries are present.""" + provinces_by_country = load_provinces_by_country() + + expected_countries = [ + "THAILAND", + "LAOS", + "VIETNAM", + "CAMBODIA", + "MYANMAR", + "MALAYSIA", + ] + + for country in expected_countries: + assert country in provinces_by_country + assert len(provinces_by_country[country]) > 0 + + def test_thailand_provinces(self): + """Test that Thailand has correct number of provinces.""" + provinces_by_country = load_provinces_by_country() + + thailand_provinces = provinces_by_country["THAILAND"] + + # Thailand has 72 provinces in the data file + assert len(thailand_provinces) == 72 + assert "bangkok" in thailand_provinces + assert "chiang mai" in thailand_provinces + assert "phuket" in thailand_provinces + + +class TestIsValidProvince: + """Tests for is_valid_province function.""" + + def test_valid_province_returns_true(self): + """Test that valid provinces return True.""" + assert is_valid_province("Bangkok") + assert is_valid_province("Vientiane") + assert is_valid_province("Hà Nội*") + assert is_valid_province("Phnom Penh") + + def test_invalid_province_returns_false(self): + """Test that invalid provinces return False.""" + assert not is_valid_province("Invalid Province") + assert not is_valid_province("Unknown City") + assert not is_valid_province("Test") + + def test_none_returns_true(self): + """Test that None is considered valid (nullable field).""" + assert is_valid_province(None) + + def test_empty_string_returns_false(self): + """Test that empty string is invalid.""" + assert not is_valid_province("") + + def test_case_insensitive(self): + """Test that validation is case-insensitive.""" + assert is_valid_province("Bangkok") + assert is_valid_province("bangkok") + assert is_valid_province("BANGKOK") + assert is_valid_province("BaNgKoK") + + def test_unicode_provinces(self): + """Test that Unicode province names work correctly.""" + # Vietnam has many provinces with Unicode characters + assert is_valid_province("Hà Nội*") + assert is_valid_province("Hồ Chí Minh*") + assert is_valid_province("Bà Rịa–Vũng Tàu") + assert is_valid_province("Đà Nẵng*") + + # Case variations + assert is_valid_province("HÀ NỘI*") + assert is_valid_province("hà nội*") + + +class TestGetCountryForProvince: + """Tests for get_country_for_province function.""" + + def test_returns_correct_country(self): + """Test that correct country is returned for provinces.""" + assert get_country_for_province("Bangkok") == "THAILAND" + assert get_country_for_province("Vientiane") == "LAOS" + assert get_country_for_province("Hà Nội*") == "VIETNAM" + assert get_country_for_province("Phnom Penh") == "CAMBODIA" + assert get_country_for_province("Yangon Region") == "MYANMAR" + assert get_country_for_province("Kuala Lumpur*") == "MALAYSIA" + + def test_returns_none_for_invalid_province(self): + """Test that None is returned for invalid provinces.""" + assert get_country_for_province("Invalid Province") is None + assert get_country_for_province("Unknown") is None + + def test_case_insensitive(self): + """Test that lookup is case-insensitive.""" + assert get_country_for_province("Bangkok") == "THAILAND" + assert get_country_for_province("bangkok") == "THAILAND" + assert get_country_for_province("BANGKOK") == "THAILAND" + assert get_country_for_province("BaNgKoK") == "THAILAND" + + def test_multiple_provinces_same_country(self): + """Test that different provinces from same country work.""" + # All should return THAILAND + assert get_country_for_province("Bangkok") == "THAILAND" + assert get_country_for_province("Chiang Mai") == "THAILAND" + assert get_country_for_province("Phuket") == "THAILAND" + + def test_unicode_provinces(self): + """Test that Unicode provinces work correctly.""" + assert get_country_for_province("Hà Nội*") == "VIETNAM" + assert get_country_for_province("hà nội*") == "VIETNAM" + assert get_country_for_province("HÀ NỘI*") == "VIETNAM" + + +class TestIntegrationWithActualData: + """Integration tests with actual reference_data file.""" + + def test_all_countries_have_provinces(self): + """Test that every country has at least one province.""" + provinces_by_country = load_provinces_by_country() + + for country, provinces in provinces_by_country.items(): + assert len(provinces) > 0, f"{country} has no provinces" + + def test_total_province_count(self): + """Test that total province count is reasonable.""" + provinces = load_allowed_provinces() + + # We expect 200+ provinces across all countries + assert len(provinces) > 200 + + def test_no_empty_province_names(self): + """Test that no province names are empty strings.""" + provinces = load_allowed_provinces() + + assert all(p.strip() for p in provinces) + + def test_round_trip_validation(self): + """Test that all loaded provinces pass validation.""" + provinces = load_allowed_provinces() + + for province in provinces: + assert is_valid_province(province) + country = get_country_for_province(province) + assert country is not None + + def test_special_characters_preserved(self): + """Test that special characters in province names are preserved.""" + provinces = load_allowed_provinces() + + # Vietnam provinces with Unicode (lowercased) + unicode_provinces = [p for p in provinces if any(ord(c) > 127 for c in p)] + assert len(unicode_provinces) > 0 + + # Provinces with asterisks (indicating cities, lowercased) + asterisk_provinces = [p for p in provinces if "*" in p] + assert len(asterisk_provinces) > 0 + + def test_case_insensitive_validation_comprehensive(self): + """Test case-insensitive validation with various cases.""" + provinces_by_country = load_provinces_by_country() + + # Get a few provinces from the data + provinces_by_country["THAILAND"] + vietnam = provinces_by_country["VIETNAM"] + + # Test that both original case and variations work + # (provinces are stored lowercase, so we test against "bangkok") + assert is_valid_province("Bangkok") # Title case + assert is_valid_province("BANGKOK") # Upper case + assert is_valid_province("bangkok") # Lower case + + # Test with Vietnamese provinces + test_province = vietnam[0] # Get first province + assert is_valid_province(test_province) + assert is_valid_province(test_province.upper()) + assert is_valid_province(test_province.title()) diff --git a/tests/test_reference/test_synonyms.py b/tests/test_reference/test_synonyms.py new file mode 100644 index 0000000..e8d7bf8 --- /dev/null +++ b/tests/test_reference/test_synonyms.py @@ -0,0 +1,386 @@ +"""Tests for column synonym mapper.""" + +from pathlib import Path + +import polars as pl +import pytest +import yaml + +from a4d.reference import ColumnMapper, load_patient_mapper, load_product_mapper +from a4d.reference.synonyms import sanitize_str + + +class TestSanitizeStr: + """Tests for sanitize_str function.""" + + def test_basic_sanitization(self): + """Test basic sanitization cases.""" + assert sanitize_str("Patient ID") == "patientid" + assert sanitize_str("Patient ID*") == "patientid" + assert sanitize_str("Age* On Reporting") == "ageonreporting" + + def test_lowercase_conversion(self): + """Test lowercase conversion.""" + assert sanitize_str("PATIENT ID") == "patientid" + assert sanitize_str("Patient Name") == "patientname" + + def test_space_removal(self): + """Test space removal.""" + assert sanitize_str("Date 2022") == "date2022" + assert sanitize_str("My Awesome Column") == "myawesomecolumn" + + def test_special_character_removal(self): + """Test special character removal.""" + assert sanitize_str("Patient ID*") == "patientid" + assert sanitize_str("My Awesome 1st Column!!") == "myawesome1stcolumn" + assert sanitize_str("D.O.B.") == "dob" + assert sanitize_str("Age (Years)") == "ageyears" + assert sanitize_str("Patient.Name..ANON") == "patientnameanon" + + def test_alphanumeric_preserved(self): + """Test that alphanumeric characters are preserved.""" + assert sanitize_str("Age1") == "age1" + assert sanitize_str("test123abc") == "test123abc" + + def test_empty_string(self): + """Test empty string.""" + assert sanitize_str("") == "" + + def test_only_special_chars(self): + """Test string with only special characters.""" + assert sanitize_str("***!!!") == "" + assert sanitize_str("...") == "" + + +class TestColumnMapper: + """Tests for ColumnMapper class.""" + + @pytest.fixture + def simple_synonyms(self, tmp_path: Path) -> Path: + """Create a simple synonym YAML file for testing.""" + synonyms = { + "age": ["Age", "Age*", "age on reporting"], + "patient_id": ["ID", "Patient ID", "Patient ID*"], + "name": ["Patient Name"], + "province": ["Province"], + "empty_column": [], # Column with no synonyms + } + + yaml_path = tmp_path / "test_synonyms.yaml" + with open(yaml_path, "w") as f: + yaml.dump(synonyms, f) + + return yaml_path + + @pytest.fixture + def duplicate_synonyms(self, tmp_path: Path) -> Path: + """Create synonym YAML with duplicate synonyms.""" + synonyms = { + "age": ["Age", "Years"], + "age_at_diagnosis": ["Age", "Age at diagnosis"], # "Age" duplicated + } + + yaml_path = tmp_path / "test_duplicates.yaml" + with open(yaml_path, "w") as f: + yaml.dump(synonyms, f) + + return yaml_path + + def test_init_loads_synonyms(self, simple_synonyms: Path): + """Test that __init__ loads synonyms from YAML file.""" + mapper = ColumnMapper(simple_synonyms) + + assert len(mapper.synonyms) == 5 + assert "age" in mapper.synonyms + assert "Age" in mapper.synonyms["age"] + # After sanitization, some synonyms collapse (e.g., "Age" and "Age*" both become "age") + assert ( + len(mapper._lookup) == 6 + ) # Sanitized synonyms (age+ageonreporting+id+patientid+patientname+province) + + def test_init_missing_file_raises_error(self): + """Test that __init__ raises error for missing file.""" + with pytest.raises(FileNotFoundError, match="YAML file not found"): + ColumnMapper(Path("/nonexistent/file.yaml")) + + def test_build_lookup_creates_reverse_mapping(self, simple_synonyms: Path): + """Test that reverse lookup is built correctly with SANITIZED keys.""" + mapper = ColumnMapper(simple_synonyms) + + # Lookup uses sanitized keys (lowercase, no spaces, no special chars) + assert mapper._lookup["age"] == "age" # "Age" and "Age*" both sanitize to "age" + assert mapper._lookup["ageonreporting"] == "age" # "age on reporting" → "ageonreporting" + assert mapper._lookup["id"] == "patient_id" # "ID" → "id" + assert ( + mapper._lookup["patientid"] == "patient_id" + ) # "Patient ID" and "Patient ID*" → "patientid" + + def test_build_lookup_handles_duplicates(self, duplicate_synonyms: Path): + """Test that duplicate SANITIZED synonyms log warning and use last definition.""" + mapper = ColumnMapper(duplicate_synonyms) + + # "Age" appears in both age and age_at_diagnosis + # After sanitization, both become "age" → duplicate! + # Should map to the last one encountered + assert "age" in mapper._lookup + assert mapper._lookup["age"] in ["age", "age_at_diagnosis"] + + def test_get_standard_name(self, simple_synonyms: Path): + """Test getting standard name for a column.""" + mapper = ColumnMapper(simple_synonyms) + + assert mapper.get_standard_name("Age") == "age" + assert mapper.get_standard_name("Patient ID*") == "patient_id" + assert mapper.get_standard_name("unknown_column") == "unknown_column" + + def test_get_standard_name_with_sanitization(self, simple_synonyms: Path): + """Test that sanitization allows flexible synonym matching.""" + mapper = ColumnMapper(simple_synonyms) + + # All these variants should map to "patient_id" after sanitization + assert mapper.get_standard_name("Patient ID") == "patient_id" + assert mapper.get_standard_name("Patient ID*") == "patient_id" + assert mapper.get_standard_name("PATIENT ID") == "patient_id" + assert mapper.get_standard_name("patient id") == "patient_id" + assert mapper.get_standard_name("ID") == "patient_id" + + # Age variants + assert mapper.get_standard_name("Age") == "age" + assert mapper.get_standard_name("Age*") == "age" + assert mapper.get_standard_name("age on reporting") == "age" + assert mapper.get_standard_name("AGE ON REPORTING") == "age" + + # Test with extra spaces/special chars (should still match) + assert mapper.get_standard_name("Patient ID*") == "patient_id" + + def test_rename_columns_basic(self, simple_synonyms: Path): + """Test basic column renaming.""" + mapper = ColumnMapper(simple_synonyms) + + df = pl.DataFrame( + { + "Age": [25, 30], + "Patient ID": ["P001", "P002"], + "Province": ["Bangkok", "Hanoi"], + } + ) + + renamed = mapper.rename_columns(df) + + assert "age" in renamed.columns + assert "patient_id" in renamed.columns + assert "province" in renamed.columns + assert "Age" not in renamed.columns + + def test_rename_columns_keeps_unmapped(self, simple_synonyms: Path): + """Test that unmapped columns are kept by default.""" + mapper = ColumnMapper(simple_synonyms) + + df = pl.DataFrame( + { + "Age": [25], + "UnknownColumn": ["value"], + "AnotherUnmapped": [42], + } + ) + + renamed = mapper.rename_columns(df) + + assert "age" in renamed.columns + assert "UnknownColumn" in renamed.columns + assert "AnotherUnmapped" in renamed.columns + + def test_rename_columns_merges_duplicate_targets(self, simple_synonyms: Path): + """Several source columns mapping to one canonical name are merged, not dropped. + + The 2023 template splits complication screening into B.P./Kidney/Eye/Foot/ + Lipids sub-columns that all map to `complication_screening`; each carries an + independent value, so keeping only the first silently discards the rest. + """ + mapper = ColumnMapper(simple_synonyms) + + df = pl.DataFrame( + { + "Age": [25, 30, 35], + "Patient ID": ["P001", None, "P003"], + "ID": [None, "P002", "P003b"], + } + ) + + renamed = mapper.rename_columns(df) + + assert list(renamed.columns) == ["age", "patient_id"] + assert renamed["patient_id"].to_list() == ["P001", "P002", "P003,P003b"] + + def test_rename_columns_duplicate_targets_all_empty_gives_null(self, simple_synonyms: Path): + """A merged group with nothing populated is null, not an empty string.""" + mapper = ColumnMapper(simple_synonyms) + + df = pl.DataFrame({"Patient ID": [None, ""], "ID": ["", None]}) + + renamed = mapper.rename_columns(df) + + assert renamed["patient_id"].to_list() == [None, None] + + def test_rename_columns_merges_non_string_duplicate_targets(self, simple_synonyms: Path): + """Merging casts to string, so numeric source columns do not raise.""" + mapper = ColumnMapper(simple_synonyms) + + df = pl.DataFrame({"Age": [25, None], "Age*": [None, 30]}) + + renamed = mapper.rename_columns(df) + + assert renamed["age"].to_list() == ["25", "30"] + + def test_rename_columns_strict_mode_raises_error(self, simple_synonyms: Path): + """Test that strict mode raises error for unmapped columns.""" + mapper = ColumnMapper(simple_synonyms) + + df = pl.DataFrame( + { + "Age": [25], + "UnknownColumn": ["value"], + } + ) + + with pytest.raises(ValueError, match="Unmapped columns found"): + mapper.rename_columns(df, strict=True) + + def test_rename_columns_no_changes_needed(self, simple_synonyms: Path): + """Test renaming when columns are already standardized.""" + mapper = ColumnMapper(simple_synonyms) + + df = pl.DataFrame( + { + "age": [25], + "patient_id": ["P001"], + } + ) + + renamed = mapper.rename_columns(df) + + assert renamed.columns == df.columns + assert renamed.equals(df) + + def test_get_expected_columns(self, simple_synonyms: Path): + """Test getting set of expected standard columns.""" + mapper = ColumnMapper(simple_synonyms) + + expected = mapper.get_expected_columns() + + assert expected == {"age", "patient_id", "name", "province", "empty_column"} + + def test_get_missing_columns(self, simple_synonyms: Path): + """Test getting missing columns from DataFrame.""" + mapper = ColumnMapper(simple_synonyms) + + df = pl.DataFrame( + { + "age": [25], + "patient_id": ["P001"], + } + ) + + missing = mapper.get_missing_columns(df) + + assert missing == {"name", "province", "empty_column"} + + def test_validate_required_columns_success(self, simple_synonyms: Path): + """Test validation passes when required columns present.""" + mapper = ColumnMapper(simple_synonyms) + + df = pl.DataFrame( + { + "age": [25], + "patient_id": ["P001"], + "name": ["Test"], + } + ) + + # Should not raise + mapper.validate_required_columns(df, ["age", "patient_id"]) + + def test_validate_required_columns_failure(self, simple_synonyms: Path): + """Test validation fails when required columns missing.""" + mapper = ColumnMapper(simple_synonyms) + + df = pl.DataFrame( + { + "age": [25], + } + ) + + with pytest.raises(ValueError, match="Required columns missing"): + mapper.validate_required_columns(df, ["age", "patient_id", "name"]) + + +class TestLoaderFunctions: + """Tests for loader convenience functions.""" + + def test_load_patient_mapper_with_actual_file(self): + """Test loading patient mapper with actual reference_data file.""" + mapper = load_patient_mapper() + + # Check that some expected columns are present + assert "age" in mapper.synonyms + assert "patient_id" in mapper.synonyms + assert "province" in mapper.synonyms + + # Check that synonyms are loaded + assert len(mapper._lookup) > 0 + assert mapper.get_standard_name("Age") == "age" + + def test_load_product_mapper_with_actual_file(self): + """Test loading product mapper with actual reference_data file.""" + mapper = load_product_mapper() + + # Check that some expected columns are present + assert "product" in mapper.synonyms + assert "clinic_id" in mapper.synonyms + + # Check that synonyms are loaded + assert len(mapper._lookup) > 0 + + +class TestIntegrationWithActualData: + """Integration tests with actual reference_data files.""" + + def test_patient_mapper_renames_all_known_synonyms(self): + """Test that patient mapper can rename all synonyms in YAML.""" + mapper = load_patient_mapper() + + # Create DataFrame with various synonyms + test_data = { + "Age": [25], + "Patient ID": ["P001"], + "D.O.B.": ["1999-01-01"], + "Gender": ["M"], + } + + df = pl.DataFrame(test_data) + renamed = mapper.rename_columns(df) + + # Check that columns are renamed correctly + assert "age" in renamed.columns + assert "patient_id" in renamed.columns + assert "dob" in renamed.columns + assert "sex" in renamed.columns + + def test_product_mapper_renames_all_known_synonyms(self): + """Test that product mapper can rename all synonyms in YAML.""" + mapper = load_product_mapper() + + # Create DataFrame with various synonyms + test_data = { + "Product": ["Insulin"], + "Date": ["2024-01-01"], + "Units Received": [10], + } + + df = pl.DataFrame(test_data) + renamed = mapper.rename_columns(df) + + # Check that columns are renamed correctly + assert "product" in renamed.columns + assert "product_entry_date" in renamed.columns + assert "product_units_received" in renamed.columns diff --git a/tests/test_state/__init__.py b/tests/test_state/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_state/test_filter.py b/tests/test_state/test_filter.py new file mode 100644 index 0000000..99402f8 --- /dev/null +++ b/tests/test_state/test_filter.py @@ -0,0 +1,120 @@ +"""Tests for filter_unchanged_trackers.""" + +import hashlib +from pathlib import Path + +from a4d.state.filter import filter_unchanged_trackers +from a4d.state.manifest import Manifest, ManifestEntry + + +def _md5(payload: bytes) -> str: + return hashlib.md5(payload, usedforsecurity=False).hexdigest() + + +def _make_tracker(parent: Path, clinic: str, name: str, payload: bytes) -> Path: + folder = parent / clinic + folder.mkdir(parents=True, exist_ok=True) + path = folder / f"{name}.xlsx" + path.write_bytes(payload) + return path + + +def test_empty_manifest_queues_everything(tmp_path: Path): + t1 = _make_tracker(tmp_path, "A", "2024_T1", b"alpha") + t2 = _make_tracker(tmp_path, "B", "2024_T2", b"beta") + + queued, summary = filter_unchanged_trackers([t1, t2], Manifest.empty()) + + assert queued == [t1, t2] + assert summary.queued == 2 + assert summary.skipped == 0 + assert summary.new == 2 + assert summary.changed == 0 + assert summary.previously_incomplete == 0 + + +def test_all_match_skips_everything(tmp_path: Path): + t1 = _make_tracker(tmp_path, "A", "2024_T1", b"alpha") + t2 = _make_tracker(tmp_path, "B", "2024_T2", b"beta") + manifest = Manifest( + entries={ + ("A", "2024_T1"): ManifestEntry(md5=_md5(b"alpha"), complete=True), + ("B", "2024_T2"): ManifestEntry(md5=_md5(b"beta"), complete=True), + } + ) + + queued, summary = filter_unchanged_trackers([t1, t2], manifest) + + assert queued == [] + assert summary.queued == 0 + assert summary.skipped == 2 + + +def test_changed_md5_queues(tmp_path: Path): + t1 = _make_tracker(tmp_path, "A", "2024_T1", b"new content") + manifest = Manifest( + entries={("A", "2024_T1"): ManifestEntry(md5=_md5(b"old content"), complete=True)}, + ) + + queued, summary = filter_unchanged_trackers([t1], manifest) + + assert queued == [t1] + assert summary.changed == 1 + assert summary.new == 0 + + +def test_previously_incomplete_queues(tmp_path: Path): + """complete=False means the previous run didn't finish all four output stages.""" + t1 = _make_tracker(tmp_path, "A", "2024_T1", b"alpha") + manifest = Manifest( + entries={("A", "2024_T1"): ManifestEntry(md5=_md5(b"alpha"), complete=False)}, + ) + + queued, summary = filter_unchanged_trackers([t1], manifest) + + assert queued == [t1] + assert summary.previously_incomplete == 1 + assert summary.changed == 0 + assert summary.new == 0 + + +def test_mixed_classification(tmp_path: Path): + new = _make_tracker(tmp_path, "A", "2024_NEW", b"new") + changed = _make_tracker(tmp_path, "A", "2024_CHANGED", b"new bytes") + incomplete = _make_tracker(tmp_path, "A", "2024_INCOMPLETE", b"same") + unchanged = _make_tracker(tmp_path, "A", "2024_UNCHANGED", b"frozen") + + manifest = Manifest( + entries={ + ("A", "2024_CHANGED"): ManifestEntry(md5=_md5(b"old bytes"), complete=True), + ("A", "2024_INCOMPLETE"): ManifestEntry(md5=_md5(b"same"), complete=False), + ("A", "2024_UNCHANGED"): ManifestEntry(md5=_md5(b"frozen"), complete=True), + } + ) + + queued, summary = filter_unchanged_trackers([new, changed, incomplete, unchanged], manifest) + + assert set(queued) == {new, changed, incomplete} + assert summary.queued == 3 + assert summary.skipped == 1 + assert summary.new == 1 + assert summary.changed == 1 + assert summary.previously_incomplete == 1 + + +def test_clinic_isolation(tmp_path: Path): + """Same file stem in two clinic folders must be treated as two distinct trackers.""" + t_a = _make_tracker(tmp_path, "CLINIC_A", "shared_name", b"clinic A bytes") + t_b = _make_tracker(tmp_path, "CLINIC_B", "shared_name", b"clinic B bytes") + + manifest = Manifest( + entries={ + ("CLINIC_A", "shared_name"): ManifestEntry(md5=_md5(b"clinic A bytes"), complete=True), + ("CLINIC_B", "shared_name"): ManifestEntry(md5=_md5(b"clinic B bytes"), complete=True), + } + ) + + queued, summary = filter_unchanged_trackers([t_a, t_b], manifest) + + assert queued == [] + assert summary.skipped == 2 diff --git a/tests/test_state/test_integration.py b/tests/test_state/test_integration.py new file mode 100644 index 0000000..79ab788 --- /dev/null +++ b/tests/test_state/test_integration.py @@ -0,0 +1,101 @@ +"""Producer/consumer round-trip for the incremental-processing manifest. + +Avoids invoking the patient/product orchestrators (which need real .xlsx +structure) — instead exercises the metadata builder → manifest loader → filter +pipeline against synthetic byte-blobs that look like trackers on disk. +""" + +from pathlib import Path + +import pytest + +from a4d.state.filter import filter_unchanged_trackers +from a4d.state.source import load_previous_manifest +from a4d.tables.metadata import create_table_tracker_metadata + + +@pytest.fixture +def fake_pipeline_run(tmp_path: Path): + """Create a 2-tracker fake data_root + fully-populated output subdirs.""" + data_root = tmp_path / "data" + output_root = tmp_path / "output" + + (data_root / "CLINIC_A").mkdir(parents=True) + t1 = data_root / "CLINIC_A" / "2024_T1_Tracker.xlsx" + t1.write_bytes(b"tracker one bytes") + t2 = data_root / "CLINIC_A" / "2024_T2_Tracker.xlsx" + t2.write_bytes(b"tracker two bytes") + + # Simulate a successful run: every tracker has all four output stages. + for subdir in ( + "patient_data_cleaned", + "patient_data_raw", + "product_data_cleaned", + "product_data_raw", + ): + (output_root / subdir).mkdir(parents=True) + for stem in ("2024_T1_Tracker", "2024_T2_Tracker"): + (output_root / subdir / f"{stem}_dummy.parquet").write_bytes(b"") + + # Publish the manifest — same call the `run` CLI command makes at end of run. + create_table_tracker_metadata(data_root, output_root) + + return {"data_root": data_root, "output_root": output_root, "t1": t1, "t2": t2} + + +def test_published_manifest_skips_all_unchanged(fake_pipeline_run): + """Round-trip 1: metadata published as complete=True ⇒ next run queues 0.""" + data_root = fake_pipeline_run["data_root"] + output_root = fake_pipeline_run["output_root"] + + discovered = sorted(data_root.rglob("*.xlsx")) + manifest = load_previous_manifest(output_root, prefer_bigquery=False) + + queued, summary = filter_unchanged_trackers(discovered, manifest) + + assert queued == [] + assert summary.skipped == 2 + assert summary.queued == 0 + + +def test_mutated_tracker_queues_exactly_one(fake_pipeline_run): + """Round-trip 2: change one tracker's bytes ⇒ filter queues exactly it.""" + data_root = fake_pipeline_run["data_root"] + output_root = fake_pipeline_run["output_root"] + t1 = fake_pipeline_run["t1"] + + # Mutate t1's bytes — t2 stays untouched. + t1.write_bytes(b"tracker one with new bytes") + + discovered = sorted(data_root.rglob("*.xlsx")) + manifest = load_previous_manifest(output_root, prefer_bigquery=False) + queued, summary = filter_unchanged_trackers(discovered, manifest) + + assert queued == [t1] + assert summary.changed == 1 + assert summary.skipped == 1 + + +def test_missing_output_stage_marks_incomplete_and_requeues(tmp_path: Path): + """If product_data_cleaned/ is missing for a tracker, complete=False ⇒ requeue.""" + data_root = tmp_path / "data" + output_root = tmp_path / "output" + + (data_root / "CLINIC_A").mkdir(parents=True) + t1 = data_root / "CLINIC_A" / "2024_T1_Tracker.xlsx" + t1.write_bytes(b"alpha") + + # Only three of four stages have output — simulates a previous run that + # crashed during the product arm. + for subdir in ("patient_data_cleaned", "patient_data_raw", "product_data_raw"): + (output_root / subdir).mkdir(parents=True) + (output_root / subdir / "2024_T1_Tracker_dummy.parquet").write_bytes(b"") + + create_table_tracker_metadata(data_root, output_root) + + manifest = load_previous_manifest(output_root, prefer_bigquery=False) + queued, summary = filter_unchanged_trackers([t1], manifest) + + assert queued == [t1] + assert summary.previously_incomplete == 1 + assert summary.changed == 0 diff --git a/tests/test_state/test_manifest.py b/tests/test_state/test_manifest.py new file mode 100644 index 0000000..57dd087 --- /dev/null +++ b/tests/test_state/test_manifest.py @@ -0,0 +1,39 @@ +"""Tests for the Manifest dataclasses.""" + +from a4d.state.manifest import Manifest, ManifestEntry + + +def test_empty_manifest(): + m = Manifest.empty() + assert len(m) == 0 + assert m.get("CLINIC_A", "tracker") is None + + +def test_lookup_by_composite_key(): + entries = { + ("CLINIC_A", "2024_T1"): ManifestEntry(md5="abc123", complete=True), + ("CLINIC_B", "2024_T1"): ManifestEntry(md5="def456", complete=False), + } + m = Manifest(entries=entries) + + assert m.get("CLINIC_A", "2024_T1") == ManifestEntry(md5="abc123", complete=True) + assert m.get("CLINIC_B", "2024_T1") == ManifestEntry(md5="def456", complete=False) + # Same file_name in a third clinic must miss — composite key isolates by clinic. + assert m.get("CLINIC_C", "2024_T1") is None + assert len(m) == 2 + + +def test_manifest_entry_is_frozen(): + """Frozen dataclass: equality + immutability are part of the contract.""" + e1 = ManifestEntry(md5="abc", complete=True) + e2 = ManifestEntry(md5="abc", complete=True) + assert e1 == e2 + + import dataclasses + + try: + e1.md5 = "different" # type: ignore[misc] + except dataclasses.FrozenInstanceError: + pass + else: + raise AssertionError("ManifestEntry must be frozen") diff --git a/tests/test_state/test_source.py b/tests/test_state/test_source.py new file mode 100644 index 0000000..2731bb5 --- /dev/null +++ b/tests/test_state/test_source.py @@ -0,0 +1,87 @@ +"""Tests for load_previous_manifest source-precedence logic.""" + +from pathlib import Path +from unittest.mock import patch + +import polars as pl + +from a4d.state.manifest import Manifest +from a4d.state.source import load_previous_manifest + + +def _manifest_df() -> pl.DataFrame: + return pl.DataFrame( + { + "file_name": ["2024_T1", "2024_T2"], + "clinic_code": ["A", "B"], + "md5": ["abc", "def"], + "complete": [True, False], + } + ) + + +def test_bigquery_first_when_available(tmp_path: Path): + """BQ wins over local parquet when prefer_bigquery=True and BQ returns rows.""" + # Both sources populated; BQ should be picked. + local = tmp_path / "tables" / "tracker_metadata.parquet" + local.parent.mkdir(parents=True) + pl.DataFrame( + { + "file_name": ["LOCAL_ONLY"], + "clinic_code": ["X"], + "md5": ["local_md5"], + "complete": [True], + } + ).write_parquet(local) + + with patch("a4d.gcp.bigquery.select_tracker_metadata", return_value=_manifest_df()): + m = load_previous_manifest(tmp_path) + + assert len(m) == 2 + assert m.get("A", "2024_T1").md5 == "abc" + # Local parquet was NOT consulted. + assert m.get("X", "LOCAL_ONLY") is None + + +def test_falls_back_to_local_parquet_when_bq_returns_none(tmp_path: Path): + local = tmp_path / "tables" / "tracker_metadata.parquet" + local.parent.mkdir(parents=True) + _manifest_df().write_parquet(local) + + with patch("a4d.gcp.bigquery.select_tracker_metadata", return_value=None): + m = load_previous_manifest(tmp_path) + + assert len(m) == 2 + assert m.get("A", "2024_T1").complete is True + assert m.get("B", "2024_T2").complete is False + + +def test_returns_empty_when_neither_source_available(tmp_path: Path): + with patch("a4d.gcp.bigquery.select_tracker_metadata", return_value=None): + m = load_previous_manifest(tmp_path) + + assert m == Manifest.empty() + + +def test_skips_bigquery_when_prefer_bigquery_false(tmp_path: Path): + local = tmp_path / "tables" / "tracker_metadata.parquet" + local.parent.mkdir(parents=True) + _manifest_df().write_parquet(local) + + # patch should never be called. + with patch("a4d.gcp.bigquery.select_tracker_metadata") as mock_bq: + m = load_previous_manifest(tmp_path, prefer_bigquery=False) + assert mock_bq.call_count == 0 + + assert len(m) == 2 + + +def test_handles_corrupt_local_parquet(tmp_path: Path): + local = tmp_path / "tables" / "tracker_metadata.parquet" + local.parent.mkdir(parents=True) + local.write_bytes(b"not a parquet file") + + with patch("a4d.gcp.bigquery.select_tracker_metadata", return_value=None): + m = load_previous_manifest(tmp_path) + + assert m == Manifest.empty() diff --git a/tests/test_tables/test_link_product_patient.py b/tests/test_tables/test_link_product_patient.py new file mode 100644 index 0000000..3342462 --- /dev/null +++ b/tests/test_tables/test_link_product_patient.py @@ -0,0 +1,148 @@ +"""Tests for `link_product_patient` — product↔patient link validation.""" + +from collections.abc import Iterator +from pathlib import Path + +import polars as pl +import pytest +from loguru import logger + +from a4d.tables.product import link_product_patient + + +@pytest.fixture +def captured_warnings() -> Iterator[list[str]]: + """Capture loguru WARNING messages emitted during the test.""" + sink: list[str] = [] + handler_id = logger.add( + lambda msg: sink.append(str(msg)), + level="WARNING", + format="{message}", + ) + yield sink + logger.remove(handler_id) + + +def _write_patient_static(path: Path, rows: list[tuple[str, str]]) -> Path: + """Write a minimal patient_data_static.parquet with (file_name, patient_id) rows.""" + pl.DataFrame( + {"file_name": [r[0] for r in rows], "patient_id": [r[1] for r in rows]} + ).write_parquet(path) + return path + + +def test_all_match_returns_zero_no_warnings(tmp_path: Path, captured_warnings: list[str]) -> None: + patient_path = _write_patient_static( + tmp_path / "patient.parquet", + [ + ("tracker_a.xlsx", "KD_EW001"), + ("tracker_a.xlsx", "KD_EW002"), + ("tracker_b.xlsx", "KD_EW003"), + ], + ) + product_df = pl.DataFrame( + { + "file_name": [ + "tracker_a.xlsx", + "tracker_a.xlsx", + "tracker_a.xlsx", + "tracker_b.xlsx", + "tracker_b.xlsx", + ], + "product_released_to": [ + "KD_EW001", + "KD_EW001", + "KD_EW002", + "KD_EW003", + "KD_EW003", + ], + } + ) + + count = link_product_patient(product_df, patient_path) + + assert count == 0 + assert captured_warnings == [] + + +def test_mixed_filters_null_and_sentinel(tmp_path: Path, captured_warnings: list[str]) -> None: + patient_path = _write_patient_static( + tmp_path / "patient.parquet", + [("tracker_a.xlsx", "KD_EW001"), ("tracker_a.xlsx", "KD_EW002")], + ) + product_df = pl.DataFrame( + { + "file_name": [ + "tracker_a.xlsx", # match + "tracker_a.xlsx", # match + "tracker_a.xlsx", # match + "tracker_a.xlsx", # mismatch + "tracker_a.xlsx", # mismatch + "tracker_a.xlsx", # null — filtered + "tracker_a.xlsx", # "Undefined" — filtered + ], + "product_released_to": [ + "KD_EW001", + "KD_EW001", + "KD_EW002", + "KD_EW999", + "KD_EW999", + None, + "Undefined", + ], + } + ) + + count = link_product_patient(product_df, patient_path) + + assert count == 2 + # Per-pair detail ("Unmatched product_released_to") is DEBUG-only now — + # console/WARNING-level output gets one aggregate line instead, so a run + # with hundreds of distinct mismatched pairs doesn't flood the terminal. + mismatch_warnings = [w for w in captured_warnings if "Unmatched product_released_to" in w] + assert mismatch_warnings == [] + summary_warnings = [w for w in captured_warnings if "Product-patient link validation" in w] + assert len(summary_warnings) == 1 + assert "2 mismatched rows" in summary_warnings[0] + assert "1 distinct (file × id) pairs" in summary_warnings[0] + assert "5 candidate product rows examined" in summary_warnings[0] + + +def test_cross_file_isolation(tmp_path: Path, captured_warnings: list[str]) -> None: + """Same patient_id matches in one file but not in another.""" + patient_path = _write_patient_static( + tmp_path / "patient.parquet", + [("tracker_a.xlsx", "KD_EW001")], + ) + product_df = pl.DataFrame( + { + "file_name": ["tracker_a.xlsx", "tracker_b.xlsx"], + "product_released_to": ["KD_EW001", "KD_EW001"], + } + ) + + count = link_product_patient(product_df, patient_path) + + assert count == 1 + mismatch_warnings = [w for w in captured_warnings if "Unmatched product_released_to" in w] + assert mismatch_warnings == [] + summary_warnings = [w for w in captured_warnings if "Product-patient link validation" in w] + assert len(summary_warnings) == 1 + assert "1 mismatched rows" in summary_warnings[0] + assert "1 distinct (file × id) pairs" in summary_warnings[0] + assert "2 candidate product rows examined" in summary_warnings[0] + + +def test_missing_patient_table_returns_zero_with_warning( + tmp_path: Path, captured_warnings: list[str] +) -> None: + product_df = pl.DataFrame( + {"file_name": ["tracker_a.xlsx"], "product_released_to": ["KD_EW001"]} + ) + missing_path = tmp_path / "does_not_exist.parquet" + + count = link_product_patient(product_df, missing_path) + + assert count == 0 + skip_warnings = [w for w in captured_warnings if "skipping" in w.lower()] + assert len(skip_warnings) == 1 diff --git a/tests/test_tables/test_metadata.py b/tests/test_tables/test_metadata.py new file mode 100644 index 0000000..2157b0d --- /dev/null +++ b/tests/test_tables/test_metadata.py @@ -0,0 +1,160 @@ +"""Tests for tracker metadata table generation.""" + +import hashlib +from pathlib import Path + +import polars as pl +import pytest + +from a4d.tables.metadata import create_table_tracker_metadata + + +@pytest.fixture +def fake_trackers(tmp_path: Path) -> Path: + """Create a fake data_root with two clinic subfolders and three trackers.""" + data_root = tmp_path / "data" + (data_root / "CLINIC_A").mkdir(parents=True) + (data_root / "CLINIC_B").mkdir(parents=True) + + (data_root / "CLINIC_A" / "2024_A1_Tracker.xlsx").write_bytes(b"clinic A tracker 1 contents") + (data_root / "CLINIC_A" / "2024_A2_Tracker.xlsx").write_bytes(b"clinic A tracker 2 contents") + (data_root / "CLINIC_B" / "2024_B1_Tracker.xlsx").write_bytes(b"clinic B tracker contents") + return data_root + + +@pytest.fixture +def fake_output_root(tmp_path: Path) -> Path: + """Create an output_root where A1 is fully processed, A2 half-processed, B1 untouched.""" + output_root = tmp_path / "output" + for subdir in ( + "patient_data_cleaned", + "patient_data_raw", + "product_data_cleaned", + "product_data_raw", + ): + (output_root / subdir).mkdir(parents=True) + + # A1: full set of outputs across all four subdirs. + for subdir, suffix in ( + ("patient_data_cleaned", "_patient_cleaned.parquet"), + ("patient_data_raw", "_patient_raw.parquet"), + ("product_data_cleaned", "_product_cleaned.parquet"), + ("product_data_raw", "_product_raw.parquet"), + ): + (output_root / subdir / f"2024_A1_Tracker{suffix}").write_bytes(b"") + + # A2: only patient outputs (product arm skipped). + (output_root / "patient_data_raw" / "2024_A2_Tracker_patient_raw.parquet").write_bytes(b"") + (output_root / "patient_data_cleaned" / "2024_A2_Tracker_patient_cleaned.parquet").write_bytes( + b"" + ) + + # B1: no outputs. + return output_root + + +def test_metadata_shape_and_schema(fake_trackers: Path, fake_output_root: Path): + out_path = create_table_tracker_metadata(fake_trackers, fake_output_root) + + assert out_path == fake_output_root / "tables" / "tracker_metadata.parquet" + assert out_path.exists() + + df = pl.read_parquet(out_path) + assert df.shape == (3, 9) + assert df.columns == [ + "file_name", + "clinic_code", + "md5", + "patient_data_cleaned", + "patient_data_raw", + "product_data_cleaned", + "product_data_raw", + "complete", + "timestamp", + ] + + +def test_metadata_presence_flags(fake_trackers: Path, fake_output_root: Path): + create_table_tracker_metadata(fake_trackers, fake_output_root) + df = pl.read_parquet(fake_output_root / "tables" / "tracker_metadata.parquet").sort("file_name") + + rows = df.to_dicts() + + a1 = next(r for r in rows if r["file_name"] == "2024_A1_Tracker") + assert a1["clinic_code"] == "CLINIC_A" + assert a1["patient_data_raw"] is True + assert a1["patient_data_cleaned"] is True + assert a1["product_data_raw"] is True + assert a1["product_data_cleaned"] is True + assert a1["complete"] is True + + a2 = next(r for r in rows if r["file_name"] == "2024_A2_Tracker") + assert a2["patient_data_raw"] is True + assert a2["product_data_raw"] is False + assert a2["complete"] is False + + b1 = next(r for r in rows if r["file_name"] == "2024_B1_Tracker") + assert b1["clinic_code"] == "CLINIC_B" + assert all( + b1[s] is False + for s in ( + "patient_data_raw", + "patient_data_cleaned", + "product_data_raw", + "product_data_cleaned", + ) + ) + assert b1["complete"] is False + + +def test_metadata_md5_matches_file_bytes(fake_trackers: Path, fake_output_root: Path): + create_table_tracker_metadata(fake_trackers, fake_output_root) + df = pl.read_parquet(fake_output_root / "tables" / "tracker_metadata.parquet") + + for row in df.iter_rows(named=True): + tracker_path = fake_trackers / row["clinic_code"] / f"{row['file_name']}.xlsx" + expected = hashlib.md5(tracker_path.read_bytes(), usedforsecurity=False).hexdigest() + assert row["md5"] == expected + + +def test_metadata_handles_missing_output_subdirs(fake_trackers: Path, tmp_path: Path): + """Empty output_root (no subdirs yet) should produce all-False presence flags.""" + output_root = tmp_path / "empty_output" + output_root.mkdir() + + create_table_tracker_metadata(fake_trackers, output_root) + df = pl.read_parquet(output_root / "tables" / "tracker_metadata.parquet") + + assert df.height == 3 + for col in ( + "patient_data_raw", + "patient_data_cleaned", + "product_data_raw", + "product_data_cleaned", + ): + assert df[col].to_list() == [False, False, False] + assert df["complete"].to_list() == [False, False, False] + + +def test_metadata_empty_data_root(tmp_path: Path): + """Empty data_root should produce a zero-row parquet with correct schema.""" + data_root = tmp_path / "data" + data_root.mkdir() + output_root = tmp_path / "output" + output_root.mkdir() + + out_path = create_table_tracker_metadata(data_root, output_root) + df = pl.read_parquet(out_path) + + assert df.height == 0 + assert df.columns == [ + "file_name", + "clinic_code", + "md5", + "patient_data_cleaned", + "patient_data_raw", + "product_data_cleaned", + "product_data_raw", + "complete", + "timestamp", + ] diff --git a/tests/test_tables/test_patient.py b/tests/test_tables/test_patient.py new file mode 100644 index 0000000..26de6b2 --- /dev/null +++ b/tests/test_tables/test_patient.py @@ -0,0 +1,366 @@ +"""Tests for patient table creation.""" + +from pathlib import Path + +import polars as pl +import pytest + +from a4d.tables.patient import ( + create_table_patient_data_annual, + create_table_patient_data_monthly, + create_table_patient_data_static, + read_cleaned_patient_data, +) + + +@pytest.fixture +def cleaned_patient_data_files(tmp_path: Path) -> list[Path]: + """Create test cleaned patient data files.""" + data_dir = tmp_path / "cleaned" + data_dir.mkdir() + + file1 = data_dir / "tracker1_2024_01.parquet" + df1 = pl.DataFrame( + { + "patient_id": ["P001", "P002", "P003"], + "clinic_id": ["C001", "C001", "C002"], + "name": ["Alice", "Bob", "Charlie"], + "dob": ["2010-01-15", "2011-03-20", "2009-08-10"], + "sex": ["F", "M", "M"], + "recruitment_date": ["2024-01-10", "2024-01-15", "2024-01-05"], + "province": ["Province1", "Province1", "Province2"], + "hba1c_baseline": [8.5, 7.2, 9.1], + "hba1c_baseline_exceeds": [True, False, True], + "fbg_baseline_mg": [120, 110, 130], + "fbg_baseline_mmol": [6.7, 6.1, 7.2], + "patient_consent": [True, True, True], + "t1d_diagnosis_date": ["2023-01-01", "2022-05-10", "2021-12-15"], + "t1d_diagnosis_age": [13, 11, 12], + "t1d_diagnosis_with_dka": [True, False, True], + "status_out": ["Active", "Active", "Active"], + "lost_date": [None, None, None], + "file_name": ["tracker1.xlsx", "tracker1.xlsx", "tracker1.xlsx"], + "tracker_date": ["2024-01-31", "2024-01-31", "2024-01-31"], + "tracker_month": [1, 1, 1], + "tracker_year": [2024, 2024, 2024], + "sheet_name": ["Jan 2024", "Jan 2024", "Jan 2024"], + "weight": [45.5, 52.3, 48.1], + "height": [155, 162, 158], + "bmi": [18.9, 19.9, 19.3], + "bmi_date": ["2024-01-15", "2024-01-18", "2024-01-20"], + "age": [14, 13, 15], + "status": ["Active", "Active", "Active"], + "hba1c_updated": [7.8, 6.9, 8.5], + "hba1c_updated_date": ["2024-01-20", "2024-01-22", "2024-01-18"], + "hba1c_updated_exceeds": [False, False, True], + "fbg_updated_mg": [115, 105, 125], + "fbg_updated_mmol": [6.4, 5.8, 6.9], + "fbg_updated_date": ["2024-01-20", "2024-01-22", "2024-01-18"], + "insulin_type": ["Rapid", "Mixed", "Rapid"], + "insulin_subtype": ["Lispro", "30/70", "Aspart"], + "insulin_regimen": ["Basal-bolus", "Twice daily", "Basal-bolus"], + "insulin_injections": [4, 2, 4], + "insulin_total_units": [35, 28, 40], + "testing_frequency": [4, 3, 4], + "support_level": ["Full", "Full", "Partial"], + "last_clinic_visit_date": ["2024-01-25", "2024-01-28", "2024-01-22"], + "last_remote_followup_date": [None, None, None], + "hospitalisation_date": [None, None, None], + "hospitalisation_cause": [None, None, None], + "observations": ["Doing well", "Good progress", "Needs improvement"], + "observations_category": ["Good", "Good", "Fair"], + "edu_occ": ["Student", "Student", "Student"], + "edu_occ_updated": ["Student", "Student", "Student"], + "blood_pressure_updated": ["110/70", "115/75", "120/80"], + "blood_pressure_sys_mmhg": [110, 115, 120], + "blood_pressure_dias_mmhg": [70, 75, 80], + "complication_screening_kidney_test_date": ["2024-01-10", None, "2024-01-08"], + "complication_screening_kidney_test_value": ["Normal", None, "Normal"], + "complication_screening_eye_exam_date": ["2024-01-10", None, None], + "complication_screening_eye_exam_value": ["Normal", None, None], + "complication_screening_foot_exam_date": [None, None, None], + "complication_screening_foot_exam_value": [None, None, None], + "complication_screening_lipid_profile_date": [None, None, None], + "complication_screening_lipid_profile_triglycerides_value": [None, None, None], + "complication_screening_lipid_profile_cholesterol_value": [None, None, None], + "complication_screening_lipid_profile_ldl_mg_value": [None, None, None], + "complication_screening_lipid_profile_ldl_mmol_value": [None, None, None], + "complication_screening_lipid_profile_hdl_mg_value": [None, None, None], + "complication_screening_lipid_profile_hdl_mmol_value": [None, None, None], + "complication_screening_thyroid_test_date": [None, None, None], + "complication_screening_thyroid_test_ft4_ng_value": [None, None, None], + "complication_screening_thyroid_test_ft4_pmol_value": [None, None, None], + "complication_screening_thyroid_test_tsh_value": [None, None, None], + "complication_screening_remarks": [None, None, None], + "dm_complication_eye": [None, None, None], + "dm_complication_kidney": [None, None, None], + "dm_complication_others": [None, None, None], + "dm_complication_remarks": [None, None, None], + "family_history": ["No diabetes", "Type 2 in family", "No diabetes"], + "other_issues": [None, None, None], + } + ) + df1.write_parquet(file1) + + file2 = data_dir / "tracker1_2024_02.parquet" + df2 = pl.DataFrame( + { + "patient_id": ["P001", "P002"], + "clinic_id": ["C001", "C001"], + "name": ["Alice", "Bob"], + "dob": ["2010-01-15", "2011-03-20"], + "sex": ["F", "M"], + "recruitment_date": ["2024-01-10", "2024-01-15"], + "province": ["Province1", "Province1"], + "hba1c_baseline": [8.5, 7.2], + "hba1c_baseline_exceeds": [True, False], + "fbg_baseline_mg": [120, 110], + "fbg_baseline_mmol": [6.7, 6.1], + "patient_consent": [True, True], + "t1d_diagnosis_date": ["2023-01-01", "2022-05-10"], + "t1d_diagnosis_age": [13, 11], + "t1d_diagnosis_with_dka": [True, False], + "status_out": ["Active", "Active"], + "lost_date": [None, None], + "file_name": ["tracker1.xlsx", "tracker1.xlsx"], + "tracker_date": ["2024-02-29", "2024-02-29"], + "tracker_month": [2, 2], + "tracker_year": [2024, 2024], + "sheet_name": ["Feb 2024", "Feb 2024"], + "weight": [46.0, 52.8], + "height": [155, 162], + "bmi": [19.1, 20.1], + "bmi_date": ["2024-02-15", "2024-02-18"], + "age": [14, 13], + "status": ["Active", "Active"], + "hba1c_updated": [7.5, 6.7], + "hba1c_updated_date": ["2024-02-20", "2024-02-22"], + "hba1c_updated_exceeds": [False, False], + "fbg_updated_mg": [110, 100], + "fbg_updated_mmol": [6.1, 5.6], + "fbg_updated_date": ["2024-02-20", "2024-02-22"], + "insulin_type": ["Rapid", "Mixed"], + "insulin_subtype": ["Lispro", "30/70"], + "insulin_regimen": ["Basal-bolus", "Twice daily"], + "insulin_injections": [4, 2], + "insulin_total_units": [36, 29], + "testing_frequency": [4, 3], + "support_level": ["Full", "Full"], + "last_clinic_visit_date": ["2024-02-25", "2024-02-28"], + "last_remote_followup_date": [None, None], + "hospitalisation_date": [None, None], + "hospitalisation_cause": [None, None], + "observations": ["Excellent progress", "Very good"], + "observations_category": ["Excellent", "Good"], + "edu_occ": ["Student", "Student"], + "edu_occ_updated": ["Student", "Student"], + "blood_pressure_updated": ["108/68", "112/72"], + "blood_pressure_sys_mmhg": [108, 112], + "blood_pressure_dias_mmhg": [68, 72], + "complication_screening_kidney_test_date": [None, None], + "complication_screening_kidney_test_value": [None, None], + "complication_screening_eye_exam_date": [None, None], + "complication_screening_eye_exam_value": [None, None], + "complication_screening_foot_exam_date": [None, None], + "complication_screening_foot_exam_value": [None, None], + "complication_screening_lipid_profile_date": [None, None], + "complication_screening_lipid_profile_triglycerides_value": [None, None], + "complication_screening_lipid_profile_cholesterol_value": [None, None], + "complication_screening_lipid_profile_ldl_mg_value": [None, None], + "complication_screening_lipid_profile_ldl_mmol_value": [None, None], + "complication_screening_lipid_profile_hdl_mg_value": [None, None], + "complication_screening_lipid_profile_hdl_mmol_value": [None, None], + "complication_screening_thyroid_test_date": [None, None], + "complication_screening_thyroid_test_ft4_ng_value": [None, None], + "complication_screening_thyroid_test_ft4_pmol_value": [None, None], + "complication_screening_thyroid_test_tsh_value": [None, None], + "complication_screening_remarks": [None, None], + "dm_complication_eye": [None, None], + "dm_complication_kidney": [None, None], + "dm_complication_others": [None, None], + "dm_complication_remarks": [None, None], + "family_history": ["No diabetes", "Type 2 in family"], + "other_issues": [None, None], + } + ) + df2.write_parquet(file2) + + return [file1, file2] + + +def test_read_cleaned_patient_data(cleaned_patient_data_files: list[Path]): + """Test reading and combining cleaned patient data files.""" + result = read_cleaned_patient_data(cleaned_patient_data_files) + + assert isinstance(result, pl.DataFrame) + assert result.shape[0] == 5 # 3 rows from file1 + 2 rows from file2 + assert "patient_id" in result.columns + assert "clinic_id" in result.columns + assert set(result["patient_id"].to_list()) == {"P001", "P002", "P003"} + + +def test_read_cleaned_patient_data_empty_list(): + """Test that empty file list raises error.""" + with pytest.raises(ValueError, match="No cleaned files provided"): + read_cleaned_patient_data([]) + + +def test_create_table_patient_data_static(cleaned_patient_data_files: list[Path], tmp_path: Path): + """Test creation of static patient data table.""" + output_dir = tmp_path / "output" + + patient_data = read_cleaned_patient_data(cleaned_patient_data_files) + output_file = create_table_patient_data_static(patient_data, output_dir) + + assert output_file.exists() + assert output_file.name == "patient_data_static.parquet" + + result = pl.read_parquet(output_file) + + assert result.shape[0] == 3 + assert set(result["patient_id"].to_list()) == {"P001", "P002", "P003"} + + p001_data = result.filter(pl.col("patient_id") == "P001") + assert p001_data["tracker_month"][0] == 2 + assert p001_data["tracker_year"][0] == 2024 + + p002_data = result.filter(pl.col("patient_id") == "P002") + assert p002_data["tracker_month"][0] == 2 + assert p002_data["tracker_year"][0] == 2024 + + p003_data = result.filter(pl.col("patient_id") == "P003") + assert p003_data["tracker_month"][0] == 1 + assert p003_data["tracker_year"][0] == 2024 + + assert "name" in result.columns + assert "dob" in result.columns + assert "recruitment_date" in result.columns + assert "weight" not in result.columns + assert "status" not in result.columns + + +def test_create_table_patient_data_monthly(cleaned_patient_data_files: list[Path], tmp_path: Path): + """Test creation of monthly patient data table.""" + output_dir = tmp_path / "output" + + patient_data = read_cleaned_patient_data(cleaned_patient_data_files) + output_file = create_table_patient_data_monthly(patient_data, output_dir) + + assert output_file.exists() + assert output_file.name == "patient_data_monthly.parquet" + + result = pl.read_parquet(output_file) + + assert result.shape[0] == 5 + + assert "weight" in result.columns + assert "bmi" in result.columns + assert "status" in result.columns + assert "insulin_type" in result.columns + assert "name" not in result.columns + assert "dob" not in result.columns + + sorted_check = result["tracker_year"].to_list() + assert sorted_check == sorted(sorted_check) + + +def test_create_table_patient_data_annual(cleaned_patient_data_files: list[Path], tmp_path: Path): + """Test creation of annual patient data table.""" + output_dir = tmp_path / "output" + + patient_data = read_cleaned_patient_data(cleaned_patient_data_files) + output_file = create_table_patient_data_annual(patient_data, output_dir) + + assert output_file.exists() + assert output_file.name == "patient_data_annual.parquet" + + result = pl.read_parquet(output_file) + + assert result.shape[0] == 3 + + assert "complication_screening_kidney_test_date" in result.columns + assert "dm_complication_eye" in result.columns + assert "family_history" in result.columns + assert "name" not in result.columns + assert "weight" not in result.columns + + p001_data = result.filter(pl.col("patient_id") == "P001") + assert p001_data.shape[0] == 1 + assert p001_data["tracker_month"][0] == 2 + assert p001_data["tracker_year"][0] == 2024 + + +def test_create_table_patient_data_annual_filters_pre_2024(tmp_path: Path): + """Test that annual table filters out data before 2024.""" + data_dir = tmp_path / "cleaned" + data_dir.mkdir() + + file1 = data_dir / "tracker_2023.parquet" + df1 = pl.DataFrame( + { + "patient_id": ["P001"], + "status": ["Active"], + "tracker_month": [12], + "tracker_year": [2023], + "tracker_date": ["2023-12-31"], + "edu_occ": ["Student"], + "edu_occ_updated": ["Student"], + "blood_pressure_updated": ["110/70"], + "blood_pressure_sys_mmhg": [110], + "blood_pressure_dias_mmhg": [70], + "complication_screening_kidney_test_date": [None], + "complication_screening_kidney_test_value": [None], + "complication_screening_eye_exam_date": [None], + "complication_screening_eye_exam_value": [None], + "complication_screening_foot_exam_date": [None], + "complication_screening_foot_exam_value": [None], + "complication_screening_lipid_profile_date": [None], + "complication_screening_lipid_profile_triglycerides_value": [None], + "complication_screening_lipid_profile_cholesterol_value": [None], + "complication_screening_lipid_profile_ldl_mg_value": [None], + "complication_screening_lipid_profile_ldl_mmol_value": [None], + "complication_screening_lipid_profile_hdl_mg_value": [None], + "complication_screening_lipid_profile_hdl_mmol_value": [None], + "complication_screening_thyroid_test_date": [None], + "complication_screening_thyroid_test_ft4_ng_value": [None], + "complication_screening_thyroid_test_ft4_pmol_value": [None], + "complication_screening_thyroid_test_tsh_value": [None], + "complication_screening_remarks": [None], + "dm_complication_eye": [None], + "dm_complication_kidney": [None], + "dm_complication_others": [None], + "dm_complication_remarks": [None], + "family_history": ["No diabetes"], + "other_issues": [None], + } + ) + df1.write_parquet(file1) + + output_dir = tmp_path / "output" + patient_data = read_cleaned_patient_data([file1]) + output_file = create_table_patient_data_annual(patient_data, output_dir) + + result = pl.read_parquet(output_file) + assert result.shape[0] == 0 + + +def test_static_table_sorting(cleaned_patient_data_files: list[Path], tmp_path: Path): + """Test that static table is sorted correctly.""" + output_dir = tmp_path / "output" + patient_data = read_cleaned_patient_data(cleaned_patient_data_files) + output_file = create_table_patient_data_static(patient_data, output_dir) + + result = pl.read_parquet(output_file) + + tracker_years = result["tracker_year"].to_list() + tracker_months = result["tracker_month"].to_list() + patient_ids = result["patient_id"].to_list() + + for i in range(len(result) - 1): + if tracker_years[i] < tracker_years[i + 1]: + continue + elif tracker_years[i] == tracker_years[i + 1]: + if tracker_months[i] < tracker_months[i + 1]: + continue + elif tracker_months[i] == tracker_months[i + 1]: + assert patient_ids[i] <= patient_ids[i + 1] diff --git a/tests/test_tables/test_product.py b/tests/test_tables/test_product.py new file mode 100644 index 0000000..74f031b --- /dev/null +++ b/tests/test_tables/test_product.py @@ -0,0 +1,143 @@ +"""Tests for `create_table_product_data` — Script 3 (steps 3.1-3.3) directly. + +`test_link_product_patient.py` already covers `link_product_patient`; this +file covers table creation, which was previously only reached indirectly. +""" + +from datetime import date +from pathlib import Path + +import polars as pl +import pytest + +from a4d.clean.schema_product import get_product_data_schema +from a4d.tables.product import create_table_product_data, read_cleaned_product_data + + +def _cleaned_product_df(rows: list[dict]) -> pl.DataFrame: + """Build a DataFrame matching clean_product_data's output schema (20 cols).""" + schema = get_product_data_schema() + return pl.DataFrame(rows, schema=schema) + + +def _base_row(**overrides) -> dict: + row = { + "product": "Insulin A", + "product_units_notes": None, + "product_entry_date": date(2024, 1, 15), + "product_units_released": 5.0, + "product_released_to": "MY_SU001", + "product_units_received": None, + "product_received_from": "HQ", + "product_balance": 10.0, + "product_units_returned": None, + "product_returned_by": None, + "product_table_month": 1, + "product_table_year": 2024, + "product_sheet_name": "Jan24", + "file_name": "tracker_a.xlsx", + "product_balance_status": "ok", + "product_category": "Insulin", + "orig_product_released_to": None, + "product_unit_capacity": 1, + "product_remarks": None, + "clinic_id": "SU", + } + row.update(overrides) + return row + + +def test_read_cleaned_product_data_empty_list_raises(): + with pytest.raises(ValueError, match="No cleaned product files"): + read_cleaned_product_data([]) + + +def test_read_cleaned_product_data_concatenates_diagonally(tmp_path: Path): + file_a = tmp_path / "a_product_cleaned.parquet" + file_b = tmp_path / "b_product_cleaned.parquet" + _cleaned_product_df([_base_row(file_name="a.xlsx")]).write_parquet(file_a) + _cleaned_product_df([_base_row(file_name="b.xlsx")]).write_parquet(file_b) + + result = read_cleaned_product_data([file_a, file_b]) + + assert result.height == 2 + assert set(result["file_name"].to_list()) == {"a.xlsx", "b.xlsx"} + + +def test_create_table_writes_output_with_full_schema(tmp_path: Path): + cleaned_file = tmp_path / "tracker_a_product_cleaned.parquet" + _cleaned_product_df([_base_row(), _base_row(product_released_to="MY_SU002")]).write_parquet( + cleaned_file + ) + output_dir = tmp_path / "tables" + + output_path = create_table_product_data([cleaned_file], output_dir) + + assert output_path == output_dir / "product_data.parquet" + assert output_path.exists() + + df = pl.read_parquet(output_path) + assert set(df.columns) == set(get_product_data_schema().keys()) + assert df.height == 2 + + +def test_create_table_preserves_original_released_to(tmp_path: Path): + """orig_product_released_to should carry the pre-fix_patient_id value.""" + cleaned_file = tmp_path / "tracker_a_product_cleaned.parquet" + _cleaned_product_df([_base_row(product_released_to="MY-SU001")]).write_parquet(cleaned_file) + output_dir = tmp_path / "tables" + + output_path = create_table_product_data([cleaned_file], output_dir) + df = pl.read_parquet(output_path) + + # fix_patient_id normalizes hyphens to underscores. + assert df["product_released_to"].to_list() == ["MY_SU001"] + assert df["orig_product_released_to"].to_list() == ["MY-SU001"] + + +def test_create_table_replaces_invalid_released_to(tmp_path: Path): + cleaned_file = tmp_path / "tracker_a_product_cleaned.parquet" + _cleaned_product_df([_base_row(product_released_to="not-a-patient-id")]).write_parquet( + cleaned_file + ) + output_dir = tmp_path / "tables" + + output_path = create_table_product_data([cleaned_file], output_dir) + df = pl.read_parquet(output_path) + + # Invalid, <=8-char-after-normalization IDs get replaced by fix_patient_id; + # the raw value survives untouched in orig_product_released_to. + assert df["orig_product_released_to"].to_list() == ["not-a-patient-id"] + assert df["product_released_to"].to_list() != ["not-a-patient-id"] + + +def test_create_table_concatenates_multiple_files(tmp_path: Path): + file_a = tmp_path / "a_product_cleaned.parquet" + file_b = tmp_path / "b_product_cleaned.parquet" + _cleaned_product_df([_base_row(file_name="a.xlsx")]).write_parquet(file_a) + _cleaned_product_df( + [ + _base_row(file_name="b.xlsx"), + _base_row(file_name="b.xlsx", product_released_to="MY_SU003"), + ] + ).write_parquet(file_b) + output_dir = tmp_path / "tables" + + output_path = create_table_product_data([file_a, file_b], output_dir) + df = pl.read_parquet(output_path) + + assert df.height == 3 + assert sorted(df["file_name"].unique().to_list()) == ["a.xlsx", "b.xlsx"] + + +def test_create_table_casts_numeric_columns(tmp_path: Path): + """safe_convert_column should cast to schema dtypes (e.g. Float64 balance).""" + cleaned_file = tmp_path / "tracker_a_product_cleaned.parquet" + _cleaned_product_df([_base_row(product_balance=12.5)]).write_parquet(cleaned_file) + output_dir = tmp_path / "tables" + + output_path = create_table_product_data([cleaned_file], output_dir) + df = pl.read_parquet(output_path) + + assert df.schema["product_balance"] == pl.Float64 + assert df["product_balance"].to_list() == [12.5] diff --git a/tests/test_validate/__init__.py b/tests/test_validate/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_validate/test_patient_source_vs_output.py b/tests/test_validate/test_patient_source_vs_output.py new file mode 100644 index 0000000..3eb37f3 --- /dev/null +++ b/tests/test_validate/test_patient_source_vs_output.py @@ -0,0 +1,311 @@ +"""Tests for source_vs_output_patient. + +Fixtures pass ``schema={...}`` explicitly so all-None columns keep ``pl.Utf8`` +dtype (per the polars all-None ``pl.Null`` inference trap). +""" + +from __future__ import annotations + +import polars as pl + +from a4d.errors import ErrorCollector +from a4d.validate.common import normalize_patient_id +from a4d.validate.source_vs_output_patient import ( + _join_for_cell_checks, + check_missing_patients, + check_out_of_range, + check_unexpected_nulls, + check_value_shifts, +) + +RAW_SCHEMA = { + "clinic_id": pl.Utf8, + "patient_id": pl.Utf8, + "tracker_year": pl.Int64, + "tracker_month": pl.Int64, + "file_name": pl.Utf8, + "weight": pl.Utf8, + "height": pl.Utf8, + "hba1c_updated": pl.Utf8, +} + +CLEAN_SCHEMA = { + "clinic_id": pl.Utf8, + "patient_id": pl.Utf8, + "tracker_year": pl.Int64, + "tracker_month": pl.Int64, + "file_name": pl.Utf8, + "weight": pl.Float64, + "height": pl.Float64, + "hba1c_updated": pl.Float64, +} + + +def _make_raw(rows: list[dict]) -> pl.DataFrame: + return pl.DataFrame(rows, schema=RAW_SCHEMA) + + +def _make_clean(rows: list[dict]) -> pl.DataFrame: + return pl.DataFrame(rows, schema=CLEAN_SCHEMA) + + +def test_normalize_patient_id_strips_transfer_suffix() -> None: + df = pl.DataFrame( + {"patient_id": ["MY_SM003_SB", "LA-MH093_LF", "TH_NK001", "SOLO"]}, + schema={"patient_id": pl.Utf8}, + ) + out = df.with_columns(normalize_patient_id(pl.col("patient_id")).alias("n")) + assert out["n"].to_list() == ["MY_SM003", "LA_MH093", "TH_NK001", "SOLO"] + + +def test_missing_patient_does_not_fire_for_transferred_id() -> None: + raw = _make_raw( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003_SB", + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": "70", + "height": "1.7", + "hba1c_updated": "7.0", + } + ] + ) + cleaned = _make_clean( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003", # cleaner stripped suffix + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": 70.0, + "height": 1.7, + "hba1c_updated": 7.0, + } + ] + ) + coll = ErrorCollector() + check_missing_patients(raw, cleaned, coll) + assert len(coll) == 0 + + +def test_missing_patient_fires_when_truly_absent() -> None: + raw = _make_raw( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003", + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": "70", + "height": "1.7", + "hba1c_updated": "7.0", + }, + { + "clinic_id": "CDA", + "patient_id": "MY_SM999", # absent in cleaned + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": "60", + "height": "1.6", + "hba1c_updated": "8.0", + }, + ] + ) + cleaned = _make_clean( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003", + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": 70.0, + "height": 1.7, + "hba1c_updated": 7.0, + } + ] + ) + coll = ErrorCollector() + check_missing_patients(raw, cleaned, coll) + msgs = [e.error_message for e in coll.errors] + assert any("MY_SM999" in m and "MISSING_ROW" in m for m in msgs) + assert not any("PHANTOM_ROW" in m for m in msgs) + + +def test_value_shift_skips_height_unit_conversion() -> None: + raw = _make_raw( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003", + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": "70", + "height": "170", # cm, cleaner divides by 100 + "hba1c_updated": "7.0", + } + ] + ) + cleaned = _make_clean( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003", + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": 70.0, + "height": 1.7, # converted from 170 cm + "hba1c_updated": 7.0, + } + ] + ) + coll = ErrorCollector() + joined = _join_for_cell_checks(raw, cleaned) + check_value_shifts(joined, coll) + # Should be zero shifts: weight matches, hba1c matches, height is skipped. + shift_msgs = [e.error_message for e in coll.errors if "VALUE_SHIFT" in e.error_message] + assert shift_msgs == [] + + +def test_value_shift_fires_on_genuine_mismatch() -> None: + raw = _make_raw( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003", + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": "70", + "height": "1.7", + "hba1c_updated": "7.0", + } + ] + ) + cleaned = _make_clean( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003", + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": 99.0, # mismatch + "height": 1.7, + "hba1c_updated": 7.0, + } + ] + ) + coll = ErrorCollector() + joined = _join_for_cell_checks(raw, cleaned) + check_value_shifts(joined, coll) + assert any("VALUE_SHIFT" in e.error_message and e.column == "weight" for e in coll.errors) + + +def test_unexpected_null_fires_when_raw_has_value_cleaned_does_not() -> None: + raw = _make_raw( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003", + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": "70", + "height": "1.7", + "hba1c_updated": "7.0", + } + ] + ) + cleaned = _make_clean( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003", + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": 70.0, + "height": 1.7, + "hba1c_updated": None, # silently dropped despite parseable raw + } + ] + ) + coll = ErrorCollector() + joined = _join_for_cell_checks(raw, cleaned) + check_unexpected_nulls(joined, coll) + msgs = [e.error_message for e in coll.errors if e.column == "hba1c_updated"] + assert msgs + assert "UNEXPECTED_NULL" in msgs[0] + assert "was_parseable=True" in msgs[0] + + +def test_out_of_range_height_skips_cm_value() -> None: + """170 cm raw should NOT fire after the cm->m auto-conversion.""" + raw = _make_raw( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003", + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": "70", + "height": "170", + "hba1c_updated": "7.0", + } + ] + ) + coll = ErrorCollector() + check_out_of_range(raw, coll) + height_findings = [e for e in coll.errors if e.column == "height"] + assert height_findings == [] + + +def test_out_of_range_height_fires_after_auto_conversion() -> None: + """height=250 (cm) -> auto-converts to 2.5 m -> still exceeds max 2.3.""" + raw = _make_raw( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003", + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": "70", + "height": "250", + "hba1c_updated": "7.0", + } + ] + ) + coll = ErrorCollector() + check_out_of_range(raw, coll) + assert any(e.column == "height" and "OUT_OF_RANGE_RAW" in e.error_message for e in coll.errors) + + +def test_out_of_range_weight_fires_for_obvious_outlier() -> None: + raw = _make_raw( + [ + { + "clinic_id": "CDA", + "patient_id": "MY_SM003", + "tracker_year": 2024, + "tracker_month": 1, + "file_name": "f", + "weight": "350", # exceeds 200 max + "height": "1.7", + "hba1c_updated": "7.0", + } + ] + ) + coll = ErrorCollector() + check_out_of_range(raw, coll) + assert any(e.column == "weight" for e in coll.errors) diff --git a/tests/test_validate/test_product_source_vs_output.py b/tests/test_validate/test_product_source_vs_output.py new file mode 100644 index 0000000..bed9c61 --- /dev/null +++ b/tests/test_validate/test_product_source_vs_output.py @@ -0,0 +1,247 @@ +"""Tests for source_vs_output_product.""" + +from __future__ import annotations + +import polars as pl + +from a4d.errors import ErrorCollector +from a4d.validate.source_vs_output_product import ( + _explode_multi_product_cells, + check_column_null_rate_delta, + check_missing_groups, + check_row_count_delta, +) + +RAW_SCHEMA = { + "file_name": pl.Utf8, + "product_sheet_name": pl.Utf8, + "product": pl.Utf8, + "product_units_received": pl.Utf8, + "product_units_released": pl.Utf8, +} + +CLEAN_SCHEMA = { + "file_name": pl.Utf8, + "product_sheet_name": pl.Utf8, + "product": pl.Utf8, + "product_units_received": pl.Float64, + "product_units_released": pl.Float64, + "product_balance_status": pl.Utf8, +} + + +def _raw(rows: list[dict]) -> pl.DataFrame: + return pl.DataFrame(rows, schema=RAW_SCHEMA) + + +def _clean(rows: list[dict]) -> pl.DataFrame: + return pl.DataFrame(rows, schema=CLEAN_SCHEMA) + + +def test_explode_splits_multi_product_cell() -> None: + df = _raw( + [ + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "Insulin A; Insulin B", + "product_units_received": "10", + "product_units_released": None, + } + ] + ) + exploded = _explode_multi_product_cells(df) + assert sorted(exploded["product"].to_list()) == ["Insulin A", "Insulin B"] + + +def test_missing_group_after_explode() -> None: + raw = _raw( + [ + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "Insulin A; Insulin B", + "product_units_received": "10", + "product_units_released": None, + } + ] + ) + cleaned = _clean( + [ + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "Insulin A", + "product_units_received": 10.0, + "product_units_released": None, + "product_balance_status": "start", + } + # Insulin B is missing from cleaned -> should fire MISSING_GROUP + ] + ) + coll = ErrorCollector() + check_missing_groups(_explode_multi_product_cells(raw), cleaned, coll) + msgs = [e.error_message for e in coll.errors] + assert any("MISSING_GROUP" in m and "Insulin B" in m for m in msgs) + + +def test_phantom_group_fires_for_invented_product() -> None: + raw = _raw( + [ + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "Insulin A", + "product_units_received": "10", + "product_units_released": None, + } + ] + ) + cleaned = _clean( + [ + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "Insulin A", + "product_units_received": 10.0, + "product_units_released": None, + "product_balance_status": "start", + }, + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "Phantom Drug", + "product_units_received": 5.0, + "product_units_released": None, + "product_balance_status": "start", + }, + ] + ) + coll = ErrorCollector() + check_missing_groups(_explode_multi_product_cells(raw), cleaned, coll) + assert any( + "PHANTOM_GROUP" in e.error_message and "Phantom Drug" in e.error_message + for e in coll.errors + ) + + +def test_row_count_delta_fires_when_counts_differ() -> None: + raw = _raw( + [ + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "Insulin A", + "product_units_received": "10", + "product_units_released": None, + }, + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "Insulin A", + "product_units_received": "20", + "product_units_released": None, + }, + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "Insulin A", + "product_units_received": "30", + "product_units_released": None, + }, + ] + ) + cleaned = _clean( + [ + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "Insulin A", + "product_units_received": 10.0, + "product_units_released": None, + "product_balance_status": "start", + } + # Cleaner dropped two rows + ] + ) + coll = ErrorCollector() + check_row_count_delta(_explode_multi_product_cells(raw), cleaned, coll) + assert any( + "ROW_COUNT_DELTA" in e.error_message and "delta=-2" in e.error_message for e in coll.errors + ) + + +def test_row_count_delta_silent_when_counts_match() -> None: + raw = _raw( + [ + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "Insulin A", + "product_units_received": "10", + "product_units_released": None, + } + ] + ) + cleaned = _clean( + [ + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "Insulin A", + "product_units_received": 10.0, + "product_units_released": None, + "product_balance_status": "start", + } + ] + ) + coll = ErrorCollector() + check_row_count_delta(_explode_multi_product_cells(raw), cleaned, coll) + assert len(coll) == 0 + + +def test_column_null_rate_delta_fires_for_silent_loss() -> None: + raw = _raw( + [ + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "A", + "product_units_received": "10", + "product_units_released": "5", + }, + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "B", + "product_units_received": "20", + "product_units_released": "5", + }, + ] + ) + cleaned = _clean( + [ + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "A", + "product_units_received": 10.0, + "product_units_released": None, # cleaner dropped both + "product_balance_status": "start", + }, + { + "file_name": "f", + "product_sheet_name": "Jan24", + "product": "B", + "product_units_received": 20.0, + "product_units_released": None, + "product_balance_status": "end", + }, + ] + ) + coll = ErrorCollector() + check_column_null_rate_delta(raw, cleaned, coll) + assert any( + "COLUMN_NULL_RATE_DELTA" in e.error_message and e.column == "product_units_released" + for e in coll.errors + ) diff --git a/tests/testthat.R b/tests/testthat.R deleted file mode 100644 index c4d21ca..0000000 --- a/tests/testthat.R +++ /dev/null @@ -1,12 +0,0 @@ -# This file is part of the standard setup for testthat. -# It is recommended that you do not modify it. -# -# Where should you do additional test configuration? -# Learn more about the roles of various files in: -# * https://r-pkgs.org/tests.html -# * https://testthat.r-lib.org/reference/test_package.html#special-files - -library(testthat) -library(a4d) - -test_check("a4d") diff --git a/tests/testthat/helper-compute-balance.R b/tests/testthat/helper-compute-balance.R deleted file mode 100644 index 82a7c61..0000000 --- a/tests/testthat/helper-compute-balance.R +++ /dev/null @@ -1,21 +0,0 @@ -# dummy data frame how it can look like after the function compute_balance_status is run in the product date script -df_balance_product <- data.frame( - product = rep("Accu-Chek Performa Test Strips (100s/box)", 10), - product_entry_date = c( - "2020-12-31", "2020-01-01", "2020-01-11", "2020-01-12", "2020-01-18", - "2020-01-22", "2020-01-24", "2020-01-28", "2020-01-29", "2020-01-31" - ), - product_balance = c(20, 10, 10, 10, 10, 15, 5, 4, 2, 6), - product_units_received = c(0, 0, 0, 0, 5, 0, 0, 0, 0, 4), - product_units_released = c(0, 10, 0, 0, 0, 0, 10, 1, 2, 0), - product_units_returned = rep(0, 10), - product_balance_status = c("start", rep("change", 8), "end") -) - - -# final balance for each product is the starting balance PLUS all products received MINUS all products released -helper_compute_correct_exit_balance <- function(df) { - exit_balance <- df$product_balance[1] + sum(df$product_units_received) - sum(df$product_units_released) - - return(exit_balance) -} diff --git a/tests/testthat/helper-data-extract-country-clinic-code.R b/tests/testthat/helper-data-extract-country-clinic-code.R deleted file mode 100644 index c1922f7..0000000 --- a/tests/testthat/helper-data-extract-country-clinic-code.R +++ /dev/null @@ -1,7 +0,0 @@ -patient_demo_data <- data.frame( - id = c("DE_GR001", "DE_GR002", "DE_GR003") -) - -patient_demo_data2 <- data.frame( - id = c("DE_AC001", "DE_AB002", "DE_AB003") -) diff --git a/tests/testthat/test-compute-balance.R b/tests/testthat/test-compute-balance.R deleted file mode 100644 index 1e49b1d..0000000 --- a/tests/testthat/test-compute-balance.R +++ /dev/null @@ -1,4 +0,0 @@ -test_that("check_compute_balance works", { - expect_equal(tail(compute_balance(df_balance_product, "2020")$product_balance, 1), helper_compute_correct_exit_balance(df_balance_product)) - expect_equal(tail(compute_balance(df_balance_product, "2021")$product_balance, 1), 2) -}) diff --git a/tests/testthat/test-data_cleaning_conifg.R b/tests/testthat/test-data_cleaning_conifg.R deleted file mode 100644 index 2bbb8bc..0000000 --- a/tests/testthat/test-data_cleaning_conifg.R +++ /dev/null @@ -1,93 +0,0 @@ -test_that("data cleaning config for support_from_a4d yields valid R", { - env <- list(support_from_a4d = "Standard", ERROR_VAL_CHARACTER = "Undefined") - - expect_equal( - eval( - parse_character_cleaning_pipeline( - "support_from_a4d", - a4d:::config$cleaning$support_from_a4d - ), - env - ), - "Standard" - ) -}) - -test_that("data cleaning config for status yields valid R", { - env <- list(status = "Active", ERROR_VAL_CHARACTER = "Undefined") - - expect_equal( - eval( - parse_character_cleaning_pipeline( - "status", - a4d:::config$cleaning$status - ), - env - ), - "Active" - ) -}) - -test_that("data cleaning config for insulin_regimen yields valid R", { - env <- list(insulin_regimen = "Basal-bolus", ERROR_VAL_CHARACTER = NA_character_) - - expect_equal( - eval( - parse_character_cleaning_pipeline( - "insulin_regimen", - a4d:::config$cleaning$insulin_regimen - ), - env - ), - "Basal-bolus (MDI)" - ) -}) - -test_that("data cleaning config for t1d_diagnosis_with_dka yields valid R", { - env <- list(t1d_diagnosis_with_dka = "N", ERROR_VAL_CHARACTER = NA_character_, patient_id = 1) - - expect_equal( - eval( - parse_character_cleaning_pipeline( - "t1d_diagnosis_with_dka", - a4d:::config$cleaning$t1d_diagnosis_with_dka - ), - env - ), - "N" - ) -}) - -test_that("data cleaning config for hospitalization_cause yields valid R", { - env <- list(hospitalization_cause = "DKA", ERROR_VAL_CHARACTER = NA_character_) - - expect_equal( - eval( - parse_character_cleaning_pipeline( - "hospitalization_cause", - a4d:::config$cleaning$hospitalization_cause - ), - env - ), - "DKA" - ) -}) - -test_that("data cleaning config for province yields valid R", { - env <- list(province = "Not a valid province", patient_id = 1) - - expect_equal( - eval( - parse_character_cleaning_pipeline( - "province", - a4d:::config$cleaning$province - ), - env - ), - "Undefined" - ) -}) - -test_that("data cleaning config for provinces had placeholder expanded", { - expect_gt(length(a4d:::config$cleaning$province$steps[[1]]$allowed_values), 1) -}) diff --git a/tests/testthat/test-get_tracker_year.R b/tests/testthat/test-get_tracker_year.R deleted file mode 100644 index ed41517..0000000 --- a/tests/testthat/test-get_tracker_year.R +++ /dev/null @@ -1,7 +0,0 @@ -test_that("check_get_tracker_year works", { - tracker_file <- "test_2022.xlsx" - month_list <- c("Jan23", "Feb23", "Dec23") - wrong_month_list <- c("Jan", "Feb", "Dec") - expect_equal(get_tracker_year(tracker_file, month_list), 2023) - expect_equal(get_tracker_year(tracker_file, wrong_month_list), 2022) -}) diff --git a/tests/testthat/test-helper_clean_data.R b/tests/testthat/test-helper_clean_data.R deleted file mode 100644 index f5a141a..0000000 --- a/tests/testthat/test-helper_clean_data.R +++ /dev/null @@ -1,8 +0,0 @@ -test_that("sanitized column name is correct", { - expect_equal(sanitize_str("keepso"), "keepso") - expect_equal(sanitize_str("remove whitespace"), "removewhitespace") - expect_equal(sanitize_str("John Doe's Column"), "johndoescolumn") - expect_equal(sanitize_str("Date 2022"), "date2022") - expect_equal(sanitize_str("My Awesome 1st Column!!"), "myawesome1stcolumn") - expect_equal(sanitize_str("Education/\nOcupation"), "educationocupation") -}) diff --git a/tests/testthat/test-helper_dates.R b/tests/testthat/test-helper_dates.R deleted file mode 100644 index e15b9b7..0000000 --- a/tests/testthat/test-helper_dates.R +++ /dev/null @@ -1,11 +0,0 @@ -test_that("parse_date_string works as expected", { - expect_equal(parse_dates("2020-01-01"), as.Date("2020-01-01")) - expect_equal(parse_dates("May-20"), as.Date("2020-05-01")) - expect_equal(parse_dates("May-99"), as.Date("1999-05-01")) - expect_equal(parse_dates("Jan-2021"), as.Date("2021-01-01")) - expect_equal(parse_dates("June2020"), as.Date("2020-06-01")) - expect_equal(parse_dates("June-2020"), as.Date("2020-06-01")) - expect_equal(parse_dates("08/05/2020"), lubridate::ymd("2020-05-08")) - expect_equal(parse_dates("6/10/17"), lubridate::ymd("2017-10-06")) - expect_equal(parse_dates("2020-05-08"), lubridate::ymd("2020-05-08")) -}) diff --git a/tests/testthat/test-helper_patient_data_fix.R b/tests/testthat/test-helper_patient_data_fix.R deleted file mode 100644 index 52b9747..0000000 --- a/tests/testthat/test-helper_patient_data_fix.R +++ /dev/null @@ -1,273 +0,0 @@ -ERROR_VAL_CHARACTER <<- "Other" -ERROR_VAL_NUMERIC <<- 999999 -ERROR_VAL_DATE <<- as.Date("9999-09-09") - -test_that("cut_numeric_value works", { - expect_equal(cut_numeric_value(1, min = 0, max = 5), 1) - expect_equal(cut_numeric_value(3.4, min = 0, max = 5), 3.4) - expect_equal(cut_numeric_value(3.4, max = 3.4, min = 1), 3.4) - expect_equal(cut_numeric_value(3.4, max = 3, min = 0), ERROR_VAL_NUMERIC) - expect_equal(cut_numeric_value(1, max = 5, min = 2), ERROR_VAL_NUMERIC) - expect_equal(cut_numeric_value(2, max = 3, min = 3), ERROR_VAL_NUMERIC) - expect_true(is.na(cut_numeric_value(NA, max = 3, min = 1))) -}) - - -test_that("fix_bmi works", { - test_df <- data.frame( - weight = c(1, 10, NA, 3, NA, 999999), - height = c(1, 2, 2, NA, NA, 1), - id = c("1", "2", "3", "4", "5", "6") - ) - expected_bmi <- c(1, 10 / 4, NA, NA, NA, 999999) - expect_equal( - test_df %>% - dplyr::rowwise() %>% - dplyr::mutate(bmi = fix_bmi(weight, height, id)) %>% - dplyr::select(bmi) %>% - dplyr::pull(), - expected_bmi - ) -}) - - -test_that("split_bp_in_sys_and_dias works", { - test_df <- data.frame(blood_pressure_mmhg = c("96/55", "101/57", NA)) - expected_df <- dplyr::tibble(blood_pressure_sys_mmhg = c("96", "101", NA), blood_pressure_dias_mmhg = c("55", "57", NA)) - expect_equal(split_bp_in_sys_and_dias(test_df), expected_df) - - test_df <- data.frame(blood_pressure_mmhg = c("96", "1,6")) - expected_df <- dplyr::tibble( - blood_pressure_sys_mmhg = c(as.character(ERROR_VAL_NUMERIC), as.character(ERROR_VAL_NUMERIC)), - blood_pressure_dias_mmhg = c(as.character(ERROR_VAL_NUMERIC), as.character(ERROR_VAL_NUMERIC)) - ) - expect_equal(split_bp_in_sys_and_dias(test_df), expected_df) -}) - - -test_that("convert_to works", { - # numeric - expect_equal(convert_to(1, as.numeric, ERROR_VAL_NUMERIC), 1) - expect_equal(convert_to(123.123, as.numeric, ERROR_VAL_NUMERIC), 123.123) - expect_equal(convert_to(0, as.numeric, ERROR_VAL_NUMERIC), 0) - expect_equal(convert_to(-0.01, as.numeric, ERROR_VAL_NUMERIC), -0.01) - expect_equal(convert_to("1234", as.numeric, ERROR_VAL_NUMERIC), 1234) - expect_equal(convert_to(".123", as.numeric, ERROR_VAL_NUMERIC), .123) - expect_equal(convert_to("a", as.numeric, ERROR_VAL_NUMERIC), ERROR_VAL_NUMERIC) - expect_true(is.na(convert_to(NA, as.numeric, ERROR_VAL_NUMERIC))) - - # logical - expect_true(convert_to(1, as.logical, FALSE)) - expect_false(convert_to(0, as.logical, FALSE)) - expect_true(convert_to(123154, as.logical, FALSE)) - expect_true(convert_to(-.123, as.logical, FALSE)) - expect_true(convert_to("true", as.logical, FALSE)) - expect_true(convert_to("TRUE", as.logical, FALSE)) - expect_true(convert_to("T", as.logical, FALSE)) - expect_false(convert_to("false", as.logical, FALSE)) - expect_false(convert_to("FALSE", as.logical, FALSE)) - expect_false(convert_to("F", as.logical, FALSE)) - expect_true(is.na(convert_to("", as.logical, FALSE))) - expect_true(is.na(convert_to(NA, as.logical, FALSE))) - - # integer - expect_equal(convert_to(1, as.integer, ERROR_VAL_NUMERIC), 1) - expect_equal(convert_to(123, as.integer, ERROR_VAL_NUMERIC), 123) - expect_equal(convert_to(-1, as.integer, ERROR_VAL_NUMERIC), -1) - expect_equal(convert_to(.1, as.integer, ERROR_VAL_NUMERIC), 0) - expect_equal(convert_to(-.1, as.integer, ERROR_VAL_NUMERIC), 0) - expect_equal(convert_to(123.123, as.integer, ERROR_VAL_NUMERIC), 123) - expect_equal(convert_to("1", as.integer, ERROR_VAL_NUMERIC), 1) - expect_equal(convert_to("1.5", as.integer, ERROR_VAL_NUMERIC), 1) - expect_true(is.na(convert_to("", as.integer, ERROR_VAL_NUMERIC))) - expect_equal(convert_to(T, as.integer, ERROR_VAL_NUMERIC), 1) - expect_true(is.na(convert_to(NA, as.integer, ERROR_VAL_NUMERIC))) - - # date - expect_equal(convert_to("2023-01-01", lubridate::as_date, ERROR_VAL_DATE), as.Date("2023-01-01")) - expect_equal(convert_to("2023", lubridate::as_date, ERROR_VAL_DATE), ERROR_VAL_DATE) - expect_equal(convert_to("45007", lubridate::as_date, ERROR_VAL_DATE), ERROR_VAL_DATE) - expect_true(is.na(convert_to("", lubridate::as_date, ERROR_VAL_DATE))) - expect_true(is.na(convert_to(NA, lubridate::as_date, ERROR_VAL_DATE))) -}) - - -test_that("fix_age works", { - dob <- "2010-05-01" - expect_equal(fix_age(age = 10, dob = dob, tracker_year = 2020, tracker_month = 6, id = "1"), 10) - expect_equal(fix_age(age = 10, dob = dob, tracker_year = 2020, tracker_month = 4, id = "1"), 9) -}) - - -test_that("fix_sex works", { - # Test case 1: Test with a female gender - expect_equal(fix_sex("female", "1"), "F", - info = "For sex 'female', expected 'F'" - ) - - # Test case 2: Test with a male gender - expect_equal(fix_sex("man", "2"), "M", - info = "For sex 'man', expected 'M'" - ) - - # Test case 3: Test with an empty gender - expect_true(is.na(fix_sex("", "3")), - info = "For empty sex, expected NA" - ) - - # Test case 4: Test with another# gender not in the synonyms list - expect_equal(fix_sex("unknown", "4"), ERROR_VAL_CHARACTER, - info = "For sex 'unknown', expected 'Other'" - ) -}) - - -test_that("extract_year_from_age works", { - expect_equal(extract_year_from_age("2y9m"), "2") - expect_equal(extract_year_from_age("10y10m"), "10") - expect_equal(extract_year_from_age("7y6m"), "7") -}) - - -# Test case for detecting 'birth' in t1d_diagnosis_age -test_that("fix_t1d_diagnosis_age works", { - expect_equal(fix_t1d_diagnosis_age("birth", "1"), "0") - expect_equal(fix_t1d_diagnosis_age("At birth", "1"), "0") - - # Test case for detecting 'born' in t1d_diagnosis_age - expect_equal(fix_t1d_diagnosis_age("born", "1"), "0") - - # Test case for detecting 'month' in t1d_diagnosis_age - expect_equal(fix_t1d_diagnosis_age("4 months", "1"), "0") - - # Test case for detecting 'y' in t1d_diagnosis_age - expect_equal(fix_t1d_diagnosis_age("5y", "1"), "5") - expect_equal(fix_t1d_diagnosis_age("10y10m", "2"), "10") - - # Test case for handling NA values in t1d_diagnosis_age - expect_true(is.na(fix_t1d_diagnosis_age(NA, "1"))) - - # Test case for default case - expect_equal(fix_t1d_diagnosis_age("10", "1"), "10") - expect_equal(fix_t1d_diagnosis_age("0", "1"), "0") - expect_equal(fix_t1d_diagnosis_age("1", "1"), "1") -}) - - -test_that("correct_decimal_sign works", { - expect_equal(correct_decimal_sign("12,2"), "12.2") - expect_equal(correct_decimal_sign(""), "") - expect_true(is.na(correct_decimal_sign(NA))) -}) - - -test_that("fix_fbg works", { - expect_equal(fix_fbg("High"), "200") - expect_equal(fix_fbg("Bad"), "200") - expect_equal(fix_fbg("Hi"), "200") - - expect_equal(fix_fbg("Medium"), "170") - expect_equal(fix_fbg("Med"), "170") - - expect_equal(fix_fbg("Low"), "140") - expect_equal(fix_fbg("Good"), "140") - expect_equal(fix_fbg("Okay"), "140") - - expect_equal(fix_fbg("123 (DKA)"), "123") - - expect_true(is.na(fix_fbg(NA))) - expect_true(is.na(fix_fbg(""))) -}) - - -test_that("fix_digit_date works", { - expect_equal(fix_digit_date(44693), "2022-05-12") - expect_equal(fix_digit_date("44693"), "2022-05-12") - expect_equal(fix_digit_date("2021-05-12"), "2021-05-12") -}) - - -test_that("check_allowed_values works", { - valid_support_values <- c( - "Standard", - "Partial", - "Semi-Partial", - "SAC", - "Monitoring" - ) - expect_equal(check_allowed_values("partial", valid_support_values, 1), "Partial") - expect_equal(check_allowed_values("Standard", valid_support_values, 1), "Standard") - expect_equal(check_allowed_values("SAc", valid_support_values, 1, ), "SAC") - expect_true(is.na(check_allowed_values("", valid_support_values, 1))) - expect_true(is.na(check_allowed_values(NA, valid_support_values, 1))) - expect_equal(check_allowed_values("abc", valid_support_values, 1, error_val = ERROR_VAL_CHARACTER), ERROR_VAL_CHARACTER) - expect_equal(check_allowed_values("abc", valid_support_values, 1, replace_invalid = FALSE), "abc") -}) - - -test_that("fix_testing_frequency works", { - patient_id <- "XX_YY001" - expect_equal(fix_testing_frequency("2", patient_id), "2") - expect_equal(fix_testing_frequency("1.5", patient_id), "1.5") - expect_equal(fix_testing_frequency("0-2", patient_id), "1") - expect_equal(fix_testing_frequency("2-3", patient_id), "2.5") - expect_true(is.na(fix_testing_frequency("", patient_id))) - expect_true(is.na(fix_testing_frequency(NA, patient_id))) -}) - - -test_that("extract_date_from_measurement works", { - test_df <- data.frame( - measurement = c( - "8.53 (28/8/2017)", - "10,75 (31/12/2020)", - "1075 (31/12/2020)", - "10.75 (31/12/2020)", - "10.75 (31-12-2020)", - "15.42 (2022-01)", - "10.75 (01-Sept-2022)", - "10.75 (01-Sept-2022", - "10.75(01-2017)", - "3" - ) - ) - expected_df <- dplyr::as_tibble(data.frame( - measurement = c( - "8.53 ", - "10,75 ", - "1075 ", - "10.75 ", - "10.75 ", - "15.42 ", - "10.75 ", - "10.75 ", - "10.75", - "3" - ), - measurement_date = c( - "28/8/2017", - "31/12/2020", - "31/12/2020", - "31/12/2020", - "31-12-2020", - "2022-01", - "01-Sept-2022", - "01-Sept-2022", - "01-2017", - NA - ) - )) - - expect_equal(extract_date_from_measurement(test_df, "measurement"), expected_df) -}) - - -test_that("fix_id works", { - expect_equal(fix_id("KD_EW004"), "KD_EW004") - expect_equal(fix_id("K_EW004"), "Other") - expect_equal(fix_id("KD_E004"), "Other") - expect_equal(fix_id("KD_EWX04"), "Other") - expect_equal(fix_id("11_EW004"), "Other") - expect_equal(fix_id("KD_E1004"), "Other") - expect_equal(fix_id("KD_EW004XY"), "KD_EW004") -}) diff --git a/tests/testthat/test-read_column_synonyms.R b/tests/testthat/test-read_column_synonyms.R deleted file mode 100644 index b8a96cd..0000000 --- a/tests/testthat/test-read_column_synonyms.R +++ /dev/null @@ -1,23 +0,0 @@ -test_that("read_column_synonyms works", { - patient_synonyms <- read_column_synonyms( - "synonyms_patient.yaml", - path_prefixes = c("test_data", "synonyms") - ) - - expect_equal(colnames(patient_synonyms), c("variable_name", "tracker_name")) - - - # TODO(KW) Medium: This test should be refactored such that it does not test - # the content the test data, but only code logic. Testing the content of the - # test data is meaningless unless it is maintained to be up to date with the - # actual reference data, and even in that case valid configuration changes - # should not be able to cause tests to fail. - age_synonyms <- patient_synonyms %>% dplyr::filter(variable_name == "age") - expect_setequal(age_synonyms$tracker_name, c("Age", "Age*", "age on reporting", "Age (Years)")) - - baseline_synonyms <- patient_synonyms %>% dplyr::filter(grepl(variable_name, pattern = "baseline_fbg")) - expect_contains(baseline_synonyms$tracker_name, c( - "FBG (mg/dL) - BASELINE", "Baseline SMBG or CBG (mg/dL)", "Baseline FBG (mg/dL)", - "Baseline FBG (mg%)", "Baseline FBG or CBG (mg/dL)" - )) -}) diff --git a/tests/testthat/test_data/synonyms/synonyms_patient.yaml b/tests/testthat/test_data/synonyms/synonyms_patient.yaml deleted file mode 100644 index a0e9caa..0000000 --- a/tests/testthat/test_data/synonyms/synonyms_patient.yaml +++ /dev/null @@ -1,210 +0,0 @@ -additional_support: -- Potential additional support or intervention from A4D e.g. complications or family support -age: -- Age -- Age* -- age on reporting -- Age (Years) -age_diagnosis: -- Age at diagnosis of T1DM -- Age at Diagnosis -- age of diagnosis -baseline_fbg: -- FBG (mg/dL) - BASELINE -- Baseline SMBG or CBG (mg/dL) -- Baseline FBG (mg/dL) -- Baseline FBG (mmol/L) -- Baseline FBG (mg%) -- Baseline FBG or CBG (mg/dL) -baseline_fbg_unit: -- baseline_fbg_unit -baseline_hba1c: -- HbA1c (%) - BASELINE -- Baseline HbA1c (%) -blood_pressure_dias_mmhg: [] -blood_pressure_sys_mmhg: [] -bmi: -- BMI -- BMI* -bmi_date: -- Date of BMI (mmm-yy) -- Date of BMI (mm-yy) -- date of bmi date -clinic_id: [] -dob: -- D.O.B. -- d.o.b. (dd-mm-yyyy) -- D.O.B. (dd-mmm-yyyy) -- DOB..ANON -edu_occ: -- Education (School Grade) Or Occupation -- Level of Education Or Occupation (Date if Applicable or NA) -- Education (School Grade) -- Education...Ocupation -- Level of Education Or Occupation -est_strips_pmoth: -- Estimated testing strips per month -gender: -- Gender -height: -- Height (Meters) -hospitalisation: -- Last hospitalisation due to diabetes emergency or glucose control -- Hospitalisation due to diabetes emergency or glucose control (Date if Applicable or NA) -- Hospitalisation due to diabetes emergency or glucose control (Include Date) -- Hospitalisation due to diabetes emergency -- Hospitalisation due to diabetes emergency or glucose control (Insert Date or NA) -- hospitalisation due to diabetes emergency or glucose control egdka include date -patient_id: -- ID -- Patient ID -- Patient ID* -insulin_dosage: -- insulin dosage -- insulin dosage tdd -insulin_regimen: -- Insulin Regimen -- Insuline regime -- Insulin Regime -- insulin regimen1 -last_clinic_visit_date: -- Last Clinic Visit -- Last Clinic Visit (dd-mmm-yy) -- last clinic visit date -latest_complication_screening_date: [] -latest_complication_screening_type: [] -no: -- No -patient_name: -- Patient Name -- Patient.Name..ANON -province: -- Province -recruitment_date: -- Date of Recruitment -- Date of recruitment into A4D Programme -- date of recruitment into a4d programme (mmm-yy) -- date of recruitment mmmyy -- Date of Recruitment into A4D -remarks: -- Any opportunity or intervention that may require support from A4D e.g. complications or family support Remarks (If Applicable) -- Complications e.g. (1,3,4) -- remarks -sheet_name: [] -status: -- Status -- patient status -support_from_a4d: -- Support received from A4D -- Level of Support -- level of support dropdown -- Support required from A4D -testing_fqr_pday: -- Testing Frequency (per day) -tracker_year: [] -updated_fbg_date: -- Updated FBG Date -- Updated FBG or CBG Date -updated_fbg: -- FBG (mg/dL) - UPDATED -- Updated FBG (mg/dL) -- Updated FBG (mmol/L) -- Updated FBG FBG (mg/dL) -- updated fbg fbg mgdl -- Updated FBG mg/dl -- Updated FBG ml/dL -- Updated FBG or CBG mg/dl -- Updated FBG or CBG mmol/L -- Updated Mean FBG (mg/%) -- Updated Mean FBG (mmol/L) -- Updated Mean FBG (mg/dL) -- Updated SMBG or CBG (mg/dL) Date -- Updated SMBG or CBG mg/dL -updated_hba1c_date: -- Updated HbA1c Date -updated_hba1c: -- HbA1c (%) - UPDATED -- Updated HbA1c (%) -- Updated HbA1c % -- updated hba1c hba1c -weight: -- Body Weight (Kg) -blood_pressure_mmhg: -- Blood Pressure (mm HG) -latest_complication_screenning: -- Complication Screening (starting from Q2 2021) -- Latest Complication Screening (select from list) -- Latest Complication Screening (dd-mmm-yy) -qol_survey: -- QoL Survey (Est. Q3 2021) -required_insulin: -- estimated insulin required per year -required_insulin_product_name: -- estimated insulin required per year col18 -- Estimated Insulin Required per year col18 -- Estimated Insulin Required per year.1 -family_support_scale: -- Family Support Scale -dm_complication_comment: -- Comments on other complications -dm_complication_other: -- Others (Y/N) -dm_complication_kidney: -- Kidney (Y/N) -dm_complication_eye: -- Eye (Y/N) -num_admin_hosp_total: -- total -num_admin_hosp_dka: -- Number DKA -num_admin_hosp_hypo: -- Number Hypo -num_admin_hosp_other_reason: -- Reason for Others -num_admin_hosp_other: -- Number Others -inactive_reason: -- Inactive Reason -lost_date: -- Lost/ Deceased Date -lost_age: -- Lost/ Deceased Age -diag_date: -- Date of Diagnosis -- Date of Diagnosis (mmm-yy) -dka_diag: -- dkaatdiagnosisyn -clinic_visit_currmonth: -- clinicvisitcurrentmonth -- clinicvisit -remote_followup_currmonth: -- remote follow up current month -- remote follow up -last_remote_followup_date: -- last remote follow update -currmonth_hospitalisation_cause: -- current month hospitalisation dka hypo other dropdown -currmonth_hospitalisation_date: -- current month hospitalisation dka hypo other date -updated_2022_date: -- updated 2022 date -currmonth_complication_screening: -- currentmonthcomplicationscreeningkidneyeyefootdropdown -- currentmonthcomplicationscreeningkidneyeyefootdropdown1 -- currentmonthcomplicationscreeningkidneyeyefootdropdown2 -- currentmonthcomplicationscreeningkidneyeyefootdropdown3 -- currentmonthcomplicationscreeningkidneyeyefootdropdown4 -currmonth_complication_screening_date: -- currentmonthcomplicationscreeningkidneyeyefootdate -currmonth_complication_screening_results: -- currentmonthcomplicationscreeningkidneyeyefootresults -current_patient_observations: -- currentpatientobservations -current_patient_observations_category: -- currentpatientobservationscategory -meterreceiveddate: -- meterreceiveddate -- isensmeterreceiveddate -- wizmeterreceiveddate -required_insulin_month: -- estimatedinsulinrequiredpermonth diff --git a/tools/LogViewerA4D/.gitignore b/tools/LogViewerA4D/.gitignore deleted file mode 100644 index 861b542..0000000 --- a/tools/LogViewerA4D/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -.Rproj.user -.Rhistory -.RData -.Ruserdata -temp.parquet -rsconnect/* diff --git a/tools/LogViewerA4D/LogViewerA4D.Rproj b/tools/LogViewerA4D/LogViewerA4D.Rproj deleted file mode 100644 index 8e3c2eb..0000000 --- a/tools/LogViewerA4D/LogViewerA4D.Rproj +++ /dev/null @@ -1,13 +0,0 @@ -Version: 1.0 - -RestoreWorkspace: Default -SaveWorkspace: Default -AlwaysSaveHistory: Default - -EnableCodeIndexing: Yes -UseSpacesForTab: Yes -NumSpacesForTab: 2 -Encoding: UTF-8 - -RnwWeave: Sweave -LaTeX: pdfLaTeX diff --git a/tools/LogViewerA4D/global.R b/tools/LogViewerA4D/global.R deleted file mode 100644 index c8217b1..0000000 --- a/tools/LogViewerA4D/global.R +++ /dev/null @@ -1,32 +0,0 @@ -library(shiny) -library(shinydashboard) -library(DT) -library(tidyverse) -library(arrow) -library(plotly) -library(ggplot2) - - -levelsValues <- c("TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL") -colorLevels <- c("TRACE", "DEBUG", "WARN", "ERROR", "FATAL") -colors <- c(rgb(0.8, 0.9, 1), rgb(0.8, 1, 0.8), rgb(1.0, 0.88, 0.7), rgb(1, 0.84, 0.8), rgb(1, 0.8, 0.94)) -truncScript <- "function(data, type, row, meta) {\n - return type === 'display' && data != null && data.length > 100 ?\n - '' + data.substr(0, 100) + '...' : data;\n - }" - - -parseLines <- function(lines) { - rows <- strsplit(lines, "\t") - malformed <- sapply(rows, function(x) length(x) != 6) - rows <- rows[!malformed] - result <- data.frame( - Timestamp = as.POSIXct(sapply(rows, function(x) x[1])), - Thread = sapply(rows, function(x) x[2]), - Level = sapply(rows, function(x) x[3]), - Package = sapply(rows, function(x) x[4]), - Function = sapply(rows, function(x) x[5]), - Message = sapply(rows, function(x) x[6]) - ) - return(result) -} diff --git a/tools/LogViewerA4D/server.R b/tools/LogViewerA4D/server.R deleted file mode 100644 index d285b89..0000000 --- a/tools/LogViewerA4D/server.R +++ /dev/null @@ -1,297 +0,0 @@ -# This file contains the server-side code for a Shiny application that serves as a log viewer. -# It loads log data from a Parquet file, allows uploading of log files, and displays the log data in a DataTable. -# The server code defines various reactive expressions and event observers to handle user interactions and data updates. -# It also renders the UI elements and defines the behavior of the log table and details panel. -options(shiny.maxRequestSize = 30 * 1024^2) -# Define the shinyServer function to handle the server-side logic of the Shiny application. -shinyServer(function(input, output, session) { - # Define a reactiveValues object to store the event log data and the current table data. - values <- reactiveValues(eventLog = NULL, currentTable = NULL, proccessedFilesInfo = NULL, clinic_info_df = NULL) - - # Define a reactive expression to get the selected row from the log table. - selectedRow <- reactive({ - idx <- input$logTable_rows_selected - if (is.null(idx)) { - return(NULL) - } else { - row <- values$currentTable[idx, ] - return(row) - } - }) - - # Observer for the "Load Last Saved Log-Data" button click event. - observeEvent(input$loadTempFile, { - print("Load Old Data") - values$eventLog <- read_parquet("temp.parquet") - }) - - - # Render the details panel to display the selected log entry details. - observeEvent(input$logTable_cell_clicked, { - row <- selectedRow() - if (is.null(row)) { - return(NULL) - } else { - lines <- list( - "", - sprintf("", row$Timestamp), - sprintf("", row$Level), - sprintf("", row$script), - sprintf("", row$fileName), - sprintf("", row$functionName), - sprintf("", row$errorCode), - sprintf("", row$warningCode), - "
Timestamp  %s
Level  %s
Script  %s
File  %s
Function  %s
ErrorCode  %s
WarningCode  %s
", - "

Message

", - sprintf("

%s

", row$message) - ) - - showModal(modalDialog( - tags$div(id = session$ns("constraintPlaceholder")), - fade = F, - easyClose = TRUE, - footer = NULL - )) - insertUI( - selector = paste0("#", session$ns("constraintPlaceholder")), - where = "afterEnd", - ui = HTML(paste(lines, collapse = "\n")) - ) - } - }) - - - # Observer for the file upload event. - observeEvent(input$fileUpload, { - req(input$fileUpload) - allLogs <- tibble() - withProgress(message = "Creating Log DF", value = 0, { - for (x in 1:length(input$fileUpload$datapath)) { - tryCatch( - { - filePath <- input$fileUpload$datapath[x] - x <- input$fileUpload$name[x] - fileConnection <- file(filePath, open = "r") - lines <- readLines(fileConnection) - close(fileConnection) - - if (!is.null(lines)) { - result <- parseLines(lines) %>% - mutate(fileName = basename(x)) %>% - mutate(fileNameParts = list(str_split(basename(x), "_") %>% - first())) %>% - unnest_wider(fileNameParts, names_sep = "_") - allLogs <- allLogs %>% - dplyr::bind_rows(result) %>% - mutate(across(where(is.character), as.factor)) %>% - mutate(Message = as.character(Message)) - } - }, - error = function(cond) { - message("Error message:") - message(conditionMessage(cond)) - NA - }, - warning = function(cond) { - message("Warning message:") - message(conditionMessage(cond)) - NULL - }, - finally = { - incProgress(1 / length(input$fileUpload$datapath), detail = paste("Processed file ", x)) - message("Some other message at the end.Processed file ", x) - } - ) - } - }) - - values$eventLog <- allLogs %>% - mutate(tracker_in_fileName = gsub("logs_\\d{4}_(.*?)\\ A4D.*", "\\1", fileName)) %>% - select(-"tracker_in_fileName") - - values$eventLog <- values$eventLog %>% - dplyr::filter(substr(Message, 1, 1) == "{") %>% - mutate(Message = map(Message, ~ jsonlite::fromJSON(.) %>% - lapply(., function(x) if (is.null(x)) NA else if (is.list(x)) toString(x) else x) %>% - as_tibble())) %>% - unnest(Message) %>% - rename(Message = message) %>% - mutate(across(c("file", "errorCode", "warningCode", "functionName"), factor)) - - values$proccessedFilesInfo <- allLogs %>% - filter(str_detect(pattern = "Found", Message)) %>% - extract(Message, into = c("number", "type"), regex = "Found (\\d+) (\\w+)") %>% - select(fileName, type, number) %>% - distinct(fileName, type, number) - write_parquet(allLogs, "temp.parquet") - }) - - - # Render the UI for the "Load Last Saved Log-Data" button if the "temp.parquet" file exists. - output$loadTempFileUI <- renderUI({ - if (file.exists("temp.parquet")) { - actionButton("loadTempFile", "Load Last Saved Log-Data") - } else { - NULL - } - }) - - output$sankey <- renderPlotly({ - req(values$proccessedFilesInfo) - - - fig <- plot_ly( - type = "sankey", - orientation = "h", - node = list( - label = unique(paste(values$proccessedFilesInfo$fileName, values$proccessedFilesInfo$type)), - color = c("blue", "blue", "blue", "blue", "blue"), - pad = 15, - thickness = 20, - line = list( - color = "black", - width = 0.5 - ) - ), - link = list( - source = c(0, 0, 0, 1, 2), - target = c(0, 1, 2, 3, 4), - value = values$proccessedFilesInfo$number - ) - ) - fig <- fig %>% layout( - title = "Basic Sankey Diagram", - font = list( - size = 10 - ) - ) - fig - }) - - - # Render the log table using the DataTable library. - output$logTable <- renderDT({ - req(values$eventLog) - - options <- list( - searching = TRUE, - autoWidth = TRUE, - paging = TRUE, - pageLength = 20, - lengthChange = T, - dom = '<"top" p>tB', - buttons = c("copy", "csv", "excel"), - scrollX = T, - ordering = T, - info = T, - columnDefs = list(list( - targets = c(5), - render = JS(truncScript) - )) - ) - selection <- list(mode = "single", target = "row") - table <- datatable(values$eventLog, - extensions = "Buttons", - options = options, - filter = list( - position = "top", clear = FALSE - ), - selection = selection, - rownames = FALSE, - escape = FALSE - ) - table <- formatStyle( - table = table, - columns = 3, - target = "row", - backgroundColor = styleEqual(colorLevels, colors) - ) - return(table) - }) - - - observeEvent(input$logTable_rows_all, { - req(values$eventLog) - values$currentTable <- values$eventLog[input$logTable_rows_all, ] - }) - - # Render the status table to display the count of log entries by file name and level. - output$status <- renderDT( - { - req(values$eventLog) - values$currentTable %>% dplyr::count(fileName, Level) - }, - filter = list( - position = "top", clear = FALSE - ) - ) - - output$logOverviewPlot <- renderPlotly({ - req(values$eventLog) - aggregatedData <- values$currentTable %>% - dplyr::count(fileName, Level) %>% - filter(grepl(input$regexFilter, fileName)) - p <- ggplot(data = aggregatedData, aes(x = Level, y = fileName, fill = n)) + - geom_tile() + - scale_y_discrete(label = abbreviate) + - theme(axis.text = element_text(size = 6)) - plotly::ggplotly(p, height = 800, width = 600) - }) - - output$trackerSummary <- renderDT({ - req(values$eventLog) - datatable( - values$eventLog %>% group_by(fileName, Level) %>% count() %>% - pivot_wider(names_from = Level, values_from = n), - filter = list( - position = "top", clear = FALSE - ), - extensions = "Buttons", - options = list( - paging = TRUE, - searching = TRUE, - fixedColumns = TRUE, - autoWidth = TRUE, - ordering = TRUE, - dom = "tB", - buttons = c("copy", "csv", "excel") - ), - rownames = FALSE, - escape = FALSE - ) - }) - - # Display reference_data df in tabItem Reference_Data - output$ref_data_df <- renderDT({ - req(values$clinic_info_df) - clinic_info_df <- values$clinic_info_df %>% select(-clinic_start_date, -clinic_quota_22) - - missing_entries_df <- clinic_info_df %>% - filter(is.na(country_name)) - multiple_entries_df <- clinic_info_df %>% - group_by(clinic_code) %>% - filter(n() > 1) - - col_filter <- switch(input$choose_tab, - "all" = clinic_info_df, - "missing entries" = missing_entries_df, - "multiple entries" = multiple_entries_df - ) - DT::datatable( - { - col_filter - }, - extensions = "Buttons", - options = list( - paging = TRUE, - searching = TRUE, - fixedColumns = TRUE, - autoWidth = TRUE, - ordering = TRUE, - dom = "tB", - buttons = c("copy", "csv", "excel") - ), - class = "display" - ) - }) -}) diff --git a/tools/LogViewerA4D/ui.R b/tools/LogViewerA4D/ui.R deleted file mode 100644 index 2fb43cc..0000000 --- a/tools/LogViewerA4D/ui.R +++ /dev/null @@ -1,83 +0,0 @@ -# This file contains the user interface code for the Log File Viewer - A4D application. -# It uses the shiny and shinydashboard libraries to create a dashboard-style interface. -# The UI consists of a header, sidebar, and body, with multiple tabs for different views. -# The user can upload log files, select a log level, and view the log data in a table. -# There is also a details section for displaying additional information about selected log entries. -# The "Overview" tab provides a summary view of the log data. -# The fileInput and selectInput functions are used to create file upload and level selection controls, respectively. -# The DTOutput and DT functions from the DT library are used to render the log table and status table. - - -shinyUI( - dashboardPage( - dashboardHeader(title = "Log File Viewer - A4D"), - dashboardSidebar( - fluidRow( - uiOutput("loadTempFileUI"), - fileInput( - "fileUpload", - "Upload Log Files", - multiple = TRUE, accept = ".log" - ) - ), - sidebarMenu( - menuItem("Dashboard", tabName = "dashboard", icon = icon("dashboard")), - menuItem("Overview", tabName = "overview", icon = icon("th")), - menuItem("Reference_Data", tabName = "ref_data", icon = icon("clipboard")) - ) - ), - dashboardBody( - tabItems( - tabItem( - tabName = "dashboard", - fluidRow( - box( - width = 8, - title = "Main Table", - DTOutput("logTable") - ), - box( - width = 4, - title = "Tracker Overview", - DTOutput("trackerSummary"), - plotlyOutput("sankey") - ), - ) - ), - tabItem( - tabName = "overview", - fluidRow(box( - title = "Overview", - DTOutput("status") - )), - fluidRow(box( - title = "Overview-Plot", height = 1000, weight = 1000, - textInput("regexFilter", label = "RegexFilter Trackerfiles"), - plotlyOutput("logOverviewPlot") - )) - ), - tabItem( - tabName = "ref_data", - fluidPage( - sidebarLayout( - sidebarPanel( - h3("Checking reference_data"), - br(), - p("We use reference_data to match clinic information - to clinic_code used in messages."), - radioButtons("choose_tab", - label = "Choose Output", - choices = c("all", "missing entries", "multiple entries"), - selected = "all" - ) - ), - mainPanel( - DTOutput("ref_data_df") - ) - ) - ) - ) - ) - ) - ) -) diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..ff278d8 --- /dev/null +++ b/uv.lock @@ -0,0 +1,1152 @@ +version = 1 +revision = 3 +requires-python = ">=3.14" + +[[package]] +name = "a4d" +version = "2.0.0" +source = { editable = "." } +dependencies = [ + { name = "fastexcel" }, + { name = "google-cloud-bigquery" }, + { name = "google-cloud-storage" }, + { name = "loguru" }, + { name = "openpyxl" }, + { name = "pandera", extra = ["polars"] }, + { name = "polars" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "rich" }, + { name = "tqdm" }, + { name = "typer" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pre-commit" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "pytest-mock" }, + { name = "ruff" }, + { name = "ty" }, +] + +[package.metadata] +requires-dist = [ + { name = "fastexcel", specifier = ">=0.16.0" }, + { name = "google-cloud-bigquery", specifier = ">=3.17.0" }, + { name = "google-cloud-storage", specifier = ">=2.14.0" }, + { name = "loguru", specifier = ">=0.7.0" }, + { name = "openpyxl", specifier = ">=3.1.0" }, + { name = "pandera", extras = ["polars"], specifier = ">=0.18.0" }, + { name = "polars", specifier = ">=0.20.0" }, + { name = "pydantic", specifier = ">=2.6.0" }, + { name = "pydantic-settings", specifier = ">=2.2.0" }, + { name = "python-dateutil", specifier = ">=2.8.0" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "rich", specifier = ">=13.7.0" }, + { name = "tqdm", specifier = ">=4.66.0" }, + { name = "typer", specifier = ">=0.9.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pre-commit", specifier = ">=4.3.0" }, + { name = "pytest", specifier = ">=8.4.2" }, + { name = "pytest-cov", specifier = ">=7.0.0" }, + { name = "pytest-mock", specifier = ">=3.15.1" }, + { name = "ruff", specifier = ">=0.14.1" }, + { name = "ty", specifier = ">=0.0.1a23" }, +] + +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, + { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, + { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, +] + +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, + { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, + { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" }, + { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" }, + { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" }, + { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, + { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" }, + { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.15.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/c3/4f2195f512fb172aa425a8803a874b2baa9ba7f80ff7b6080998761fc701/coverage-7.15.4.tar.gz", hash = "sha256:0548198fff07ccf4faf469520bce1c2eceb1ce3e62891921138dec10907f9d00", size = 936952, upload-time = "2026-08-06T13:50:24.442Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/ac/748cf29eeb2d6be34a3176ce26a4f49e38085ee08e8935f05f6f26ed7e0f/coverage-7.15.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:770e9325ab5ea6d56f77e59b29ecfe0ac20b57a82a601876f90494a4dda0386f", size = 222608, upload-time = "2026-08-06T13:48:26.806Z" }, + { url = "https://files.pythonhosted.org/packages/0b/02/1abbf5c984677b0aa439cdacaccbf38d248939d8ef8fe1cc7a50d73edb77/coverage-7.15.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d12b33a3a50a1676b7784dc8d00a0c6d66a9f2add4b85a041c19b6a7e53ef23c", size = 222940, upload-time = "2026-08-06T13:48:28.432Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e1/ff8f9f53d9fcf586125b55d0b1f04ec1c14955fee41e83d5814bee141bb5/coverage-7.15.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5669c8378ebde86f5def7a25d29586631b58acc27ffde04399f678f3dfc6e082", size = 253985, upload-time = "2026-08-06T13:48:29.995Z" }, + { url = "https://files.pythonhosted.org/packages/a1/26/595759762e514e81be1d7d01ed03444303bcd152226a6529998d253f9201/coverage-7.15.4-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ff97a14362eef486483ed44042ca2027ea257df6ff768e62358ee0c9776925ac", size = 256492, upload-time = "2026-08-06T13:48:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/24/68/b79aabac54d482be23b5fcdd4f4662bff24a78edc4ee29201726929936d5/coverage-7.15.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a325e815318638aed1655d9c06e6d7c2d3d46c09231ce988070428a8762d734", size = 257837, upload-time = "2026-08-06T13:48:33.186Z" }, + { url = "https://files.pythonhosted.org/packages/09/0f/bf7f297885a5bf6fd71e5782404e0ff059ca09e8711ceb3a08544abde45a/coverage-7.15.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:474223409d88eb20d2d6a0d37ea60e8647a65a90cc008dc1f0410af5f64f1e0d", size = 260152, upload-time = "2026-08-06T13:48:34.75Z" }, + { url = "https://files.pythonhosted.org/packages/fd/f1/296744e854ff8368542343457414380465e9ceefb9192342feb9d3bc461d/coverage-7.15.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f2f62ae3cd189dd2e13aece758c57b3eecbd27be070dbd4cbd10936049e5dbf", size = 253978, upload-time = "2026-08-06T13:48:36.434Z" }, + { url = "https://files.pythonhosted.org/packages/55/b0/bbdb2e9057493e66220a2e149ca2d301ba0e3a58a83bd6b90de9826d16f3/coverage-7.15.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39ece820e29e0a2ba34b3ecb3be83c27e997eed8926f2ba6fe7ce7a0bda5843b", size = 255846, upload-time = "2026-08-06T13:48:38.317Z" }, + { url = "https://files.pythonhosted.org/packages/96/e4/38015b2b6d21258713bd17e76b59d033b191efb5703589cffd037dfbca20/coverage-7.15.4-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f21b56dcace11dfe013014201f577dcd592b2a9b72182d930361b47cf6f73f25", size = 253808, upload-time = "2026-08-06T13:48:39.993Z" }, + { url = "https://files.pythonhosted.org/packages/0b/64/0d515c1e60ee6fbfd1a0e79c07cd87d388a233b7adc37758735677203808/coverage-7.15.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:93a3a0b662abcc10c73a47cbc72cd60f63618d6989fb2d1286e50eacd974f303", size = 258081, upload-time = "2026-08-06T13:48:41.971Z" }, + { url = "https://files.pythonhosted.org/packages/91/71/04d9e7a3642146c6351338aef4ef85ab11dbbb54744c13245caba1aad1c0/coverage-7.15.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:141fae2cabf5569b782c10afc4c850ce10f618c13f8db54765cba99cc839da1f", size = 253624, upload-time = "2026-08-06T13:48:43.731Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a7/6c28b74c81ebff66987b0e2522ba5cffa3e90b0c33cb6a2eb264d4ee8cf1/coverage-7.15.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:81294c7e6ab30c5f74c0353b11b2fd6320e72d9bee6ac73b357caa8b916323a5", size = 255280, upload-time = "2026-08-06T13:48:45.58Z" }, + { url = "https://files.pythonhosted.org/packages/52/af/bc19996a7014b98d7bbb0f0939453c67074af65784a3aa16a789a07381fa/coverage-7.15.4-cp314-cp314-win32.whl", hash = "sha256:7bbd7d6418e0dab31a206af5203bd43ae36edb8e7fba1940b055d3e9249290d7", size = 224768, upload-time = "2026-08-06T13:48:47.525Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/219484e476d6e101ba0a444852579e05f5b75c37c611a42ed1190f73ef62/coverage-7.15.4-cp314-cp314-win_amd64.whl", hash = "sha256:f0204ed122758782970526057093f448051a39db9d810d4e344bb87a3546f425", size = 225259, upload-time = "2026-08-06T13:48:49.513Z" }, + { url = "https://files.pythonhosted.org/packages/b7/66/fa77daf4e383e5f776dac62c2409b6af81910ae6fe326bd5170dba74cc63/coverage-7.15.4-cp314-cp314-win_arm64.whl", hash = "sha256:9e71e7bc71c686a123347ae47a0de33a175e797a85bb57b791492adf4eec8ed8", size = 224684, upload-time = "2026-08-06T13:48:51.235Z" }, + { url = "https://files.pythonhosted.org/packages/58/5b/f03bf0ce362bbf3f785fa5219620d00778d4ac6fc9e407734828e9c672f6/coverage-7.15.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7c922735321eef3f87c280a3d39afff6b646723a2880b862cda4ac7a093b8aa8", size = 223338, upload-time = "2026-08-06T13:48:52.896Z" }, + { url = "https://files.pythonhosted.org/packages/0f/76/e77d0ae22501831cc9f92193e8a957a5caa1dd177f90a6d1d9b106242d92/coverage-7.15.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f41c17c4668a655ce96d090d8d5ffdc24ef64b5a02f9753884d08483e8a4a41a", size = 223609, upload-time = "2026-08-06T13:48:54.688Z" }, + { url = "https://files.pythonhosted.org/packages/82/1a/b1f089da8d38ac612fa2dd6dc7f4a1a7657d12f3e261d2996edd3a838d0b/coverage-7.15.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:46822e9b6ff1c6a72b518c162c44a8f45a61a1d609c51084bf5b16c023c5037b", size = 264970, upload-time = "2026-08-06T13:48:56.403Z" }, + { url = "https://files.pythonhosted.org/packages/bf/31/e66d98d6e9c7fcc88470f1e234eaf6b1950dc0dfbf797f7282c1c861da24/coverage-7.15.4-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d6f4955b73b5445271379a59e3792b0d978f42d4a01e0cf7a67d9c33a3bb0a5", size = 267088, upload-time = "2026-08-06T13:48:58.41Z" }, + { url = "https://files.pythonhosted.org/packages/59/a1/ae94eb2c541add426378408379f233591e069040b1e2cdb33df9498a0682/coverage-7.15.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3fc9e047706fb4a9abb54f719d3aa643e80e5bb3818182c40aee01ac0f0247ba", size = 269508, upload-time = "2026-08-06T13:49:00.42Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c7/88a10694a1c6a213569766aba9f25847b28155d4ac731b13226db216356d/coverage-7.15.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05e491d4f3165d62d4f5c8fd48dfeabf2ae8f42cbbd484319af33ea851b78982", size = 270629, upload-time = "2026-08-06T13:49:02.234Z" }, + { url = "https://files.pythonhosted.org/packages/b3/34/d8b8232e5e55169933b59aabcef2fedfa4b9d8897361bb80fcbda146505f/coverage-7.15.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:226c66e80ec0598d3b9b4874123df167ccca342aca8714f77cac6829688ee09c", size = 264043, upload-time = "2026-08-06T13:49:04.102Z" }, + { url = "https://files.pythonhosted.org/packages/7e/35/58b009dbf8c471c7224716478b9fed4a7e1af15320e1ed41660978504663/coverage-7.15.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ac41cc14bebda0dbfb0628036b7f75706935c95bcc07fefe9a0f93614aa60a57", size = 266963, upload-time = "2026-08-06T13:49:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/62/aa/57fbda1b42c892968273c56b6ee9dc0f1310850859230a507bc7873b1f65/coverage-7.15.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8af623e5cd92080acddd02b38f2f406a2c3a0893c38950b211890361448fbf26", size = 264569, upload-time = "2026-08-06T13:49:07.706Z" }, + { url = "https://files.pythonhosted.org/packages/98/8a/360e6e7f24d477b7e889703af0afa878d15b6d4d8d2a822b2835c169a879/coverage-7.15.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:07545711d4f0f32852a18f18ad11f76f0109909d09e78b9008b4cfc67e829429", size = 268299, upload-time = "2026-08-06T13:49:09.587Z" }, + { url = "https://files.pythonhosted.org/packages/4e/89/6f701261aee21b6b5fa8f7872229406dc917e125069448292223bf213606/coverage-7.15.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:a0865421cfdc53654b342d515e5a233187590882d20b95752150e53f65460017", size = 263413, upload-time = "2026-08-06T13:49:11.604Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0f/6f04036edc260ed425af83e834f627fad48941ce97b50bfe6edd8b6fa623/coverage-7.15.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:460115e32ee40566476db5048f9bec1e842c127ad8e6f8be745aad3ac9cbc839", size = 265725, upload-time = "2026-08-06T13:49:13.38Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ce/d19b5d4d5c49a7bfb925fd74310fee7d28bc99520ac3367ccbc54e662518/coverage-7.15.4-cp314-cp314t-win32.whl", hash = "sha256:cbde877ef9dd7baf272b9bfef2b8a25edd45d9170fc326951dd20eb480335e85", size = 225079, upload-time = "2026-08-06T13:49:15.265Z" }, + { url = "https://files.pythonhosted.org/packages/26/bb/7aa1b3b173faee0679037ca950bbbe1247273656697994d8d13f80f8d4b4/coverage-7.15.4-cp314-cp314t-win_amd64.whl", hash = "sha256:3da9e92d1c551fd7563833e9ade686efb0c4b7363ab7681a94283958c950bf5e", size = 225911, upload-time = "2026-08-06T13:49:17.279Z" }, + { url = "https://files.pythonhosted.org/packages/81/1c/4ea9e47426d80038d9222db3c4534cb6021a74b237d3ff97ffd33b6600dd/coverage-7.15.4-cp314-cp314t-win_arm64.whl", hash = "sha256:3a54f5a0d85050c73a38f6793090ee83974531e67fe5e57a1da9bee11398aa5e", size = 225219, upload-time = "2026-08-06T13:49:19.293Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c4/dc5d2ac8f9142e7ec7de66e7bf0591db29d78955a040bd915870d9c0e657/coverage-7.15.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:2c9872e4d9dc5d3cf616bf4b382f5a00359305a5be666a3dd0b5cdb4e49597f9", size = 222604, upload-time = "2026-08-06T13:49:21.279Z" }, + { url = "https://files.pythonhosted.org/packages/70/39/33e63df81fe2ee100897451841c821467635923e58e37c6bd4b46dd8106c/coverage-7.15.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:e101dbb4b9b72f0cddd8cdc8c9c5b47f456766f5e0ac82dbfb75e5c55409b78a", size = 222944, upload-time = "2026-08-06T13:49:23.187Z" }, + { url = "https://files.pythonhosted.org/packages/99/1f/ef3ffb5557febc75a0d97aa459d0266d7d741110265121cc6d8539343d44/coverage-7.15.4-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7d1abebdb047729e852b9c77a00497dfbeb11eb3a117e037d7dbc3ac8e5f5c54", size = 254050, upload-time = "2026-08-06T13:49:25.008Z" }, + { url = "https://files.pythonhosted.org/packages/6f/f5/1f0f6f77698c3601ca0ae7431e34b24c62ca2f06fecb23b73ed1f651d2be/coverage-7.15.4-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d28a4a899354d0ea6214cc59b4fa19eefbce1b9ff1688ab579acf49e894bd3fb", size = 256967, upload-time = "2026-08-06T13:49:26.896Z" }, + { url = "https://files.pythonhosted.org/packages/03/7a/2ed9bed79925f4367c83c77f66a89e5ca7229c288d2d19ad5f36d1ca0070/coverage-7.15.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffb3c2aacea411cc7e1d27712490c11108e2de1d39019ae32915493a59a8b9ed", size = 258587, upload-time = "2026-08-06T13:49:28.692Z" }, + { url = "https://files.pythonhosted.org/packages/45/8c/fa34044f71b7cc4ecb6da9c2408770959b0591fa9b5fb6fb6bca38f94298/coverage-7.15.4-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a9447978a92f405d301123cfd39ff49895490efb769a758fe2734c7f631bf8ce", size = 260785, upload-time = "2026-08-06T13:49:30.472Z" }, + { url = "https://files.pythonhosted.org/packages/4f/54/d5727ce36b4524a7394ab9f5f1df378e1f23affcdab01037dc8655185cc7/coverage-7.15.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:050467a7983b8e2fe7dd41a78bb30c3e7f8c0b8cafda14b1c46f8b5e3cf2dd3c", size = 254545, upload-time = "2026-08-06T13:49:32.271Z" }, + { url = "https://files.pythonhosted.org/packages/dc/e6/6e3783e576719590194bdffb6dd6d85490801785b7c331e35a245d8cb8b5/coverage-7.15.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:d003b7a5708ddad5c206c79607a6b92abb6fc13c57d99d8a4468cc03a2941ced", size = 256682, upload-time = "2026-08-06T13:49:34.089Z" }, + { url = "https://files.pythonhosted.org/packages/dc/f2/bacdbde18b69ed2de424fcf64d9fb0a4913753d4f0eca8bae9daad69f4bd/coverage-7.15.4-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c38efe30fd74e5c19e9433f11fb1f5dc9c6522770971b7c6145bbaa413dc8800", size = 254560, upload-time = "2026-08-06T13:49:36.052Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a3/1fb927196e3477c1b48831169ab58ba08f451ba87ae311ff1de68b26a616/coverage-7.15.4-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:1f4f826d70f772ab8b0c052329580d7fe8b8abd191e4ce0c8f81aec6614665d3", size = 258792, upload-time = "2026-08-06T13:49:38.01Z" }, + { url = "https://files.pythonhosted.org/packages/41/58/30d4c149c69053de0edfe325614c1d28d508f62b1783e0e4a234d2e49136/coverage-7.15.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:4a4bf917c9953f57c957be31c1cd504e3bd2f34d4a352b9d391a3025336f6768", size = 253968, upload-time = "2026-08-06T13:49:39.934Z" }, + { url = "https://files.pythonhosted.org/packages/89/e4/77f639371b918aad30dda4051f95404b43578f7f2e2f87ba73e02ed1ff37/coverage-7.15.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:1c9bf40ebef178a45192c75c4964760bb261b0e6ad725da5fc4c93f674f19753", size = 255893, upload-time = "2026-08-06T13:49:41.825Z" }, + { url = "https://files.pythonhosted.org/packages/5c/62/13be29b3ddab35f14c87967a4820a05106d2a3eccb4fa4ff550bf30b75e0/coverage-7.15.4-cp315-cp315-win32.whl", hash = "sha256:43619d04c3671792d2c4706ae8bf45e265dc87bbd4078189ef8b847ea1e74be2", size = 224768, upload-time = "2026-08-06T13:49:44.08Z" }, + { url = "https://files.pythonhosted.org/packages/a1/70/af0c6be0f964af6954f6b74bc109b0dbca02824696d2520fb17fe1ab06e3/coverage-7.15.4-cp315-cp315-win_amd64.whl", hash = "sha256:be619439dbcd31a2eab10b32de9fff62c26ed4bab69dc32b8363fdaaa0882809", size = 225242, upload-time = "2026-08-06T13:49:45.899Z" }, + { url = "https://files.pythonhosted.org/packages/4f/2d/f3bd3aab899fc9efc18b53133ee68f5f98574ef480649b23e12962226387/coverage-7.15.4-cp315-cp315-win_arm64.whl", hash = "sha256:def597967dafc2e8d97c9097ea453c464e0bb8ed38f193a43070f10dc623bb6d", size = 224674, upload-time = "2026-08-06T13:49:48.322Z" }, + { url = "https://files.pythonhosted.org/packages/f5/ca/f69251cd63eabc6438321aea22148754cce758a26bde07dd490e3fe7cfc5/coverage-7.15.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c7dbc748ac8a1e3e59a2b28bea47675e6e778081dbbf081bde0d75def2fcbe1d", size = 223333, upload-time = "2026-08-06T13:49:50.293Z" }, + { url = "https://files.pythonhosted.org/packages/a7/a7/037b53b2885b0d8447064432491a4d5a1014cd9f97a594d53acd0c04541a/coverage-7.15.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:2413074a5ecbb61a01a7888fc72db0ca324d13588c5b38bc0dd8564cdcdfea26", size = 223630, upload-time = "2026-08-06T13:49:52.637Z" }, + { url = "https://files.pythonhosted.org/packages/80/4f/152b8a4779ae90da11bb24f7467df8a59f0be48a5c52acb856325ca48289/coverage-7.15.4-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4e6f6f632b7b2f714bf7a1346e8f97b650ee71f3c298aaad42a2ab60f0f07645", size = 264489, upload-time = "2026-08-06T13:49:54.52Z" }, + { url = "https://files.pythonhosted.org/packages/10/2d/84b4b9e0e1dd6528a51920ff7031f35b789382e467a28ec6a5a578cb8812/coverage-7.15.4-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8df457da2249d3c75ca2e5e835d59c725abfe92d27fdff6cd99eed85b51d5e9a", size = 267567, upload-time = "2026-08-06T13:49:56.721Z" }, + { url = "https://files.pythonhosted.org/packages/53/fc/ba01cc25299f9f8a2c8b02d3b28c53f3543d9fbfbe4e74fa2760b48f163e/coverage-7.15.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:050f66a08805acb5b8a23c6d4a517b1ecf82c08e81ed0e4bd727df065e5c6624", size = 270123, upload-time = "2026-08-06T13:49:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d0/db2647cbf40b14f8c308f94ff7bf89c06d564e59f396906edf50086ec788/coverage-7.15.4-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1587fb771d1ccceef708fdde1e5af8c7ed24b486b61d13a321acb7d8145390aa", size = 271107, upload-time = "2026-08-06T13:50:00.811Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/4d2d17924552c458bb4f77dd631f0e3bc92fbbdf2d2d916cd4b33bbfd5b1/coverage-7.15.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8b4f1c3a69ca580f3fbd6b2046915f536d7f586874f25c1bb23add2a3c88d50f", size = 264955, upload-time = "2026-08-06T13:50:03.023Z" }, + { url = "https://files.pythonhosted.org/packages/ee/de/dc010c7a3691f396d93bbc26bfcafa1c2a3a351cd520470f15faf5795bd5/coverage-7.15.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:ffb58d7eff5b7f6ecc6fa21d6288ab7f968a212cb67d682c269c09b9eba3b66f", size = 267949, upload-time = "2026-08-06T13:50:05.557Z" }, + { url = "https://files.pythonhosted.org/packages/78/ea/dc96a11375e83c045c2f7c61fb6918277cfe9401db7c0f7b1d111a84b2e5/coverage-7.15.4-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:d9df165544774574ee004b953023d1bebada1894a80b1052a43d798b0f676e67", size = 264421, upload-time = "2026-08-06T13:50:07.612Z" }, + { url = "https://files.pythonhosted.org/packages/c8/86/b77131a0f9503ce461cd577076147d7a9040f0c5dda772686f729e2cc9cb/coverage-7.15.4-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:f9de0a24a4079b53e523b5c5e2c5945ec251ab486652659955187cf255a259bc", size = 269121, upload-time = "2026-08-06T13:50:09.58Z" }, + { url = "https://files.pythonhosted.org/packages/24/24/944bc35007862955e7ebf05754e645419dcf5d7526c52735cfa2715e8ebf/coverage-7.15.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:150089274bdc9f940628552cb92844e0223c987f1902ab8efe9f45a2ec758d88", size = 264565, upload-time = "2026-08-06T13:50:11.722Z" }, + { url = "https://files.pythonhosted.org/packages/c7/cc/a3bb9f93e7e740659163e2ea584f8196ddcd2c456a5dbe15f6c50105fec1/coverage-7.15.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:a58a94fed5da6997d258e8f7668c1e195fbd04a691d781b7558f1e468f9e68bc", size = 266522, upload-time = "2026-08-06T13:50:13.786Z" }, + { url = "https://files.pythonhosted.org/packages/49/dd/e0e40f3560d878d888c580698ff5ad1179f5e1c3ac949684ef66b41a3817/coverage-7.15.4-cp315-cp315t-win32.whl", hash = "sha256:ebd5a6d8466ff30836572f3ba2cae8a5e8f85029b1c6d5e2ed338dc472a5166a", size = 225068, upload-time = "2026-08-06T13:50:15.825Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7e/37732ea80eebc30e976e4cdab15c190bc42d96959a42e38ddf6f8c60468f/coverage-7.15.4-cp315-cp315t-win_amd64.whl", hash = "sha256:288bde2a2d7ab6b6c2d7252fcde8b524387f2d970bdba9658fc6f8bbcaef0f9b", size = 225895, upload-time = "2026-08-06T13:50:17.928Z" }, + { url = "https://files.pythonhosted.org/packages/c6/08/1e00f7923eaaba45fb3d51dd794125fc766304b1df264f3a9c6557bfb30e/coverage-7.15.4-cp315-cp315t-win_arm64.whl", hash = "sha256:68be5e1de60ff13c9095bbec0e5a7fa45b33b101752215b91345ea1f61c4a278", size = 225213, upload-time = "2026-08-06T13:50:19.981Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d9/e70c286c979378f061d8266e279b686ab0b0b688e1fe0af864684f23a77d/coverage-7.15.4-py3-none-any.whl", hash = "sha256:964730a1e9de9c0cf11be6a1a3c79ce419c34882842abd256086ba4698705e84", size = 214332, upload-time = "2026-08-06T13:50:22.192Z" }, +] + +[[package]] +name = "cryptography" +version = "50.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/5c/59086b4aac5e879d38ddbcf74e4be7ade89cebc3eb199a55da998c3bb46a/cryptography-50.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:031e2d5dd4bb9caa3ca9c82e5a197fd8ae680232cee62603d1a813f3f07e3d03", size = 4001252, upload-time = "2026-07-31T14:23:33.331Z" }, + { url = "https://files.pythonhosted.org/packages/57/ef/8f2df13c7216bcad3e1c74e07f6e193d93e998e114f524a53877c9af27ad/cryptography-50.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645", size = 4719554, upload-time = "2026-07-31T14:23:35.611Z" }, + { url = "https://files.pythonhosted.org/packages/d9/41/029086c34d91052fc3b88bcc8056f709a7c915c7a23b235a54eb800b1c97/cryptography-50.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7", size = 4702130, upload-time = "2026-07-31T14:23:37.635Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ff/b6ce0954962e7f7b969f850a883744197bb3910bdfd7b6da162eab7d9f68/cryptography-50.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3", size = 4725244, upload-time = "2026-07-31T14:23:39.471Z" }, + { url = "https://files.pythonhosted.org/packages/06/1e/63a1027cb7fec360a182208e1b7767d5aa1fe57be3d6aa856e69a321edc0/cryptography-50.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:8d89f3976b10b4ce31118de72329025f70d2c6ead14a8217c5514dd2c6d5a78f", size = 5342265, upload-time = "2026-07-31T14:23:41.286Z" }, + { url = "https://files.pythonhosted.org/packages/6b/72/a1116d683a6d7ece94590013882515de087edf9ef0e6292aae615a44df73/cryptography-50.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae", size = 4734609, upload-time = "2026-07-31T14:23:43.139Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/36a9c479bbe49acea2636c7fd3360d20f7b7e079c300352011c44850b181/cryptography-50.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a", size = 4356517, upload-time = "2026-07-31T14:23:44.939Z" }, + { url = "https://files.pythonhosted.org/packages/32/98/8a151d64367204cbc63ec65d37502f1d9c53cf4bfc6ec3c532614dbec60d/cryptography-50.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987", size = 4724529, upload-time = "2026-07-31T14:23:46.93Z" }, + { url = "https://files.pythonhosted.org/packages/22/f6/ec13b470172126464a86bf54d2294a46d29837fc51ba3e45d4047946fb5e/cryptography-50.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f89831ef99dd7dd169ab06d63a831adb9e20a87aac6d380266bbda5823349169", size = 5299852, upload-time = "2026-07-31T14:23:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/da/3a/f05e32c99d440c9bb891ea0e36c9091891e36be5a9a87ab2ee6ea20729f6/cryptography-50.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f", size = 4734462, upload-time = "2026-07-31T14:23:50.861Z" }, + { url = "https://files.pythonhosted.org/packages/ca/dc/bd72b26be8953f80625f63151efd38eee71c76ca6cf591c08ff34615a79e/cryptography-50.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105", size = 4852708, upload-time = "2026-07-31T14:23:52.715Z" }, + { url = "https://files.pythonhosted.org/packages/27/20/c930314a2ab476d15dec966ec87e2e9637bb02b06106b12c0396c57bb603/cryptography-50.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef", size = 5004179, upload-time = "2026-07-31T14:23:54.887Z" }, + { url = "https://files.pythonhosted.org/packages/32/2e/c9db68a0c4bfa28e310707527c0ee3a2bd254104d2e02e68f368e197aa4c/cryptography-50.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30", size = 3840395, upload-time = "2026-07-31T14:23:56.677Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fb/951032a3bf22a5697c83183fb6294a4843772947a70e616c57b3ff5f522e/cryptography-50.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:49e7d93abdbd2990caced757e5fade25302f719c3c8fb6e6fff2dde98999fc41", size = 3989258, upload-time = "2026-07-31T14:23:58.881Z" }, + { url = "https://files.pythonhosted.org/packages/d4/67/91eb047e69c5e845f2f14b8a2e4a1aab0f283cb885531e9e22c8adb176bc/cryptography-50.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:19736989797678c6af1e55cd49055cdbcb55d8f6b5583ac5335f933aba9101dc", size = 4700648, upload-time = "2026-07-31T14:24:00.702Z" }, + { url = "https://files.pythonhosted.org/packages/30/82/85f0f7425c856b9f96459411eb12e74ef72df9caf6f8f15bf23a33ff131f/cryptography-50.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80b63928fa35083b33966ce1efb70e5b9607181e49dcd1c22c8c005e319f667f", size = 4682442, upload-time = "2026-07-31T14:24:02.538Z" }, + { url = "https://files.pythonhosted.org/packages/1a/28/b555a365adff1cca2fbe7b9e487d68a40de6bc67ff2cb587473eb43de0e7/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:d58c3db7cd6eed54e6c06744db55456b65ebd7492ddeae9c1e93cfca7aa857d3", size = 4707596, upload-time = "2026-07-31T14:24:04.394Z" }, + { url = "https://files.pythonhosted.org/packages/72/d8/f52538140cc719df62a01cf87d1c7142318d235817109d6f4054d7c352d6/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:df2a58a472f332225671c35b0a830208b86d004f82baa8530fa3782c85646533", size = 5314552, upload-time = "2026-07-31T14:24:06.31Z" }, + { url = "https://files.pythonhosted.org/packages/38/14/6120e5bd7c5aa022ad15424ba4d5c5269d0d9448ed4d55e492ea91e3c1c4/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:11b74db56cdbe3cdee6e3f6982ecb70334fa10dce99ed58bf7894aaaa3b2a037", size = 4717113, upload-time = "2026-07-31T14:24:08.349Z" }, + { url = "https://files.pythonhosted.org/packages/fa/71/190bf38c3ee2e0f8efc9860ae100c9df4169742eef274b91e7aa1cb133b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f59e38625469987d7ef6d495323c55e7db6c212eaf6112267e0d3b565a2e9c9f", size = 4338580, upload-time = "2026-07-31T14:24:10.227Z" }, + { url = "https://files.pythonhosted.org/packages/3a/63/504ccfbbe61fd8aa983f7f146399cdf034c72c2fc55f5b2dfdcdcdb20c99/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ecfed7367f965a0328cfbdd70da860f15441f002f613185668c6e6ebf5a0ac11", size = 4707038, upload-time = "2026-07-31T14:24:12.169Z" }, + { url = "https://files.pythonhosted.org/packages/01/77/2cf79bbfc4d12ca106437a6e170d6aaa01a373e93093118aaaef0e801bd4/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:9aa87839c383bdbab6ef865787a1fb877af8dd03464c4400322726feaaadfc6d", size = 5273110, upload-time = "2026-07-31T14:24:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/e5/45/8aae2972c520145377ea3559a605a899bebe227bf070b33cdb445929a9b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:6ba6a53445bd3cfa809ef3ef5f1589aa6ba08784a1d962bf47d0940e871dab1c", size = 4716439, upload-time = "2026-07-31T14:24:16.415Z" }, + { url = "https://files.pythonhosted.org/packages/7b/20/4fe50b619a48c2525cc46e2dbc1ac490708d704be5d467bdaac6dc955682/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3f5735ffe4996d28b809371756219f5354864902a3b9e7c0b9ee87041209fc9c", size = 4837383, upload-time = "2026-07-31T14:24:18.553Z" }, + { url = "https://files.pythonhosted.org/packages/92/91/3a31366e183343d3703f8995c095f5734676bd6938118047e50fcf279eb4/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1b4a266766514614f8aa60416e71f2fc6e575d36e7bdc90f644fadb2f4b75b95", size = 4985772, upload-time = "2026-07-31T14:24:20.385Z" }, + { url = "https://files.pythonhosted.org/packages/74/9a/02ffe35b2853d121689871eb5dce862092562b3a1ed5cc98f1aaed441506/cryptography-50.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:12b9c6996425c76ea6c457ace4f3073e715b8c545add07cd1a8f3a4f90691269", size = 3816291, upload-time = "2026-07-31T14:24:22.125Z" }, + { url = "https://files.pythonhosted.org/packages/03/37/73d005be173aff344af30e9fd2a576575cb2391a7101d9cd3842e1fa8cce/cryptography-50.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ccdc4a71a4dabae05de219404f9f4abc38e3b58422177ff93d0da05967dafa07", size = 4036009, upload-time = "2026-07-31T14:24:24.122Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c6/7a6202a534e32103a285b7834a120869557fe198d51d7cfe59754c8bda9c/cryptography-50.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3", size = 4745252, upload-time = "2026-07-31T14:24:26.118Z" }, + { url = "https://files.pythonhosted.org/packages/85/4f/0fa8c2f4428198f15d9ff8d63400e27afbf94ce833f6108da1eb3753f945/cryptography-50.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f", size = 4728939, upload-time = "2026-07-31T14:24:27.994Z" }, + { url = "https://files.pythonhosted.org/packages/d1/63/54dd723490ba2dc09b299682c10b38db38f159728bcaae8c591b8af2f22d/cryptography-50.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5", size = 4748483, upload-time = "2026-07-31T14:24:30.254Z" }, + { url = "https://files.pythonhosted.org/packages/1d/dd/7c77d26285cc7f6991efce64a0f5b4f9383bfa5dd8c5033003eaf7db4cdb/cryptography-50.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d764dcf130c428ef66786f866dd750f53182bc608813489915e9fc106bb0c82f", size = 5367599, upload-time = "2026-07-31T14:24:32.457Z" }, + { url = "https://files.pythonhosted.org/packages/46/c9/f60aed34c013f317f92817b6c171c2d22a78270fa41109bd4b08af26b194/cryptography-50.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025", size = 4762647, upload-time = "2026-07-31T14:24:34.599Z" }, + { url = "https://files.pythonhosted.org/packages/be/f3/f9a0173b139372c3a48ed98154b45cc6b9de17c789d5ab552e621c293609/cryptography-50.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a", size = 4385197, upload-time = "2026-07-31T14:24:36.647Z" }, + { url = "https://files.pythonhosted.org/packages/d8/36/83bb81f6e569bc38e1e4a7bc80f29b46bb9601920bc455fc8e888f5d5742/cryptography-50.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b", size = 4748095, upload-time = "2026-07-31T14:24:39.493Z" }, + { url = "https://files.pythonhosted.org/packages/6b/16/d3008eff98c764979865834c3d386d4fd041b5f52e7f34fc29ac1a5eb515/cryptography-50.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6e7d61120573a7f2cd94cc095f9e81f6967c61ccdf194285aa143ecec8e0b708", size = 5325948, upload-time = "2026-07-31T14:24:41.556Z" }, + { url = "https://files.pythonhosted.org/packages/9c/f8/d97f9603efda3888187bfdb893f26c41be4735c10631d05d284ee6b047c4/cryptography-50.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47", size = 4762400, upload-time = "2026-07-31T14:24:43.636Z" }, + { url = "https://files.pythonhosted.org/packages/64/a2/4615c8f7d81a00b1d6e6afe19f694e1543582349fb5f4076f6cb5dc36485/cryptography-50.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9", size = 4878208, upload-time = "2026-07-31T14:24:45.522Z" }, + { url = "https://files.pythonhosted.org/packages/d2/1a/efcfb02f91407149a0dacffffab791f7e19bf6385f63b3666dc8b5e5c9c8/cryptography-50.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7", size = 5037050, upload-time = "2026-07-31T14:24:47.697Z" }, + { url = "https://files.pythonhosted.org/packages/57/30/4a22984d4f1bdfb8c054f07a92bc176b97a3134cc1d6c4b3bffb1f3688b4/cryptography-50.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba", size = 3874135, upload-time = "2026-07-31T14:24:50.085Z" }, +] + +[[package]] +name = "distlib" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/02/bd72be9134d25ed783ecbbc38a539ffaefbf90c78418c7fb7229600dbac7/distlib-0.4.3.tar.gz", hash = "sha256:f152097224a0ae24be5a0f6bae1b9359af82133bce63f98a95f86cae1aede9ed", size = 615141, upload-time = "2026-06-12T08:04:52.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl", hash = "sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b", size = 470628, upload-time = "2026-06-12T08:04:50.506Z" }, +] + +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + +[[package]] +name = "fastexcel" +version = "0.20.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/96/24314765a3204081a55ecc96e83dfdbd2ef82aa80a521ac3005b2c91e941/fastexcel-0.20.2.tar.gz", hash = "sha256:a4be10eb0f4dd819d2b36950b3f26bf87260b53e54150ac02b78e524c7063902", size = 60862, upload-time = "2026-05-04T12:28:36.01Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/29/acc5b2eb95fb4eaeb3486b6f034f2d9754dcc0cc0b5dbc79b2fa9a31f66b/fastexcel-0.20.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2a240b9f4b6af42cafc247c1b8c956cf32f5393a3956ad0ed49e07b2b5bf0e09", size = 3432179, upload-time = "2026-05-04T12:28:24.611Z" }, + { url = "https://files.pythonhosted.org/packages/3f/2b/f78a2df2d93ec4a23d7fc94ed967bf004d5115e1f4d2da389bb888d2cc32/fastexcel-0.20.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:6c7e58cf83c873743c18006e3866d5c2b69c34df7a4a253a72e81169853ddf0e", size = 3260476, upload-time = "2026-05-04T12:28:26.191Z" }, + { url = "https://files.pythonhosted.org/packages/36/9c/a8f45e5c195a70a408893b348595bae4c113091dddba7d1b029e9c980b8e/fastexcel-0.20.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdab3cc439c18c4ee14c4ec2b321e1d927cae84f557525f390a3c28517f6e721", size = 3625132, upload-time = "2026-05-04T12:28:12.173Z" }, + { url = "https://files.pythonhosted.org/packages/b4/01/da0ae9b3ee86daca5ac9f0a7b05477cf0c1624321d95f1e52492a772dba5/fastexcel-0.20.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15d41bcc4433bb3ab778426ecaeaf3a5f38d949555c631a83e958c57bb0a116e", size = 3789901, upload-time = "2026-05-04T12:28:13.884Z" }, + { url = "https://files.pythonhosted.org/packages/c5/15/80c614c8b07c631c7c2acd015e7f2e1cd683cd04279e5615b39c36a0e43b/fastexcel-0.20.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:441f9b7e97b4fd1c9487e5c0a0d9cad7aa4097a39e65fcd494551f5ec3e5d02f", size = 3792388, upload-time = "2026-05-04T12:28:15.448Z" }, + { url = "https://files.pythonhosted.org/packages/c0/80/4548305ce4d70a0ecf38205988e5caeb9347dae7e5d06bcd2b939586fc52/fastexcel-0.20.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b15f21baef3e4914071e34f176d8692dd79e3464c10f8373047aab1384cfaae", size = 3960316, upload-time = "2026-05-04T12:28:16.821Z" }, + { url = "https://files.pythonhosted.org/packages/22/b3/7de6b49406fda63c9fee817d5fc70fe3f52c24431605f1de2f7b0f3a0938/fastexcel-0.20.2-cp310-abi3-win_amd64.whl", hash = "sha256:ce28748390ae6a7d286ba29d8ee2badcfb6478d32ed700db6c2030de32a16e7b", size = 3275637, upload-time = "2026-05-04T12:28:30.45Z" }, + { url = "https://files.pythonhosted.org/packages/d8/3b/c251f38596d630e87280c191f53803e7409e7723e95e53a477f6760c7298/fastexcel-0.20.2-cp310-abi3-win_arm64.whl", hash = "sha256:c3e58f5053bc05b0ee8fa735056cea7af6e1c41c39fc4b2b06cc86c1025d140b", size = 2990952, upload-time = "2026-05-04T12:28:32.434Z" }, + { url = "https://files.pythonhosted.org/packages/37/d3/fcc1ba752415327f0fd5a1b52060e66f1c0bf8b7b90bbba781b91750ce7e/fastexcel-0.20.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e5b677d669cc7c099998ef27d9442b78d7190916d3b40418bda619cdc4ebae2c", size = 3427401, upload-time = "2026-05-04T12:28:27.741Z" }, + { url = "https://files.pythonhosted.org/packages/80/bc/e492c343c096e93e269c157fc11def297430047f1ee09db4f8a39bcdc1a0/fastexcel-0.20.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9692ac135ede4d6fc45b25cc22f1c6bd1eb8750bda14097b7f319450c33d3124", size = 3253915, upload-time = "2026-05-04T12:28:29.053Z" }, + { url = "https://files.pythonhosted.org/packages/dd/ef/bd7b98149b50e01fdaa3b146b025ce50b8e983f35522f8e69bd94d2711a4/fastexcel-0.20.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:675d7eab2110bd8d5520b98d5eb3f3e15173b69a5f64b43412f917e9717e9aa0", size = 3621364, upload-time = "2026-05-04T12:28:18.457Z" }, + { url = "https://files.pythonhosted.org/packages/4c/e0/f67911db6515df326900e8ac1ae17cc08233ca5fb8f71c633f57a50cf786/fastexcel-0.20.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db049d51e5323459f534ce9a65869f997bebd4dde75ef1d3b3af53dd64b8b9ca", size = 3784924, upload-time = "2026-05-04T12:28:19.955Z" }, + { url = "https://files.pythonhosted.org/packages/34/82/fabe2d1de28e89a90bb2c884c37d862b5d85063fd41fffcb56b2980d0b51/fastexcel-0.20.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e5b23f69898f6a32c2f279d8f72eb8ac93c336bf7f453a9b8930cab9dc494520", size = 3791264, upload-time = "2026-05-04T12:28:21.612Z" }, + { url = "https://files.pythonhosted.org/packages/f7/34/b3ed819dd6d35d71c6640ff857c6362d42f19e8b15354e64b06f355df3a4/fastexcel-0.20.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c3058d034e75967b6f01e0c82767adeb40cbf4533b6bc54e3b93f77f5f80dec4", size = 3953234, upload-time = "2026-05-04T12:28:23.224Z" }, + { url = "https://files.pythonhosted.org/packages/3a/51/1d9a72473cc213d5647d8cf7c2393ef995b3453ac5e4b544ba652d1428dd/fastexcel-0.20.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d71e2a1fce006cb8bc1ae06da6ff5e5fe83e6496be5aef01b623b15aa0d0f82d", size = 3271344, upload-time = "2026-05-04T12:28:34.443Z" }, +] + +[[package]] +name = "filelock" +version = "3.32.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/57/3ba6e6cb097f85b855b00163d169f35365f44277df044dcf96d55b8f62a3/filelock-3.32.2.tar.gz", hash = "sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8", size = 217172, upload-time = "2026-07-29T22:46:04.895Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl", hash = "sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82", size = 98830, upload-time = "2026-07-29T22:46:03.52Z" }, +] + +[[package]] +name = "google-api-core" +version = "2.34.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos" }, + { name = "proto-plus" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7b/7c/9be3903e3d45415e8ca493c75f8990a0f6f579d168015d44c379350d0ab0/google_api_core-2.34.0.tar.gz", hash = "sha256:98a779fe72de956eb1c9c2f47ff4c4432a668ece1a002ec38bed07ec2698ae59", size = 187953, upload-time = "2026-08-06T06:23:58.128Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/c1/a8a92ae1bc4b1a8f804c776d7d3f0c771b78a62c3ad4df1be41b3fd8c767/google_api_core-2.34.0-py3-none-any.whl", hash = "sha256:cdf9c67e7ca2402d86ccbfde5f2503fc83e3cc3f58cc78456ae96cad24a6d2de", size = 180545, upload-time = "2026-08-06T06:22:47.502Z" }, +] + +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, + { name = "grpcio-status" }, +] + +[[package]] +name = "google-auth" +version = "2.56.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/4c/fa42116a48bab3f7a143cf5042ecff7df9c8b73f8a376203cd534d1dc966/google_auth-2.56.3.tar.gz", hash = "sha256:40e229fc901f0a305b553050e5fce562d509bee0435be053abfa91582b51b90c", size = 367110, upload-time = "2026-08-06T06:24:01.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/b3/6117b2f24065cd7e2c4f140e9a193e215f089ca8ba314cf91eb9d0b7fe0a/google_auth-2.56.3-py3-none-any.whl", hash = "sha256:8ec438808f813ad034535000261eed1067475d229d05bbf4216e78c3f2362e53", size = 259116, upload-time = "2026-08-06T06:22:51.788Z" }, +] + +[package.optional-dependencies] +pyopenssl = [ + { name = "cryptography" }, +] + +[[package]] +name = "google-cloud-bigquery" +version = "3.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth", extra = ["pyopenssl"] }, + { name = "google-cloud-core" }, + { name = "google-resumable-media" }, + { name = "packaging" }, + { name = "python-dateutil" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/a2/5ee4a9eae62dbd61e59504458d5cf870809f790ed136967d94c294b9a3ba/google_cloud_bigquery-3.43.0.tar.gz", hash = "sha256:e3dc25ab9ac8b2b089408493177d4d4508b098c80c3931786fbc20b075298fe6", size = 519709, upload-time = "2026-08-06T06:24:11.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/c0/f491506bdff4d73750f74cf18fea7d6ca409ada2f81c4c4d10a32edf5688/google_cloud_bigquery-3.43.0-py3-none-any.whl", hash = "sha256:a39217f14f215472ce9da816f20ebaf77fdb1db7ccdc8360772d8bf6bafb55c2", size = 265159, upload-time = "2026-08-06T06:23:04.281Z" }, +] + +[[package]] +name = "google-cloud-core" +version = "2.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/c6/9d7d9ed6703eb35306ca7bf381fb66ba8d978c61a1b550a2e1730a4c4ce8/google_cloud_core-2.6.1.tar.gz", hash = "sha256:1e044b131f2ae097b92312fa195164b0aeb6dc6a88e00231e1210516314c420c", size = 36017, upload-time = "2026-08-06T06:24:18.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/4f/37960d5255988b218c40c9b5a2b731013684294a50735d1dd1ab4894e531/google_cloud_core-2.6.1-py3-none-any.whl", hash = "sha256:2682a8a4474a32f56292fb4bca7fa7e4fb0b4af958f6abfe4bca8d195747fd45", size = 29393, upload-time = "2026-08-06T06:23:11.405Z" }, +] + +[[package]] +name = "google-cloud-storage" +version = "3.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-cloud-core" }, + { name = "google-crc32c" }, + { name = "google-resumable-media" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/7e/73bb7512df1d1aad6ce3f9aed847cd40e0cd400ba4a85d86ab8eb412e9cc/google_cloud_storage-3.13.1.tar.gz", hash = "sha256:a80bf8cac2794808aa61c50c5f769ecbbe2d10331bacd0d69d30e59b14b346b2", size = 17341051, upload-time = "2026-08-06T06:24:42.229Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/6f/d69f0e185e08ddb58c323a0a935af2b492907b5de362bc08933b0a3b5644/google_cloud_storage-3.13.1-py3-none-any.whl", hash = "sha256:98208de6c21e85cecd3eb44551894efff33d98365500e178867d4305854a770a", size = 341486, upload-time = "2026-08-06T06:23:36.548Z" }, +] + +[[package]] +name = "google-crc32c" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/88/8ecf3c2b864a490b9e7010c84fd203ec8cf3b280651106a3a74dd1b0ca72/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697", size = 31301, upload-time = "2025-12-16T00:24:48.527Z" }, + { url = "https://files.pythonhosted.org/packages/36/c6/f7ff6c11f5ca215d9f43d3629163727a272eabc356e5c9b2853df2bfe965/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651", size = 30868, upload-time = "2025-12-16T00:48:12.163Z" }, + { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, + { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, + { url = "https://files.pythonhosted.org/packages/08/a5/7b059810934a09fb3ccb657e0843813c1fee1183d3bc2c8041800374aa2c/google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2", size = 34878, upload-time = "2025-12-16T00:35:23.142Z" }, +] + +[[package]] +name = "google-resumable-media" +version = "2.10.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-crc32c" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/f5/f35505e6091614e285056a495488cb0a9c1a9dcc88a4a3c91bbc5fd4835b/google_resumable_media-2.10.1.tar.gz", hash = "sha256:224975032ddb73f7ed9e2f0f4cc08ed1b06874c52d48cc8533e3eb72980b21a0", size = 2164548, upload-time = "2026-08-06T06:24:50.489Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/ba/77ef49baf338c03a11deadac984e3161b3f2b4fa4bb5aab160e7ca0fd522/google_resumable_media-2.10.1-py3-none-any.whl", hash = "sha256:4e2cbc704207ddc09f23b1f18e8ef4a4ccbfe0f1768b370e5c969704adbd0a1c", size = 81533, upload-time = "2026-08-06T06:23:45.464Z" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.75.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/73/74bcab964c9a7a61f2bb71e8179b0f13e6fa98f7ce00fd168aab291e4a2e/googleapis_common_protos-1.75.1.tar.gz", hash = "sha256:d3042c6c5a2d4e67113104d6b6818b59b6bd92a197f2a91508e801fe815cf071", size = 150967, upload-time = "2026-08-06T06:24:51.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/51/186c02b8549b69ccda44429cf6ff5081e4b61a602ddfe6a8020d1be31d1b/googleapis_common_protos-1.75.1-py3-none-any.whl", hash = "sha256:28a1934bcd33b9c9da66ac301a0a4227e3367f095a17d0375cb98f0a09d93b79", size = 300626, upload-time = "2026-08-06T06:23:46.696Z" }, +] + +[[package]] +name = "grpcio" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/98/304898ac4e04e2d5e4e4c2eadc178b1f2a16d5f4bc2f91306c87d64680b9/grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24", size = 13428824, upload-time = "2026-07-23T15:20:37.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/60/f2cca8147ea213d3e43ae9158d03ad04e020fdf32ff027253e1fe93f921d/grpcio-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3f351629f6ae16ecc0ec3553e586a6763ffd9f6114044286d0cbec3e09241bfa", size = 6305607, upload-time = "2026-07-23T15:20:15.353Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ab/d3874931d123a95e83a3ebf8aa04537988fb62425cedb8bf3cefc5ad41b2/grpcio-1.83.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d05ff664100d429335b93c91b8b34ddf9e94a112205e7fa06dede309e44a4e4c", size = 12166617, upload-time = "2026-07-23T15:20:17.435Z" }, + { url = "https://files.pythonhosted.org/packages/92/ff/6f18f9426b69306f4e00a9add3b0ee2748da8aad53836ef80cab0d62d04f/grpcio-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7936f2a56cf04f6514705c0fedf400971de01b6aa1719327e4718f410a765e2b", size = 6880213, upload-time = "2026-07-23T15:20:19.98Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/706d1147c6b93b98f179240c13991fbcc56880eba0c868abb1ad40d8a0a6/grpcio-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b0a0be840e51b6b7ee9df9269770faf77bdf4b771053c257c21d12bad607714c", size = 7618335, upload-time = "2026-07-23T15:20:22.161Z" }, + { url = "https://files.pythonhosted.org/packages/74/04/1a8443c889115ec9e213a213e86bc93a71ee9088027e5befa09aaa0edd9d/grpcio-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:009667eaf3dcd5224c713589cdc98e7ca4ed0ff0b61132c6b276e930eb83a2df", size = 7043416, upload-time = "2026-07-23T15:20:24.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/c6/94e0fee5b12bc1da1370185b680988db6f739d19b42d9959db01a7ea50bf/grpcio-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb669918fd88936b15599caff4160a77ab74bdeb25f2231f6e45b61282d6107b", size = 7583253, upload-time = "2026-07-23T15:20:26.313Z" }, + { url = "https://files.pythonhosted.org/packages/a0/97/de1ccb671fb85575bc5192faedf9ecdbdf5b390d2e6584dcf552bcbd370e/grpcio-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c19b454d3d3f28db81f2c7c4dbaee96e7f6fd149721733ffe79d6bc530f17404", size = 8605102, upload-time = "2026-07-23T15:20:28.437Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/0e0ec749a7034ffcbaa050e39779872950ead90c22e7e0116be3f28b2b46/grpcio-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:61007cd08640abc5c54547ee32505474c482cd733a53cb87551ea81faa6350af", size = 7979826, upload-time = "2026-07-23T15:20:31.182Z" }, + { url = "https://files.pythonhosted.org/packages/83/fa/c3fda157287f64bc65acee6c5aa90c41acf9e0d3a8e69a265eecff6d00a1/grpcio-1.83.0-cp314-cp314-win32.whl", hash = "sha256:32e11c37f5285b0c6fa3042c05fe06903696689749833fc64e67dec71b9bbe33", size = 4471765, upload-time = "2026-07-23T15:20:33.195Z" }, + { url = "https://files.pythonhosted.org/packages/a1/00/b1b26431c9d54eee11724fd6e5585473a2ed47fbc1fb95e5204906a642ce/grpcio-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:2bb48cb5e6dd005ca12b89ce4b6ac0b48ff3112c747542ee7986ef611a8ca6d9", size = 5298932, upload-time = "2026-07-23T15:20:35.48Z" }, +] + +[[package]] +name = "grpcio-status" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/fd/848dd7e009de85f8ca59999d1cc618ff8ebf7ea5636d083a47455d212d24/grpcio_status-1.83.0.tar.gz", hash = "sha256:837219c6de9afdccb6f6f72b34bc71e151a2011ef04040e3faaca746a57e54ae", size = 13965, upload-time = "2026-07-23T15:24:26.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/00/73204406228cf989bea6b0fd9fe4702fab49a8a152a0c6f90856dadb6ac7/grpcio_status-1.83.0-py3-none-any.whl", hash = "sha256:f6a838a7c5fb84ae98833ec0ef81ed438c26e11e54b2ddb8e92ad328c861de69", size = 14636, upload-time = "2026-07-23T15:23:49.044Z" }, +] + +[[package]] +name = "identify" +version = "2.6.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/63/51723b5f116cc04b061cb6f5a561790abf249d25931d515cd375e063e0f4/identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842", size = 99567, upload-time = "2026-04-17T18:39:50.265Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a", size = 99397, upload-time = "2026-04-17T18:39:49.221Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "loguru" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "win32-setctime", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pandera" +version = "0.32.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "pydantic" }, + { name = "typeguard" }, + { name = "typing-extensions" }, + { name = "typing-inspect" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/74/7ef34611e49990b7e630dcf02c2225d76c2e328ec2804186c2b9c52e916b/pandera-0.32.1.tar.gz", hash = "sha256:72ecd74226847abf0f0437c05f7f10cc8368e306d88a2acc78fa93762c5a0a02", size = 875349, upload-time = "2026-06-29T16:01:48.72Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/d0/411c82285a7586e97326020f6b5ecbc2f2ffcbef72aa108c897de1b0a540/pandera-0.32.1-py3-none-any.whl", hash = "sha256:1a17a3ffa906174d19207715f4f082ec3db3709647927ad8c095c147d74d8454", size = 447840, upload-time = "2026-06-29T16:01:46.877Z" }, +] + +[package.optional-dependencies] +polars = [ + { name = "polars" }, +] + +[[package]] +name = "platformdirs" +version = "4.11.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/36/0a/062135c9a98dac804265073cc3afdbec5ae1aa37980bb354f461bafe81b4/platformdirs-4.11.1.tar.gz", hash = "sha256:bb1af68078f25e2f3e111e2d43b8d536df41b73c8a684b40bb018223b66fae27", size = 32396, upload-time = "2026-08-07T23:06:48.516Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/85/9b31b44296cfa3bb56cddb35e6a0f6578bab0b490c0806c0245e32c6110c/platformdirs-4.11.1-py3-none-any.whl", hash = "sha256:2efd27d363e8dd2e661639ffb398865a5e0a46442a11d266bf375a0e0c10e386", size = 23261, upload-time = "2026-08-07T23:06:47.219Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "polars" +version = "1.43.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "polars-runtime-32" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/89/13/3873f213304bcbaaf39e63c8b905ceb460a0524448d57f86a829f6d4d0fd/polars-1.43.2.tar.gz", hash = "sha256:c699671b99eb71ff53334d237917aaa3db5ad4dda480abcb6c80e0eaee7b677b", size = 750312, upload-time = "2026-08-01T06:28:30.872Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/fe/0888040a24e4504098b85d8ad486b14cb01cf6b030bbe479dfc2dcffc2ac/polars-1.43.2-py3-none-any.whl", hash = "sha256:22aa0cb92a1ee2d60d6a15a638b2e8e0dd99aea21ac0cd8fb29da8e382e075a9", size = 847150, upload-time = "2026-08-01T06:27:15.543Z" }, +] + +[[package]] +name = "polars-runtime-32" +version = "1.43.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/06/11b578eeef05f867e3ee31b2a2fdd8e7684c2aa47822c49935d1be789c38/polars_runtime_32-1.43.2.tar.gz", hash = "sha256:d7b7c486bccee75a6af0158b87077da3d054657e3c60036b28644f4e1c7fdbf7", size = 3095669, upload-time = "2026-08-01T06:28:32.315Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/fc/12e6d4ca34d820297651134cfa35f86c33e898539fc6629cbb35d0089697/polars_runtime_32-1.43.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:91abf205d4ec93f92ba95386b7f8776559ae3dfce425ed2e527efa75d117d04a", size = 53088908, upload-time = "2026-08-01T06:27:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/32/81/833b0853551deb810854f96b43dea342b6e6c9b0ea1afcccf774157d519d/polars_runtime_32-1.43.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:2cc3ff96fd44789b02eb5c15b98dfcb000101636b177d3034fae2feec19b118f", size = 47540529, upload-time = "2026-08-01T06:27:21.391Z" }, + { url = "https://files.pythonhosted.org/packages/83/55/7b2a75af14c9294d97f3bec132dd3018ddcd988bef32b5d28322150b8c11/polars_runtime_32-1.43.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10ed36e615ab362feb7406e6d084e124b445ad284caa73bd93ae7e65745ed894", size = 51366340, upload-time = "2026-08-01T06:27:24.776Z" }, + { url = "https://files.pythonhosted.org/packages/62/60/64deacb3abc70c52e2d88a808a052d1621c86a48fe9194f2c065579ab1cd/polars_runtime_32-1.43.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d5a7ae004a2723ebf4427f6d6a639f30f86af4cf077075f6b35d04711154fc3", size = 57304599, upload-time = "2026-08-01T06:27:27.875Z" }, + { url = "https://files.pythonhosted.org/packages/52/95/d6e3a236d7630e17c40d0ddee839bf2be9acf548fdc0e5ad65ed9ff0cac6/polars_runtime_32-1.43.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:09339eacc6d392206e78aabbaaa37d7276eb969b798f46cb1f367fd718798c60", size = 51520580, upload-time = "2026-08-01T06:27:30.913Z" }, + { url = "https://files.pythonhosted.org/packages/b6/5a/2deb8eac70e9a2ac26d88a66ae7cf52612865026f4f4a5e7ab11ad9d52bf/polars_runtime_32-1.43.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:452b400e59e7f56e4c6437f435e796903272a9388feee12de2bea049ae87025e", size = 55204471, upload-time = "2026-08-01T06:27:33.886Z" }, + { url = "https://files.pythonhosted.org/packages/29/9e/647401ae8a607bc0cc40ed7b8592d5b1be90ded0dc9b9d6d3aeb03f9524b/polars_runtime_32-1.43.2-cp310-abi3-win_amd64.whl", hash = "sha256:00e33c28e321410c8d66e814a90043101e3bdd9ed2c6dabda07565aa8adbbdf1", size = 52572176, upload-time = "2026-08-01T06:27:37.048Z" }, + { url = "https://files.pythonhosted.org/packages/96/8d/60a50c3f36c85218a7ffcb48c6fe2ce1f7bec799152d68b8658ebed2179c/polars_runtime_32-1.43.2-cp310-abi3-win_arm64.whl", hash = "sha256:350a4868cae85bf8b3f81b33ba47927c15256bd9264dfc8c0753f1b927eac9d3", size = 46582513, upload-time = "2026-08-01T06:27:40.025Z" }, +] + +[[package]] +name = "pre-commit" +version = "4.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/3a/ddb78f32a0814e66b18a099377a106a2dcdce92d86a034d69d65df9b256e/pre_commit-4.6.1.tar.gz", hash = "sha256:03e809865c7d178b9979d06c761fcbfe6808fdaded8581a745bb110e52050421", size = 198646, upload-time = "2026-07-21T20:56:58.225Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/49/bc925106abcdac498074f2cbe6137e94e09f418dd2b7775df5b577dc0313/pre_commit-4.6.1-py2.py3-none-any.whl", hash = "sha256:0e3b2942510d1fb34eec167a3ec57331bf8442122f1153a9fb8b58f5c49b2717", size = 226186, upload-time = "2026-07-21T20:56:57.064Z" }, +] + +[[package]] +name = "proto-plus" +version = "1.28.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/6a/056256feb4bd000869aba5c16cf2aa911572ca2a2feb185f86e457b5171e/proto_plus-1.28.3.tar.gz", hash = "sha256:5f91b30dafa6bb38d432c5557a6ee1d35ffd40b4b1e0e3ca27260448560b91d9", size = 58051, upload-time = "2026-08-06T06:24:55.581Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/3a/cfee3c50294f55a2f0f9575052dec2c2a48891ad4b1c2a133b05a87026cd/proto_plus-1.28.3-py3-none-any.whl", hash = "sha256:dc76880b8ee951cca002098574376cf71e055f9f16d9ba6570fb8a06f726d281", size = 50795, upload-time = "2026-08-06T06:23:50.653Z" }, +] + +[[package]] +name = "protobuf" +version = "7.35.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" }, + { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" }, + { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, +] + +[[package]] +name = "pyasn1" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + +[[package]] +name = "pytest-mock" +version = "3.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-discovery" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/b7/1581a8103855c43567776aa34135e5ec3c597346c23bfd10c7eb5e0b10a4/python_discovery-1.5.1.tar.gz", hash = "sha256:e2ea8b884cd1701f386eda8cf327b87743f1dc21b7f784470799537d95635384", size = 77200, upload-time = "2026-07-31T22:06:02.48Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/07/a89b539750a159d5101c4eb9fc84e2961f65cefbd5e0b7440b284471c0b0/python_discovery-1.5.1-py3-none-any.whl", hash = "sha256:ac07f44cade589d954e9d6a1e1468539fdddd2cf676beb51da73e0f156b7c932", size = 35752, upload-time = "2026-07-31T22:06:01.116Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/73/e1/4508a569211b35599016e84ba65c1a992b7a4004b4b6c4bea02a851cba1b/ruff-0.16.2.tar.gz", hash = "sha256:c3d7828d12e8927a6fc65fe38e2c2541b9e762d360a1786d752cb1b8883b3c9c", size = 4885811, upload-time = "2026-08-07T13:31:01.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/57/db19951540f98859c956b50bdb4d31089b4d91e9f15e2968e7d5193806d5/ruff-0.16.2-py3-none-linux_armv6l.whl", hash = "sha256:3c8de4cf2181f01d57946d87d777aa52916976fc09942aed89938fab5e013318", size = 10847925, upload-time = "2026-08-07T13:30:14.468Z" }, + { url = "https://files.pythonhosted.org/packages/13/5a/995fe85a8470d3e391ac0f7fa8054bb454eaf33ee138196d6172ed1079c0/ruff-0.16.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a48cc05c6fbc811ca81b5d7ba95375affea6582d1b8024e455e41afbbf55344", size = 11072662, upload-time = "2026-08-07T13:30:18.143Z" }, + { url = "https://files.pythonhosted.org/packages/32/53/370d767c61c71a971a4ace36703a7ecd8c393956349a7325d7fab2b56827/ruff-0.16.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2c0d14fcbb26c91f0f867a6dc9bd71bbc30b1b6151829c884f23faeab2e5700", size = 10566771, upload-time = "2026-08-07T13:30:20.899Z" }, + { url = "https://files.pythonhosted.org/packages/85/d6/9d96948caf5a632be62d62202d5ec914d6856f204fd79eb036e5915e79ea/ruff-0.16.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:335c621622c4650330be50842561c6586ac6971bb8ab5407fe34dcc9efb16bbe", size = 10975825, upload-time = "2026-08-07T13:30:23.517Z" }, + { url = "https://files.pythonhosted.org/packages/3b/92/ea87129b3414acb0b5770563779c51804d37ac67675c7ba35447ddb14773/ruff-0.16.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20e66910f2c37cc753f9ef6580c914a621b80c4fa3549d3e3521e29d0f5bfc3f", size = 10649437, upload-time = "2026-08-07T13:30:26.097Z" }, + { url = "https://files.pythonhosted.org/packages/ac/43/f8f291dcd4af5bb7872b74fdfa41a7cd7c856ca1d4069670971cf1b9f5cb/ruff-0.16.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e36fbfba65510548156902bcf1350a979a958ce0347ce0f90d73894036b39f", size = 11446761, upload-time = "2026-08-07T13:30:28.752Z" }, + { url = "https://files.pythonhosted.org/packages/71/4a/ef991fb2fcf516ab71f0808adcdd8da5e18c8cde447f4ceaf5f47a5132a5/ruff-0.16.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f0eab35f80df8f134aae5d1630e751901321d317cc8e50dc39e36fa3ed34cd12", size = 12336364, upload-time = "2026-08-07T13:30:31.468Z" }, + { url = "https://files.pythonhosted.org/packages/f3/24/f615e74f307e6ca0e56a482872477b856c70d530aa356abfb6dfe5ca8a80/ruff-0.16.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ea8c0594feb894e89c8c61ab9c103d38b0ea72dfde6c594107147ca31b1140", size = 11630720, upload-time = "2026-08-07T13:30:34.426Z" }, + { url = "https://files.pythonhosted.org/packages/c5/d3/8ef50149e8412a77f7ab409efdef0e2b23803707a3863da4fc64cb23d459/ruff-0.16.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab3d62dde0b19facdd632008cc4827fc28ada7736c6bd35ab6f1050f0bfed53f", size = 11466130, upload-time = "2026-08-07T13:30:36.958Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a7/a19334985c4dea8c381981fa252cd854c7ee52dc4b1686dc16f4a911c702/ruff-0.16.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e43e1f5b8388da9eca1b9e88328d47a5cec794633ccf6f7484ac2dd15eee92c0", size = 11523634, upload-time = "2026-08-07T13:30:39.822Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6c/96d192b0e742412ceda08c0a50f9669b253dde9fd6a60ea1a10c9fa79a63/ruff-0.16.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c24788a980581e1d7ea3a0cbe4344c4fbeb0a6a9b1f4713aa46bb104f8294690", size = 10949807, upload-time = "2026-08-07T13:30:42.745Z" }, + { url = "https://files.pythonhosted.org/packages/fa/51/e26599ceca11e79ee255c7df515995561edf87e9ca1893284e44d98f5a86/ruff-0.16.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:81806b08329130005dd4a8a8394a0c9da8c6f4cafb16ba438d2a2ee6a18bedf1", size = 10646891, upload-time = "2026-08-07T13:30:45.522Z" }, + { url = "https://files.pythonhosted.org/packages/68/01/800c4b1f97bc8d7c6029e06b1f20473a3cf1e13c4933d8f3342add83fc55/ruff-0.16.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4ce4e02bad779bef557f541a1b31f20d6abeae1cc05ed1b1ac019d4ffd1044c8", size = 11162063, upload-time = "2026-08-07T13:30:48.131Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d0/1477ea50fc5a0d4b0b71d1d63d50770bdd794d90b43e37a7618e63ec9894/ruff-0.16.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0422abdf70070255fc4073ce9dfc814cc03db577013761ddd09bc1e4a9a4fbd", size = 11556038, upload-time = "2026-08-07T13:30:50.686Z" }, + { url = "https://files.pythonhosted.org/packages/b8/76/a7776f32048d991e16d4fa8ff91790b877342d3596cc3ed04acdbf1aaedc/ruff-0.16.2-py3-none-win32.whl", hash = "sha256:bf3a63d78fb39f4bf5ac8ae52051c5520505301abe19ba4e204c453b3f09bb0b", size = 10872850, upload-time = "2026-08-07T13:30:53.471Z" }, + { url = "https://files.pythonhosted.org/packages/00/0d/929c800d920e61397d82a01b60bffc68da3052c17d31de59efaad2e4ed75/ruff-0.16.2-py3-none-win_amd64.whl", hash = "sha256:bcabe2f6d0fc7819f1431793005af4e4de7371927d037345bf941252b195b9fa", size = 12023338, upload-time = "2026-08-07T13:30:56.193Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6c/93e26c22c5f78ff87363e07da49c84955affbeb1098bd1936bf3b3f293bf/ruff-0.16.2-py3-none-win_arm64.whl", hash = "sha256:d614e95cedf38a2053fd351c55b103ba30d017d61688fdbfd40ee0412852a99f", size = 11374065, upload-time = "2026-08-07T13:30:58.775Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "tqdm" +version = "4.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/3b/6c24bec5be5e743ffd99576daa5cc077722fc7d5bbc00bd133fa0c698dc6/tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220", size = 795438, upload-time = "2026-07-27T11:33:15.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, +] + +[[package]] +name = "ty" +version = "0.0.69" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/5b/7a618632dfe9373b7df572ecd7a08c8f799d772fbc317da82dd3aa363207/ty-0.0.69.tar.gz", hash = "sha256:b65106e9ff24fa76e25e1142fb09c85244e815c40450e3021d2bf652c231bb43", size = 6565094, upload-time = "2026-08-06T10:04:25.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/60/6534092f4d2c15e2491807edd609c2e50d527c1fed957acf40b9f110b64a/ty-0.0.69-py3-none-linux_armv6l.whl", hash = "sha256:98bfd383b273540829af673e7f98b9c1c4bcc8547d12a1a3806cd0bec7f0e087", size = 12364185, upload-time = "2026-08-06T10:03:47.137Z" }, + { url = "https://files.pythonhosted.org/packages/34/2b/5c29689bd4f74c2e3394d983d85e4011b629f2ce3730c9442553b8554bf8/ty-0.0.69-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:964621ddd05771660017c51b4e74078d861d9fc863c21ef2a500db1ab62c9ccf", size = 12042510, upload-time = "2026-08-06T10:03:49.481Z" }, + { url = "https://files.pythonhosted.org/packages/09/46/fa085bde4d23516d7ef14b24736fc5dd7dc498f60f52b3d077e59ffdea20/ty-0.0.69-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3ffea4048dd0da4c9c97393b4be0901098a9065b06fa81be2477cbde65d8a151", size = 11549397, upload-time = "2026-08-06T10:03:51.747Z" }, + { url = "https://files.pythonhosted.org/packages/25/cc/97b9efb2061dcab6fef1e94a4ad99df0bb45bd2cc15d4f5794c787ee0552/ty-0.0.69-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8684d4a70aadd1eab0f41bdba835e3288ef49db8402a8e6ca81bab52ed5d610", size = 12115567, upload-time = "2026-08-06T10:03:53.79Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c1/a5e0404965093835f3e62544e661784ec0aa8ef0b006ed50af50b19c107e/ty-0.0.69-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:afaaba240ab4122e2069a796836d10be81b4ddb053ae268b3dff962a0b4ca5c7", size = 12149770, upload-time = "2026-08-06T10:03:55.993Z" }, + { url = "https://files.pythonhosted.org/packages/e2/39/8cad6b205a4abe8a044ca0c84aea71e8ccda29b07a75a5f090e310605580/ty-0.0.69-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11ea63ef07d4e33aeb1a775cf5f2c736b3ed22fa6f8b1b608591612c36795044", size = 12941278, upload-time = "2026-08-06T10:03:58.324Z" }, + { url = "https://files.pythonhosted.org/packages/d6/8b/8766d96b732c2a060d70dc8ccafcc4d6a54109a2a95f1deb0705de88892b/ty-0.0.69-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cb3730b1268e92a2907d7aea3afe8dd1b360ae65862f0557080cf479d481b424", size = 13426509, upload-time = "2026-08-06T10:04:00.621Z" }, + { url = "https://files.pythonhosted.org/packages/02/1f/e991b2cde953ea5b94d6a9a4c45c87937bd916bc09235f764407bf471c0a/ty-0.0.69-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a544ff57a752ef186ed40b5a2f44c17402af4cdefeb74a311ca02ebd57c4fca0", size = 13106582, upload-time = "2026-08-06T10:04:02.818Z" }, + { url = "https://files.pythonhosted.org/packages/ea/bb/73538f1b99e3558fd9db87b98698426f0f60fc8666da0b1efd0e70e275eb/ty-0.0.69-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87ed2cbca20caddfdf8e3e14d213ce91b67e75feed78900f4aaf3ef884954028", size = 12708931, upload-time = "2026-08-06T10:04:05.233Z" }, + { url = "https://files.pythonhosted.org/packages/87/cd/484a5208d74c4ad1155933906295ccdce9aa81a257d8df2ab9e41bd60133/ty-0.0.69-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2684efcbce5b6fe45045faf610b377b50781b6d2aa7e61ea23ecf5b3d2bce421", size = 12985322, upload-time = "2026-08-06T10:04:07.587Z" }, + { url = "https://files.pythonhosted.org/packages/6e/81/b75003f0d4da9ab3bc8fd4f4802f836cb9921ff7e70f460604f7b769a0b5/ty-0.0.69-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:da9aeb26fdac1d2214937542b59e0d4d1ba94ec7a3f45444f33c846de1eb1d63", size = 12063910, upload-time = "2026-08-06T10:04:09.835Z" }, + { url = "https://files.pythonhosted.org/packages/8a/76/088469f547ef63dceefc4a75826aedee5014f9371dc5171cde931896a82c/ty-0.0.69-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:00e7677cd14ede381f705f71104ea7b8ea0ce217a8634e19a89781953de0e9ad", size = 12166823, upload-time = "2026-08-06T10:04:12.114Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c9/ce88a0bec0d46d8ae180b99c6ec014866fecc4cba1727b5feec8877b2765/ty-0.0.69-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d91965eb799649833d0d6042db09cd03d15289125245337cc46a2606effb7bda", size = 12483136, upload-time = "2026-08-06T10:04:14.33Z" }, + { url = "https://files.pythonhosted.org/packages/63/9e/6fae0ff225a0012642cf72c077e20f8f448c0a80771bc3360e8178fe2f32/ty-0.0.69-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1f03359cd8e5c412aa0c181118fa9b9061a4dddaedbb61bac0a424fb0814d402", size = 12799025, upload-time = "2026-08-06T10:04:16.445Z" }, + { url = "https://files.pythonhosted.org/packages/e4/43/78a658d18b2a4ccf35b053392f2213bf12e3c63b2abea512d3b6751d1f4c/ty-0.0.69-py3-none-win32.whl", hash = "sha256:ec460e01586b1eb91894c4a8403bee3e045a47e7a4ada943cc27ce8e348e88cf", size = 11787774, upload-time = "2026-08-06T10:04:18.622Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5e/88db1f674403f2b81316a853a44a81ed220621fa96f8f7ae586fb6ca7513/ty-0.0.69-py3-none-win_amd64.whl", hash = "sha256:18976ca26a4e28fc3249477f79a695d5502e670803f2e080d89ac905baef3c6e", size = 12864038, upload-time = "2026-08-06T10:04:20.748Z" }, + { url = "https://files.pythonhosted.org/packages/4d/7b/6fc6efd00c69103d70f2bdbe824343089cd70b17b3079170057d3e5a3ac0/ty-0.0.69-py3-none-win_arm64.whl", hash = "sha256:7d4ca3bb74d91cb9947ba3f3b4cb131ad6a2b3ecc76d34040c4ec6092d2e411d", size = 12196693, upload-time = "2026-08-06T10:04:22.902Z" }, +] + +[[package]] +name = "typeguard" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/de/4420db493fa8fc0856d5e5c1b159c63a323d2de2317babe36b01568928e8/typeguard-4.6.0.tar.gz", hash = "sha256:e7414f09111317de3e335de92cd397c5c0ca00b1cc1676de12e1d444a79b3f21", size = 82330, upload-time = "2026-07-26T08:40:23.207Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/eb/461d5f167b6f5c7d97696f397c82f82e3480e003fce3f0a1cd1dd26e2eb2/typeguard-4.6.0-py3-none-any.whl", hash = "sha256:79878165bb86f2cf5d41d159a0ff1792a796cf496882d2fe1b1c6c7049b9cdd7", size = 36884, upload-time = "2026-07-26T08:40:21.868Z" }, +] + +[[package]] +name = "typer" +version = "0.27.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/40/4a3db7990d1f62a53182aa96eaef57aeb2886a27f90a195bc66713565d31/typer-0.27.1.tar.gz", hash = "sha256:a79bef8469a79c45498e7b814ecf8d603cc7644e9acbd9e19cac0334240b18df", size = 203994, upload-time = "2026-08-03T14:41:03.438Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/89/9518bc0c3929bee36b3a4a8e3daddd6e03f92f9961c66d4983b837160543/typer-0.27.1-py3-none-any.whl", hash = "sha256:53150287edd11baeb4e4722c8e394fcdf8181c0ae89485cba8d25c778d5edd56", size = 122874, upload-time = "2026-08-03T14:41:04.391Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspect" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825, upload-time = "2023-05-24T20:25:47.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827, upload-time = "2023-05-24T20:25:45.287Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "virtualenv" +version = "21.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, + { name = "python-discovery" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9c/10/8b7a5454efc032be50c1c5641467dbb5d31314500205212b2aa66d3c8af4/virtualenv-21.7.3.tar.gz", hash = "sha256:5e9e287f5c808070eea3b40403d3368248e64b24f1b60bd9a36e85ac841d2c3e", size = 5525482, upload-time = "2026-08-08T14:43:20.286Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/2a/83d779d2dfb61f101d7b1c10073d18984e37262d8b4f171c99911a952430/virtualenv-21.7.3-py3-none-any.whl", hash = "sha256:26dfda3c34f29bf1a3ca167426a67658d59979b9954e705aef60a5f724ce1773", size = 5504590, upload-time = "2026-08-08T14:43:18.57Z" }, +] + +[[package]] +name = "win32-setctime" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b663af0e9bb3d4de6578d08f46b1b101c2442fd9aecaa2/win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0", size = 4867, upload-time = "2024-12-07T15:28:28.314Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, +]