diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d84455e..0eebd563 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ as of 2.0.0. ### Added +- Add USAID DEC Archive source (`usaiddec`, alias `USAID`) for AidData's archive of USAID evaluation reports, with PDF downloads and catalog selection by Formatted Document ID (#165) - Add author biographies and portraits from Wikipedia with `--with-author-details` (#31) - Add support for sources without popularity: the source declares it, `config.json` carries the flag, and the UI hides flames and the most popular book and falls back to alphabetical sorting (#14) - Wikisource: add the library mission to the about page (#18) diff --git a/README.md b/README.md index 0355cccf..2a62fdfd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Papers Offline -This multi-source scraper downloads content from [Project Gutenberg](https://www.gutenberg.org), the [Open Textbook Library](https://open.umn.edu/opentextbooks/) and [Wikisource](https://wikisource.org) and packages it into a [ZIM](https://openzim.org) file, a clean and user-friendly format for storing content for offline usage. +This multi-source scraper downloads content from [Project Gutenberg](https://www.gutenberg.org), the [Open Textbook Library](https://open.umn.edu/opentextbooks/), [Wikisource](https://wikisource.org), [One More Library](https://onemorelibrary.com), and the [USAID DEC Archive](https://usaid-archive.aiddata.org/) and packages it into a [ZIM](https://openzim.org) file, a clean and user-friendly format for storing content for offline usage. The ZIM file includes a modern, responsive Vue.js interface with features like: - Browse books by title, author, or source-specific categories such as Library of Congress Classification (LCC) shelves or Open Textbook Library subjects @@ -66,8 +66,8 @@ Note: You'll need to install system dependencies (zim-tools) separately. See [CO The scraper supports several sources, select a source with `--source`: `--source=` Source slug or short name: - gutenberg (PG), onemorelibrary (OML), opentextbooks (OTL) - or wikisource (WS) + gutenberg (PG), onemorelibrary (OML), opentextbooks (OTL), + usaiddec (USAID) or wikisource (WS) Default: gutenberg ### Project Gutenberg @@ -128,6 +128,36 @@ The One More Library-specific options are: `--oml-ids` and `--books` cannot be used together. `--refresh-oml-catalog` requires `--cache-dir`. Selecting a whole library crawls the entire catalog; use `--cache-dir` to cache it across runs. +### USAID DEC Archive + +Select the USAID DEC Archive with `--source=usaiddec` (or `--source=USAID`). + +The source packages [USAID's Development Experience Clearinghouse](https://usaid-archive.aiddata.org/) +evaluation reports, preserved and published by [AidData](https://www.aiddata.org/) with +permission, from the AidData dataset catalog. + +```bash +docker run -v $(pwd)/output:/output ghcr.io/openzim/papers \ + papers2zim --source=usaiddec -f pdf +``` + +To select specific reports by Formatted Document ID: + +```bash +docker run -v $(pwd)/output:/output ghcr.io/openzim/papers \ + papers2zim --source=usaiddec --usaid-ids=PA00JMRT,PA00ZH1K +``` + +The USAID DEC Archive-specific options are: + +```text +--usaid-ids= Exact USAID DEC report IDs (Formatted Document IDs) +--refresh-usaid-catalog Refresh the USAID DEC dataset catalog and exit +``` + +`--usaid-ids` and `--books` cannot be used together. `--refresh-usaid-catalog` +requires `--cache-dir`. + ### Open Textbook Library Select the Open Textbook Library with `--source=opentextbooks` (or `--source=OTL`). @@ -187,7 +217,8 @@ Spaces and underscores are interchangeable, and percent-encoded names are accept ```text --source= Source slug or short name: gutenberg (PG), onemorelibrary (OML), - opentextbooks (OTL) or wikisource (WS) + opentextbooks (OTL), usaiddec (USAID) + or wikisource (WS) Default: gutenberg ``` @@ -247,6 +278,13 @@ Spaces and underscores are interchangeable, and percent-encoded names are accept --wikisource-pages= Wikisource page names, as in the page URL ``` +### USAID DEC Archive Options + +```text +--usaid-ids= Exact USAID DEC report IDs (Formatted Document IDs) +--refresh-usaid-catalog Refresh the USAID DEC dataset catalog and exit +``` + ### Caching Caching is opt-in. Pass `--cache-dir` to persist source metadata and catalog diff --git a/offliner-definition.json b/offliner-definition.json index eb0c84ff..cf90aa4f 100644 --- a/offliner-definition.json +++ b/offliner-definition.json @@ -40,6 +40,13 @@ "dependents": [ "wikisource_pages" ] + }, + { + "title": "USAID DEC Archive", + "value": "usaiddec", + "dependents": [ + "usaid_ids" + ] } ] }, @@ -202,6 +209,12 @@ "title": "Page names (Wikisource only)", "description": "Comma-separated Wikisource page names, as in the page URL (e.g. The_Time_Machine). Write a comma inside a name as %2C. Cannot be combined with Books." }, + "usaid_ids": { + "type": "string", + "required": false, + "title": "Report IDs (USAID DEC Archive only)", + "description": "Comma-separated exact USAID DEC report IDs (Formatted Document IDs). Cannot be combined with Books." + }, "with_author_bio": { "type": "boolean", "required": false, diff --git a/scraper/src/papers2zim/core/assets/icons/sources/usaiddec.svg b/scraper/src/papers2zim/core/assets/icons/sources/usaiddec.svg new file mode 100644 index 00000000..055ad645 --- /dev/null +++ b/scraper/src/papers2zim/core/assets/icons/sources/usaiddec.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/scraper/src/papers2zim/sources/registry.py b/scraper/src/papers2zim/sources/registry.py index 27da9a06..e575d3aa 100644 --- a/scraper/src/papers2zim/sources/registry.py +++ b/scraper/src/papers2zim/sources/registry.py @@ -33,6 +33,13 @@ from papers2zim.sources.opentextbooks.catalog import OpenTextbookLibraryCatalog from papers2zim.sources.opentextbooks.metadata import OpenTextbookLibraryMetadata from papers2zim.sources.opentextbooks.pipeline import OpenTextbookLibraryPipeline +from papers2zim.sources.usaiddec import cli as usaiddec_cli +from papers2zim.sources.usaiddec.catalog import ( + DEFAULT_CATALOG_URL as USAIDDEC_CATALOG_URL, +) +from papers2zim.sources.usaiddec.catalog import UsaidDecCatalog +from papers2zim.sources.usaiddec.metadata import UsaidDecMetadata +from papers2zim.sources.usaiddec.pipeline import UsaidDecPipeline from papers2zim.sources.wikisource import cli as wikisource_cli from papers2zim.sources.wikisource.catalog import WikisourceCatalog from papers2zim.sources.wikisource.metadata import WikisourceMetadata @@ -176,6 +183,35 @@ class SourceProfile: pipeline_class=OnemorelibraryPipeline, ) +USAIDDEC_PROFILE = SourceProfile( + slug="usaiddec", + aliases=("USAID",), + locale_namespace="usaiddec", + display_name="USAID DEC Archive", + source_creator="aiddata.org", + zim_tags="_category:usaiddec;usaiddec", + zim_name_prefix="usaiddec", + icon_svg="usaiddec.svg", + tagline="USAID evaluation reports preserved by AidData", + source_description=( + "USAID's Development Experience Clearinghouse evaluation reports, " + "archived by AidData." + ), + collection_label="Subjects", + collection_icon_style="subject", + has_popularity=False, + default_mirror_url=USAIDDEC_CATALOG_URL, + catalog_feed_path="", + catalog=UsaidDecCatalog, + cli_options=usaiddec_cli.CLI_OPTIONS, + parse_cli_options=usaiddec_cli.parse_options, + handle_cli_action=usaiddec_cli.handle_cli_action, + pipeline_options=lambda _mirror_url, _cache_dir, _source_options, _languages: {}, + metadata_options=lambda _mirror_url, _cache_dir, _engine: {}, + metadata_class=UsaidDecMetadata, + pipeline_class=UsaidDecPipeline, +) + WIKISOURCE_PROFILE = SourceProfile( slug="wikisource", aliases=("WS",), @@ -281,3 +317,4 @@ def get_source(slug: str) -> SourceProfile: register_source(OPEN_TEXTBOOK_LIBRARY_PROFILE) register_source(WIKISOURCE_PROFILE) register_source(ONEMORELIBRARY_PROFILE) +register_source(USAIDDEC_PROFILE) diff --git a/scraper/src/papers2zim/sources/usaiddec/__init__.py b/scraper/src/papers2zim/sources/usaiddec/__init__.py new file mode 100644 index 00000000..b4463d72 --- /dev/null +++ b/scraper/src/papers2zim/sources/usaiddec/__init__.py @@ -0,0 +1,7 @@ +"""USAID DEC Archive source. + +Implements the USAID Development Experience Clearinghouse (DEC) archive +source: a curated collection of USAID-funded evaluation reports made +available by AidData. Catalog metadata and PDF download links come from +the AidData dataset CSV. +""" diff --git a/scraper/src/papers2zim/sources/usaiddec/catalog.py b/scraper/src/papers2zim/sources/usaiddec/catalog.py new file mode 100644 index 00000000..f8db7522 --- /dev/null +++ b/scraper/src/papers2zim/sources/usaiddec/catalog.py @@ -0,0 +1,331 @@ +"""USAID DEC Archive catalog access backed by the AidData dataset CSV. + +The AidData dataset is distributed as a ZIP file containing a CSV with +metadata and permanent PDF/TXT URLs for each evaluation report. The +catalog downloads and caches that CSV for discovery and filtering. +""" + +import csv +import io +import zipfile +from collections.abc import Iterable +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +import babel + +from papers2zim.constants import logger +from papers2zim.core.download_engine import DownloadEngine +from papers2zim.core.language import ISO_MATRIX, ISO_MATRIX_REV +from papers2zim.core.ports import CatalogFilters, CatalogPort, WorkRef +from papers2zim.core.utils import critical_error + +USAIDDEC_SOURCE = "usaiddec" +CATALOG_FILENAME = "usaiddec_catalog.csv" +DEFAULT_CATALOG_URL = ( + "https://docs.aiddata.org/ad4/datasets/" + "AidData_2000-2024_Archive_of_USAID_DEC_Evaluation_Reports_Dataset_Version_1_0.zip" +) +SUPPORTED_FORMATS = frozenset({"pdf"}) + + +@lru_cache(maxsize=1) +def _language_name_codes() -> dict[str, str]: + """Map English language names ("French") to ISO 639-1 codes ("fr"). + + Babel ships the CLDR names, so this stays in sync with the ISO matrix in + ``papers2zim.core.language`` instead of duplicating a hand-written map. + """ + codes: dict[str, str] = {} + for code in ISO_MATRIX: + try: + name = babel.Locale(code).get_language_name("en") + except (babel.UnknownLocaleError, ValueError) as exc: + logger.debug("Cannot resolve the name of language %r: %s", code, exc) + continue + if name: + codes.setdefault(name.casefold(), code) + return codes + + +def normalize_language(value: str | None) -> str | None: + """Normalize a dataset language label to a valid ISO 639-1 code. + + Accepts ISO codes (``en``/``eng``) as well as English language names + (``English``). Unrecognized labels yield ``None`` rather than an invented + code such as ``ger``/``hai``, which would surface as a raw fragment in the + UI and silently break ``--languages`` filtering. + """ + if not value: + return None + text = value.strip() + if not text: + return None + key = text.casefold() + if key in ISO_MATRIX: + return key + if key in ISO_MATRIX_REV: + return ISO_MATRIX_REV[key] + code = _language_name_codes().get(key) + if code is None: + logger.warning("Ignoring unrecognized USAID DEC language label %r", text) + return code + + +def _split_pipe(value: str | None) -> list[str]: + if not value: + return [] + return [item.strip() for item in value.split("|") if item.strip()] + + +def normalize_languages(value: str | None) -> tuple[str, ...]: + """Normalize a pipe-separated language field into ISO 639-1 codes. + + Reports are often bilingual ("French|English"), so each label is resolved + independently and duplicates are dropped while keeping dataset order. + """ + codes: list[str] = [] + for label in _split_pipe(value): + code = normalize_language(label) + if code and code not in codes: + codes.append(code) + return tuple(codes) + + +@dataclass(frozen=True, slots=True) +class UsaidDecEntry: + book_id: str + title: str + description: str | None + languages: tuple[str, ...] + authors: tuple[str, ...] + date_published: str | None + pdf_url: str | None + source_url: str | None + file_size: str | None + bibtype: str | None + class_name: str | None + subjects: tuple[str, ...] + geographies: tuple[str, ...] + + +def _extract_csv_from_zip(data: bytes) -> str: + with zipfile.ZipFile(io.BytesIO(data)) as archive: + csv_names = [ + name + for name in archive.namelist() + if name.endswith(".csv") + and "original_source" in name.lower() + and "__MACOSX" not in name + ] + if not csv_names: + csv_names = [ + name + for name in archive.namelist() + if name.endswith(".csv") and "__MACOSX" not in name + ] + if not csv_names: + critical_error("USAID DEC dataset ZIP does not contain a CSV file") + return archive.read(csv_names[0]).decode("utf-8-sig") + + +def _parse_catalog(csv_text: str) -> list[UsaidDecEntry]: + reader = csv.DictReader(io.StringIO(csv_text)) + entries: list[UsaidDecEntry] = [] + for row in reader: + book_id = (row.get("Formatted Document ID") or "").strip() + if not book_id: + continue + title = (row.get("Document Title") or "").strip() + if not title: + continue + abstract = (row.get("Abstract") or "").strip() or None + ancillary = (row.get("Ancilary Data") or "").strip() or None + description = abstract or ancillary + authors = tuple( + name + for name in _split_pipe(row.get("Personal Author")) + if name and name.casefold() != "unknown" + ) + if not authors: + inst_author = (row.get("Institution or USAID Bureau Author") or "").strip() + if inst_author: + authors = (inst_author,) + class_value = (row.get("Class") or "").strip() + subject_values = [ + *_split_pipe(row.get("Descriptions from Thesaurus")), + *([class_value] if class_value else []), + ] + subjects = tuple(dict.fromkeys(s for s in subject_values if s)) + geographies = tuple( + s for s in _split_pipe(row.get("Descriptors Geographic")) if s + ) + date_published = ( + (row.get("Date of Publication Fixed") or "").strip() + or (row.get("Date of Publication Freeforrm") or "").strip() + or None + ) + pdf_url = (row.get("AidDataURL__PDF") or "").strip() or None + entries.append( + UsaidDecEntry( + book_id=book_id, + title=title, + description=description, + languages=normalize_languages(row.get("Language of Text")), + authors=authors, + date_published=date_published, + pdf_url=pdf_url, + source_url=(row.get("[DetailLink]") or "").strip() or None, + file_size=(row.get("File Size") or "").strip() or None, + bibtype=(row.get("Bibtype Name") or "").strip() or None, + class_name=class_value or None, + subjects=subjects, + geographies=geographies, + ) + ) + return entries + + +class UsaidDecCatalog(CatalogPort): + """Discover USAID DEC evaluation reports from the AidData dataset CSV.""" + + def __init__( + self, + engine: DownloadEngine, + base_url: str = DEFAULT_CATALOG_URL, + cache_dir: Path | None = None, + *, + refresh_catalog: bool = False, + **_: object, + ): + self._engine = engine + self._catalog_url = base_url + self._csv_path = cache_dir.resolve() / CATALOG_FILENAME if cache_dir else None + self._refresh_catalog = refresh_catalog + self._entries: list[UsaidDecEntry] | None = None + + def discover(self, filters: CatalogFilters) -> Iterable[WorkRef]: + requested_formats = set(filters.formats or SUPPORTED_FORMATS) + if not requested_formats.intersection(SUPPORTED_FORMATS): + logger.info( + "USAID DEC Archive only offers PDF; no requested format can be " + "downloaded" + ) + return [] + entries = self._load_catalog() + + downloadable = [entry for entry in entries if entry.pdf_url] + source_book_ids = { + str(book_id).strip() + for book_id in filters.options.get("book_ids", []) + if str(book_id).strip() + } + positions = { + int(book_id) + for book_id in filters.book_ids or [] + if str(book_id).strip().isdigit() + } + if source_book_ids: + if filters.languages: + logger.info( + "Ignoring language filters for explicitly requested USAID DEC IDs" + ) + wanted = {book_id.casefold() for book_id in source_book_ids} + selected = [ + entry for entry in downloadable if entry.book_id.casefold() in wanted + ] + elif positions: + if filters.languages: + logger.info( + "Ignoring language filters for explicitly requested USAID DEC " + "positions" + ) + selected = [ + entry + for position, entry in enumerate(downloadable, start=1) + if position in positions + ] + else: + selected = self._filter_entries(downloadable, filters) + + logger.info( + " Selected %s USAID DEC reports from %s catalog entries (languages: %s)", + len(selected), + len(entries), + filters.languages or "all", + ) + return [self._to_ref(entry) for entry in selected] + + @staticmethod + def _filter_entries( + entries: list[UsaidDecEntry], filters: CatalogFilters + ) -> list[UsaidDecEntry]: + if filters.languages: + wanted = {lang.casefold() for lang in filters.languages} + # accept ISO 639-1 and 639-3 codes interchangeably + mapped = set(wanted) + for lang in wanted: + mapped.add(ISO_MATRIX.get(lang, lang)) + mapped.add(ISO_MATRIX_REV.get(lang, lang)) + entries = [ + entry + for entry in entries + if not entry.languages + or any(lang.casefold() in mapped for lang in entry.languages) + ] + return entries + + @staticmethod + def _to_ref(entry: UsaidDecEntry) -> WorkRef: + return WorkRef( + id=entry.book_id, + source=USAIDDEC_SOURCE, + extra={ + "title": entry.title, + "description": entry.description, + "languages": list(entry.languages) or ["en"], + "authors": list(entry.authors), + "date_published": entry.date_published, + "pdf_url": entry.pdf_url, + "source_url": entry.source_url, + "file_size": entry.file_size, + "bibtype": entry.bibtype, + "class_name": entry.class_name, + "subjects": list(entry.subjects), + "geographies": list(entry.geographies), + }, + ) + + def refresh(self) -> None: + if self._csv_path is None: + critical_error("--refresh-catalog requires --cache-dir") + self._entries = None + self._load_catalog(force=True) + + def _load_catalog(self, *, force: bool = False) -> list[UsaidDecEntry]: + if self._entries is not None and not force: + return self._entries + + csv_text = self._read_csv_text(force=force) + self._entries = _parse_catalog(csv_text) + return self._entries + + def _read_csv_text(self, *, force: bool) -> str: + if self._csv_path is None: + logger.info("Downloading USAID DEC catalog from %s", self._catalog_url) + return _extract_csv_from_zip(self._engine.fetch_bytes(self._catalog_url)) + + if force or self._refresh_catalog or not self._csv_path.exists(): + logger.info( + "%s USAID DEC catalog from %s", + "Refreshing" if self._csv_path.exists() else "Downloading", + self._catalog_url, + ) + data = self._engine.fetch_bytes(self._catalog_url) + csv_text = _extract_csv_from_zip(data) + self._csv_path.write_text(csv_text, encoding="utf-8") + return csv_text + + logger.info("Using cached USAID DEC catalog at %s", self._csv_path) + return self._csv_path.read_text(encoding="utf-8") diff --git a/scraper/src/papers2zim/sources/usaiddec/cli.py b/scraper/src/papers2zim/sources/usaiddec/cli.py new file mode 100644 index 00000000..e7a7e9ac --- /dev/null +++ b/scraper/src/papers2zim/sources/usaiddec/cli.py @@ -0,0 +1,37 @@ +"""USAID DEC Archive-specific command-line options.""" + +from typing import Any + +from papers2zim.core.utils import critical_error + +CLI_OPTIONS = { + "--usaid-ids": ( + " --usaid-ids= Exact USAID DEC report IDs " + "(Formatted Document IDs, comma-separated)" + ), + "--refresh-usaid-catalog": ( + " --refresh-usaid-catalog Refresh the USAID DEC dataset catalog and " + "exit" + ), +} + + +def parse_options(arguments: dict[str, Any]) -> dict[str, Any]: + book_ids = [ + item.strip() + for item in (arguments.get("--usaid-ids") or "").split(",") + if item.strip() + ] + if book_ids and arguments.get("--books"): + critical_error("Use either --books or --usaid-ids, not both") + return { + "book_ids": book_ids, + "refresh_catalog": bool(arguments.get("--refresh-usaid-catalog")), + } + + +def handle_cli_action(catalog: Any, options: dict[str, Any]) -> bool: + if options.get("refresh_catalog"): + catalog.refresh() + return True + return False diff --git a/scraper/src/papers2zim/sources/usaiddec/collections.py b/scraper/src/papers2zim/sources/usaiddec/collections.py new file mode 100644 index 00000000..3878570e --- /dev/null +++ b/scraper/src/papers2zim/sources/usaiddec/collections.py @@ -0,0 +1,282 @@ +"""Curated thematic collections for the USAID DEC Archive. + +AidData tags each evaluation report with a free-text ``Class`` label drawn +from a few hundred sector names, and with a long list of thesaurus +descriptors. Exposing those labels directly would create well over a +thousand near-duplicate collections ("HIV/AIDS" and "HIV / AIDS", "Health" +and "Health care", ...), which is unusable in the UI. + +Instead, this module folds every report into a small curated set of thematic +sectors (about twenty). The rules are ordered: the first matching keyword +wins, so more specific sectors such as HIV/AIDS are checked before the +generic Health sector. +""" + +from collections.abc import Iterable + +OTHER_COLLECTION = "Other" + +# Ordered (collection name, keywords). Keywords are matched against the +# lowercased report class, then against its thesaurus descriptors. +COLLECTION_RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("HIV/AIDS", ("hiv",)), + ( + "Maternal & Child Health", + ( + "maternal", + "child survival", + "child health", + "child nutrition", + "breastfeeding", + "infant", + "family health", + ), + ), + ( + "Reproductive Health & Family Planning", + ("reproductive", "family planning", "contracept"), + ), + ("Nutrition", ("nutrition",)), + ( + "Health", + ( + "health", + "disease", + "tuberculosis", + "malaria", + "immunization", + "medical", + "hospital", + "clinic", + "epidemic", + "pandemic", + ), + ), + ("Water & Sanitation", ("water", "sanitation", "hydrology", "potable")), + ( + "Agriculture & Food Security", + ( + "agricultur", + "food security", + "food aid", + "agribusiness", + "crop", + "livestock", + "fisher", + "aquaculture", + "irrigation", + "farming", + "fertilizer", + "seed", + "maize", + "food processing", + "food supply", + "cooperative", + "rural development", + "animal husbandry", + ), + ), + ( + "Environment & Climate", + ( + "environment", + "natural resources", + "biological diversity", + "ecology", + "forestry", + "climate", + "pollution", + "conservation", + "wildlife", + "soil", + "coastal", + ), + ), + ( + "Education", + ("education", "school", "teacher", "literacy", "learning", "vocational"), + ), + ( + "Energy & Infrastructure", + ( + "energy", + "electric", + "power", + "renewable", + "road", + "transport", + "housing", + "infrastructure", + "construction", + "urban", + "steel", + ), + ), + ( + "Technology & Communications", + ( + "technology", + "information", + "communication", + "media", + "science", + "data collection", + ), + ), + ( + "Economic Growth & Trade", + ( + "economic", + "trade", + "financ", + "investment", + "credit", + "market", + "business", + "enterprise", + "employment", + "industry", + "exports", + "income", + "private sector", + "capital", + "privatization", + "banking", + "labor", + ), + ), + ( + "Governance & Public Administration", + ( + "governance", + "government", + "public administration", + "budget", + "management reform", + "legislative", + "laws and legislation", + "rule of law", + "public land", + "demobilization", + "land reform", + "development cooperation", + "development assistance", + "foreign assistance", + "international politics", + "embargoes", + ), + ), + ( + "Democracy & Civil Society", + ( + "civil society", + "democrat", + "election", + "human rights", + "social justice", + "political", + "advocacy", + "organizations", + ), + ), + ( + "Conflict & Security", + ( + "conflict", + "security", + "displaced", + "land mines", + "emergency", + "transition", + "resilient societies", + ), + ), + ( + "Humanitarian Assistance & Disaster", + ("disaster", "humanitarian", "relief", "refugee"), + ), + ("Gender Equality", ("gender", "women", "girls")), + ( + "Poverty & Social Development", + ( + "poverty", + "social", + "communit", + "quality of life", + "population groups", + "regional", + "socioeconomic", + "human behavior", + "psychology", + "culture and society", + "safety", + ), + ), + ( + "Population & Demography", + ("population", "demograph", "vital statistics", "migration"), + ), + ( + "Capacity Building & Evaluation", + ( + "training", + "capacity", + "evaluation", + "performance", + "planning", + "management", + "administration", + "personnel", + "research", + "strategic", + ), + ), +) + +COLLECTION_NAMES: tuple[str, ...] = ( + *(name for name, _ in COLLECTION_RULES), + OTHER_COLLECTION, +) + + +def normalize_class(value: str | None) -> str: + """Normalize a dataset ``Class`` value into a keyword-matching haystack. + + A few values are full paths ("Top/Health/Diseases/Tuberculosis"); the + leaf is the meaningful part. Labels such as "HIV / AIDS" keep their + spaces around the slash and are left intact. + """ + text = (value or "").strip() + if not text: + return "" + if text.casefold().startswith("top/"): + text = text[len("top/") :].split("/")[-1] + return text.casefold() + + +def _match(haystack: str) -> str | None: + for name, keywords in COLLECTION_RULES: + if any(keyword in haystack for keyword in keywords): + return name + return None + + +def collection_for(class_name: str | None, subjects: Iterable[str] = ()) -> str: + """Return the curated collection a report belongs to.""" + matched = _match(normalize_class(class_name)) + if matched: + return matched + subject_text = " | ".join( + subject.casefold() for subject in subjects if subject and subject.strip() + ) + matched = _match(subject_text) + if matched: + return matched + return OTHER_COLLECTION + + +def collection_slug(name: str) -> str: + """Build a stable, URL-friendly collection id from its display name.""" + slug = name.casefold().replace("&", " and ") + slug = "".join(character if character.isalnum() else "-" for character in slug) + return "-".join(part for part in slug.split("-") if part) diff --git a/scraper/src/papers2zim/sources/usaiddec/metadata.py b/scraper/src/papers2zim/sources/usaiddec/metadata.py new file mode 100644 index 00000000..6620b446 --- /dev/null +++ b/scraper/src/papers2zim/sources/usaiddec/metadata.py @@ -0,0 +1,130 @@ +"""USAID DEC Archive metadata access. + +Catalog discovery already carries the dataset CSV metadata for each report, +so fetching a work is a direct conversion from the catalog reference. +""" + +import hashlib +from collections.abc import Iterable, Iterator +from pathlib import Path + +from papers2zim.core.download_engine import DownloadEngine +from papers2zim.core.models import CollectionRef, Creator, Format, Work +from papers2zim.core.ports import MetadataPort, WorkRef +from papers2zim.core.utils import slugify +from papers2zim.sources.usaiddec.catalog import DEFAULT_CATALOG_URL, USAIDDEC_SOURCE +from papers2zim.sources.usaiddec.collections import ( + collection_for, + collection_slug, +) + + +def _author_id(name: str) -> str: + """Deterministic id shared by every work of the same author. + + Python's built-in ``hash`` is randomized per process, so it cannot be used + here: it would give the same author a different id on every run, making + author pages and their work counts non-reproducible. + """ + normalized = " ".join(name.split()) + slug = slugify(normalized, max_encoded_length=80) + digest = hashlib.sha256(normalized.casefold().encode()).hexdigest()[:8] + return f"{slug}-{digest}" if slug else digest + + +def _sort_name(name: str) -> str: + """Surname from the AidData ``Personal Author`` ``"Last, First"`` format.""" + last, separator, _ = name.partition(",") + if separator and last.strip(): + return last.strip() + parts = name.split() + return parts[-1] if parts else name + + +def _author_creators(names: list[str]) -> list[Creator]: + creators: list[Creator] = [] + for raw_name in names: + name = " ".join(raw_name.split()) + if not name: + continue + creators.append( + Creator( + id=_author_id(name), + display_name=name, + sort_name=_sort_name(name), + ) + ) + return creators + + +def _collections(class_name: str | None, subjects: list[str]) -> list[CollectionRef]: + """Attach the report to exactly one curated thematic collection.""" + name = collection_for(class_name, subjects) + return [ + CollectionRef( + id=collection_slug(name), + name=name, + kind="subject", + ) + ] + + +class UsaidDecMetadata(MetadataPort): + """Builds Work records from USAID DEC catalog references.""" + + def __init__( + self, + base_url: str = DEFAULT_CATALOG_URL, + *, + engine: DownloadEngine | None = None, + cache_dir: Path | None = None, + ): + del base_url, engine, cache_dir + + def fetch(self, refs: Iterable[WorkRef]) -> Iterator[Work]: + for ref in refs: + extra = dict(ref.extra) + pdf_url = extra.get("pdf_url") + formats = ( + [Format(name="pdf", media_type="application/pdf", url=pdf_url)] + if isinstance(pdf_url, str) and pdf_url + else [] + ) + authors = extra.get("authors") or [] + if not isinstance(authors, list): + authors = [] + subjects = extra.get("subjects") or [] + geographies = extra.get("geographies") or [] + work = Work( + id=ref.id, + source=USAIDDEC_SOURCE, + title=str(extra.get("title") or ref.id), + creators=_author_creators([str(a) for a in authors]), + languages=[ + str(lang) for lang in (extra.get("languages") or ["en"]) if lang + ], + license="With permission from AidData", + formats=formats, + collections=_collections( + (str(extra["class_name"]) if extra.get("class_name") else None), + [str(s) for s in subjects if s], + ), + description=( + str(extra["description"]) if extra.get("description") else None + ), + published=( + str(extra["date_published"]) + if extra.get("date_published") + else None + ), + source_url=( + str(extra["source_url"]) if extra.get("source_url") else None + ), + extra={ + "bibtype": extra.get("bibtype"), + "file_size": extra.get("file_size"), + "subjects": subjects, + "geographies": geographies, + }, + ) + yield work diff --git a/scraper/src/papers2zim/sources/usaiddec/pipeline.py b/scraper/src/papers2zim/sources/usaiddec/pipeline.py new file mode 100644 index 00000000..15d29989 --- /dev/null +++ b/scraper/src/papers2zim/sources/usaiddec/pipeline.py @@ -0,0 +1,100 @@ +"""Per-work processing for USAID DEC Archive reports.""" + +import requests + +from papers2zim.constants import logger +from papers2zim.core.content_validation import is_valid_book_file +from papers2zim.core.covers import extract_cover +from papers2zim.core.download_engine import DownloadEngine, is_fatal_http_error +from papers2zim.core.models import Cover +from papers2zim.core.pipeline import Pipeline +from papers2zim.core.ports import WorkRef +from papers2zim.core.utils import archive_name_for +from papers2zim.sources.usaiddec.catalog import SUPPORTED_FORMATS +from papers2zim.sources.usaiddec.resolver import UsaidDecFormatResolver + + +class UsaidDecPipeline(Pipeline): + """Download USAID DEC evaluation report PDFs and add them to the ZIM.""" + + def __init__(self, *, engine: DownloadEngine, **kwargs): + super().__init__(**kwargs) + self.engine = engine + self.resolver = UsaidDecFormatResolver() + + def process_ref(self, ref: WorkRef) -> None: + works = list(self.metadata.fetch([ref])) + if not works: + return + work = works[0] + unsupported = work.extra.setdefault("unsupported_formats", []) + downloaded: dict[str, bytes] = {} + cover_image: bytes | None = None + + for format_name in self.formats: + if format_name not in SUPPORTED_FORMATS: + unsupported.append(format_name) + continue + request = self.resolver.resolve(work, format_name) + if request is None: + unsupported.append(format_name) + continue + try: + content = self.engine.fetch_bytes(request.url) + except requests.RequestException as exc: + log = logger.debug if is_fatal_http_error(exc) else logger.warning + log( + "Could not download %s for USAID DEC report #%s: %s", + format_name, + work.id, + exc, + ) + unsupported.append(format_name) + continue + if not is_valid_book_file(content, format_name): + logger.warning( + "USAID DEC report #%s returned an invalid %s file", + work.id, + format_name.upper(), + ) + unsupported.append(format_name) + continue + + media_type = next( + ( + book_format.media_type + for book_format in work.formats + if book_format.name == format_name + ), + "application/pdf", + ) + self.assembler.add_item_for( + path=archive_name_for(work, format_name), + content=content, + mimetype=media_type, + is_front=False, + ) + downloaded[format_name] = content + if cover_image is None: + cover_image = extract_cover(content, format_name) + + if not downloaded: + logger.warning( + "USAID DEC report #%s has no downloadable requested format", + work.id, + ) + return + + if cover_image is not None: + self.assembler.add_item_for( + path=f"covers/{work.id}_cover_image.webp", + content=cover_image, + mimetype="image/webp", + is_front=False, + ) + work.cover = Cover() + work.extra["has_cover"] = True + else: + work.extra["has_cover"] = False + + self.store.add(work) diff --git a/scraper/src/papers2zim/sources/usaiddec/resolver.py b/scraper/src/papers2zim/sources/usaiddec/resolver.py new file mode 100644 index 00000000..2fd9ae52 --- /dev/null +++ b/scraper/src/papers2zim/sources/usaiddec/resolver.py @@ -0,0 +1,20 @@ +"""USAID DEC Archive format resolution.""" + +from papers2zim.core.models import Work +from papers2zim.core.ports import DownloadRequest, FormatResolverPort + + +class UsaidDecFormatResolver(FormatResolverPort): + """Resolve the direct PDF download URL of a USAID DEC report.""" + + def resolve(self, work: Work, format_name: str) -> DownloadRequest | None: + if format_name != "pdf": + return None + for book_format in work.formats: + if book_format.name == "pdf" and book_format.url: + return DownloadRequest( + url=book_format.url, + format_name="pdf", + extra={"media_type": book_format.media_type}, + ) + return None diff --git a/scraper/tests/test_offliner_definition.py b/scraper/tests/test_offliner_definition.py index 5bc2e6e0..584e1af2 100644 --- a/scraper/tests/test_offliner_definition.py +++ b/scraper/tests/test_offliner_definition.py @@ -43,6 +43,11 @@ def test_recipe_defines_supported_sources_and_source_specific_filters(): "value": "wikisource", "dependents": ["wikisource_pages"], }, + { + "title": "USAID DEC Archive", + "value": "usaiddec", + "dependents": ["usaid_ids"], + }, ], } assert { @@ -52,6 +57,7 @@ def test_recipe_defines_supported_sources_and_source_specific_filters(): "subjects", "otl_ids", "wikisource_pages", + "usaid_ids", "with_author_bio", "with_author_portrait", }.issubset(flags) diff --git a/scraper/tests/test_source_registry.py b/scraper/tests/test_source_registry.py index e5f85902..405a734f 100644 --- a/scraper/tests/test_source_registry.py +++ b/scraper/tests/test_source_registry.py @@ -14,6 +14,8 @@ from papers2zim.sources.opentextbooks.catalog import OpenTextbookLibraryCatalog from papers2zim.sources.opentextbooks.metadata import OpenTextbookLibraryMetadata from papers2zim.sources.registry import get_source +from papers2zim.sources.usaiddec.catalog import UsaidDecCatalog +from papers2zim.sources.usaiddec.metadata import UsaidDecMetadata from papers2zim.sources.wikisource.catalog import WikisourceCatalog from papers2zim.sources.wikisource.metadata import WikisourceMetadata @@ -42,6 +44,8 @@ def _profile(slug: str, aliases: tuple[str, ...]): ("OTL", "opentextbooks"), ("OML", "onemorelibrary"), ("oml", "onemorelibrary"), + ("USAID", "usaiddec"), + ("usaid", "usaiddec"), ("WS", "wikisource"), ("ws", "wikisource"), ), @@ -104,10 +108,23 @@ def test_wikisource_profile_is_registered_with_its_opds_adapters(): assert profile.has_popularity is False +def test_usaiddec_profile_is_registered_with_its_adapters(): + profile = get_source("usaiddec") + + assert profile.display_name == "USAID DEC Archive" + assert profile.has_popularity is False + assert profile.catalog is UsaidDecCatalog + assert isinstance(profile.catalog, type) + assert issubclass(profile.catalog, CatalogPort) + assert profile.metadata_class is UsaidDecMetadata + assert profile.pipeline_class is not None + + def test_popularity_is_declared_per_source(): assert get_source("gutenberg").has_popularity is True assert get_source("opentextbooks").has_popularity is True assert get_source("onemorelibrary").has_popularity is True + assert get_source("usaiddec").has_popularity is False assert get_source("wikisource").has_popularity is False diff --git a/scraper/tests/test_usaiddec_catalog.py b/scraper/tests/test_usaiddec_catalog.py new file mode 100644 index 00000000..b9ff99c3 --- /dev/null +++ b/scraper/tests/test_usaiddec_catalog.py @@ -0,0 +1,402 @@ +"""Tests for USAID DEC Archive catalog discovery.""" + +import csv +import io +import zipfile + +import pytest + +from papers2zim.core.download_engine import DownloadEngine +from papers2zim.core.ports import CatalogFilters +from papers2zim.core.utils import CriticalError +from papers2zim.sources.usaiddec.catalog import ( + CATALOG_FILENAME, + DEFAULT_CATALOG_URL, + USAIDDEC_SOURCE, + UsaidDecCatalog, + _extract_csv_from_zip, + _parse_catalog, + _split_pipe, + normalize_language, + normalize_languages, +) + +CSV_HEADER = [ + "row_id", + "Abstract", + "Ancilary Data", + "Personal Author", + "Bibliographic Type Code", + "Bibtype Name", + "Contract_Grant_Number", + "Date of Publication Freeforrm", + "Date of Publication Fixed", + "Descriptions from Thesaurus", + "Secondary Descriptors", + "Descriptors Geographic", + "Document ID", + "Edition", + "AID Contact Number Evaluated", + "File Size", + "USAID Geography Code", + "USAID Geography Name", + "New Thesaurus Terms", + "Institution Authority Code", + "Institution Name", + "Institution Role", + "Language of Text", + "Primary Subject", + "Document Title", + "Series Title", + "Document Title Translated", + "Title Word Alternate Spelling", + "USAID Project Number", + "Institution or USAID Bureau Author", + "Institution or USAID Sponsor", + "Institution or USAID Publisher", + "Class", + "DateCreated", + "inmCreatedBy", + "Descriptors_Geographic_t", + "Contract_Number_t", + "Primary_Subject_t", + "RegionsAndCities", + "RegionsAndCities_t", + "DOI", + "DigitalObjectIdentifier", + "[DetailLink]", + "Formatted Document ID", + "AidDataURL__PDF", + "AidDataURL__TXT", +] + + +def _row(**overrides: str) -> dict[str, str]: + row = dict.fromkeys(CSV_HEADER, "") + row.update( + { + "row_id": "1", + "Personal Author": "Doe, Jane", + "Bibtype Name": "Final Evaluation Report", + "Date of Publication Fixed": "2020-01", + "Descriptions from Thesaurus": "Education|Health", + "Descriptors Geographic": "Kenya", + "Language of Text": "English", + "Document Title": "Example Evaluation", + "Class": "Education (General)", + "[DetailLink]": "https://dec.usaid.gov/example", + "Formatted Document ID": "PA000001", + "AidDataURL__PDF": "https://docs.aiddata.org/ad4/pdfs/usaid-archive/PA000001.pdf", + } + ) + row.update(overrides) + return row + + +def _csv_text(*rows: dict[str, str]) -> str: + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=CSV_HEADER) + writer.writeheader() + for row in rows: + writer.writerow(row) + return output.getvalue() + + +def _zip_bytes(csv_text: str, name: str | None = None) -> bytes: + name = name or ( + "AidData_2000-2024_Archive_of_USAID_DEC_Evaluation_Reports_Dataset_Version_1_0/" + "AidData_USAID_DEC_Archive_original_source.csv" + ) + output = io.BytesIO() + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr(name, csv_text) + return output.getvalue() + + +class StubEngine(DownloadEngine): + def __init__(self, payload: bytes): + self.payload = payload + self.downloaded: list[str] = [] + + def fetch_bytes(self, url, headers=None): + del headers + self.downloaded.append(url) + return self.payload + + +def test_extract_csv_from_zip_finds_original_source_csv(): + payload = _zip_bytes(_csv_text(_row())) + text = _extract_csv_from_zip(payload) + assert "Formatted Document ID" in text + assert "PA000001" in text + + +def test_parse_catalog_extracts_report_fields(): + (entry,) = _parse_catalog(_csv_text(_row())) + + assert entry.book_id == "PA000001" + assert entry.title == "Example Evaluation" + assert entry.languages == ("en",) + assert entry.authors == ("Doe, Jane",) + assert entry.class_name == "Education (General)" + assert entry.pdf_url is not None + assert entry.pdf_url.endswith("/PA000001.pdf") + assert entry.subjects[:2] == ("Education", "Health") + assert entry.geographies == ("Kenya",) + + +def test_discover_returns_pdf_reports(): + engine = StubEngine(_zip_bytes(_csv_text(_row()))) + catalog = UsaidDecCatalog(engine) + + refs = list(catalog.discover(CatalogFilters(formats=["pdf"]))) + + assert [ref.id for ref in refs] == ["PA000001"] + assert refs[0].source == USAIDDEC_SOURCE + assert refs[0].extra["languages"] == ["en"] + assert engine.downloaded == [DEFAULT_CATALOG_URL] + + +def test_discover_returns_nothing_when_no_supported_format_requested(): + engine = StubEngine(_zip_bytes(_csv_text(_row()))) + catalog = UsaidDecCatalog(engine) + + assert list(catalog.discover(CatalogFilters(formats=["html"]))) == [] + assert engine.downloaded == [] + + +def test_discover_filters_by_formatted_id_and_position(): + rows = _csv_text( + { + **_row(row_id="1"), + "Formatted Document ID": "PA000001", + "Document Title": "One", + }, + { + **_row(row_id="2"), + "Formatted Document ID": "PA000002", + "Document Title": "Two", + "AidDataURL__PDF": "https://docs.aiddata.org/ad4/pdfs/usaid-archive/PA000002.pdf", + }, + ) + engine = StubEngine(_zip_bytes(rows)) + catalog = UsaidDecCatalog(engine) + + assert [ + ref.id + for ref in catalog.discover(CatalogFilters(options={"book_ids": ["pa000002"]})) + ] == ["PA000002"] + assert [ref.id for ref in catalog.discover(CatalogFilters(book_ids=["1"]))] == [ + "PA000001" + ] + + +def test_discover_uses_cached_csv(tmp_path): + csv_path = tmp_path / CATALOG_FILENAME + csv_path.write_text( + _csv_text( + _row(**{"Formatted Document ID": "PA009999", "Document Title": "Cached"}) + ), + encoding="utf-8", + ) + engine = StubEngine(b"unused") + catalog = UsaidDecCatalog(engine, cache_dir=tmp_path) + + refs = list(catalog.discover(CatalogFilters())) + + assert [ref.id for ref in refs] == ["PA009999"] + assert engine.downloaded == [] + + +def test_normalize_language_of_blank_values(): + assert normalize_language(None) is None + assert normalize_language(" ") is None + + +def test_normalize_language_of_an_unmapped_name_uses_its_iso_code(): + assert normalize_language("Kinyarwanda") == "rw" + assert normalize_language("German") == "de" + assert normalize_language("Haitian Creole") == "ht" + + +def test_normalize_language_accepts_iso_codes(): + assert normalize_language("en") == "en" + assert normalize_language("eng") == "en" + assert normalize_language("fra") == "fr" + + +def test_normalize_language_rejects_unknown_labels(): + assert normalize_language("Klingon") is None + + +def test_normalize_languages_splits_bilingual_values(): + assert normalize_languages("French|English") == ("fr", "en") + assert normalize_languages("English|French") == ("en", "fr") + assert normalize_languages("English|English") == ("en",) + assert normalize_languages(None) == () + + +def test_discover_filters_by_iso_language_aliases(): + engine = StubEngine( + _zip_bytes( + _csv_text( + _row( + **{ + "Formatted Document ID": "PA000001", + "Language of Text": "French|English", + } + ), + _row( + **{ + "row_id": "2", + "Formatted Document ID": "PA000002", + "Language of Text": "French", + } + ), + ) + ) + ) + catalog = UsaidDecCatalog(engine) + + assert [ref.id for ref in catalog.discover(CatalogFilters(languages=["fra"]))] == [ + "PA000001", + "PA000002", + ] + assert [ref.id for ref in catalog.discover(CatalogFilters(languages=["en"]))] == [ + "PA000001" + ] + + +def test_split_pipe_drops_blank_items(): + assert _split_pipe(None) == [] + assert _split_pipe("Education | | Health") == ["Education", "Health"] + + +def test_extract_csv_from_zip_falls_back_to_the_only_csv(): + text = _extract_csv_from_zip( + _zip_bytes(_csv_text(_row()), name="package/AidData_reports.csv") + ) + + assert "PA000001" in text + + +def test_extract_csv_from_zip_without_a_csv_is_fatal(): + with pytest.raises(CriticalError): + _extract_csv_from_zip(_zip_bytes("no tables here", name="package/README.txt")) + + +def test_parse_catalog_skips_rows_without_an_id_or_a_title(): + entries = _parse_catalog( + _csv_text( + _row(row_id="1", **{"Formatted Document ID": ""}), + _row(row_id="2", **{"Document Title": ""}), + _row(row_id="3"), + ) + ) + + assert [entry.book_id for entry in entries] == ["PA000001"] + + +def test_parse_catalog_falls_back_to_the_institution_author(): + (entry,) = _parse_catalog( + _csv_text( + _row( + **{ + "Personal Author": "Unknown", + "Institution or USAID Bureau Author": "USAID/Kenya", + } + ) + ) + ) + + assert entry.authors == ("USAID/Kenya",) + + +def test_parse_catalog_keeps_reports_without_any_author(): + (entry,) = _parse_catalog( + _csv_text(_row(**{"Personal Author": "", "Abstract": "A summary"})) + ) + + assert entry.authors == () + assert entry.description == "A summary" + + +def test_discover_ignores_language_filters_for_explicit_ids(): + catalog = UsaidDecCatalog(StubEngine(_zip_bytes(_csv_text(_row())))) + + refs = catalog.discover( + CatalogFilters(languages=["fr"], options={"book_ids": ["PA000001"]}) + ) + + assert [ref.id for ref in refs] == ["PA000001"] + + +def test_discover_ignores_language_filters_for_explicit_positions(): + catalog = UsaidDecCatalog(StubEngine(_zip_bytes(_csv_text(_row())))) + + refs = catalog.discover(CatalogFilters(languages=["fr"], book_ids=["1"])) + + assert [ref.id for ref in refs] == ["PA000001"] + + +def test_discover_filters_reports_by_language(): + rows = _csv_text( + _row( + row_id="1", + **{"Formatted Document ID": "PA000001", "Language of Text": "English"}, + ), + _row( + row_id="2", + **{"Formatted Document ID": "PA000002", "Language of Text": "French"}, + ), + _row( + row_id="3", + **{"Formatted Document ID": "PA000003", "Language of Text": ""}, + ), + ) + catalog = UsaidDecCatalog(StubEngine(_zip_bytes(rows))) + + assert [ref.id for ref in catalog.discover(CatalogFilters(languages=["eng"]))] == [ + "PA000001", + "PA000003", + ] + assert [ref.id for ref in catalog.discover(CatalogFilters(languages=["en"]))] == [ + "PA000001", + "PA000003", + ] + assert [ref.id for ref in catalog.discover(CatalogFilters(languages=["fr"]))] == [ + "PA000002", + "PA000003", + ] + + +def test_refresh_downloads_the_dataset_and_caches_its_csv(tmp_path): + engine = StubEngine(_zip_bytes(_csv_text(_row()))) + catalog = UsaidDecCatalog(engine, cache_dir=tmp_path) + + catalog.refresh() + + assert engine.downloaded == [DEFAULT_CATALOG_URL] + assert ( + (tmp_path / CATALOG_FILENAME).read_text(encoding="utf-8").startswith("row_id") + ) + + +def test_refresh_without_a_cache_dir_is_fatal(): + catalog = UsaidDecCatalog(StubEngine(b"unused")) + + with pytest.raises(CriticalError): + catalog.refresh() + + +def test_refresh_catalog_flag_replaces_the_cached_csv(tmp_path): + csv_path = tmp_path / CATALOG_FILENAME + csv_path.write_text(_csv_text(_row(row_id="0")), encoding="utf-8") + engine = StubEngine(_zip_bytes(_csv_text(_row(row_id="1")))) + catalog = UsaidDecCatalog(engine, cache_dir=tmp_path, refresh_catalog=True) + + refs = list(catalog.discover(CatalogFilters())) + + assert [ref.id for ref in refs] == ["PA000001"] + assert engine.downloaded == [DEFAULT_CATALOG_URL] + assert csv_path.read_text(encoding="utf-8").startswith("row_id") diff --git a/scraper/tests/test_usaiddec_cli.py b/scraper/tests/test_usaiddec_cli.py new file mode 100644 index 00000000..f84c6579 --- /dev/null +++ b/scraper/tests/test_usaiddec_cli.py @@ -0,0 +1,47 @@ +"""Tests for USAID DEC Archive-specific CLI option parsing.""" + +from unittest.mock import MagicMock + +import pytest + +from papers2zim.core.utils import CriticalError +from papers2zim.sources.usaiddec.cli import handle_cli_action, parse_options + + +def test_without_options_no_ids_are_requested_and_no_refresh(): + assert parse_options({}) == {"book_ids": [], "refresh_catalog": False} + + +def test_usaid_ids_are_trimmed_and_blank_entries_dropped(): + assert parse_options({"--usaid-ids": " PA000001 , ,PA000002,"}) == { + "book_ids": ["PA000001", "PA000002"], + "refresh_catalog": False, + } + + +def test_refresh_flag_is_reported_as_an_option(): + assert parse_options({"--refresh-usaid-catalog": True}) == { + "book_ids": [], + "refresh_catalog": True, + } + + +def test_usaid_ids_and_books_cannot_be_combined(): + with pytest.raises(CriticalError): + parse_options({"--usaid-ids": "PA000001", "--books": "1"}) + + +def test_refresh_option_refreshes_the_catalog_and_ends_the_run(): + catalog = MagicMock() + + assert handle_cli_action(catalog, {"refresh_catalog": True}) is True + + catalog.refresh.assert_called_once_with() + + +def test_without_refresh_the_catalog_is_left_alone(): + catalog = MagicMock() + + assert handle_cli_action(catalog, {"refresh_catalog": False}) is False + + catalog.refresh.assert_not_called() diff --git a/scraper/tests/test_usaiddec_collections.py b/scraper/tests/test_usaiddec_collections.py new file mode 100644 index 00000000..75b3b4f3 --- /dev/null +++ b/scraper/tests/test_usaiddec_collections.py @@ -0,0 +1,80 @@ +"""Tests for the curated USAID DEC Archive collections.""" + +import pytest + +from papers2zim.sources.usaiddec.collections import ( + COLLECTION_NAMES, + OTHER_COLLECTION, + collection_for, + collection_slug, + normalize_class, +) + + +@pytest.mark.parametrize( + ("class_name", "expected"), + ( + ("HIV / AIDS", "HIV/AIDS"), + ("Maternal child health care", "Maternal & Child Health"), + ("Reproductive health care", "Reproductive Health & Family Planning"), + ("Nutrition", "Nutrition"), + ("Health (General)", "Health"), + ("Water supply and sanitation", "Water & Sanitation"), + ("Food security", "Agriculture & Food Security"), + ("Climate change", "Environment & Climate"), + ("Basic education", "Education"), + ("Energy technology", "Energy & Infrastructure"), + ( + "Information management, systems and equipment", + "Technology & Communications", + ), + ("Economic growth and development", "Economic Growth & Trade"), + ("Rule of law", "Governance & Public Administration"), + ("Human rights and social justice", "Democracy & Civil Society"), + ("Conflict prevention", "Conflict & Security"), + ("Disaster relief and response", "Humanitarian Assistance & Disaster"), + ("Gender based violence", "Gender Equality"), + ("Poverty reduction", "Poverty & Social Development"), + ("Population surveys", "Population & Demography"), + ("Performance measurement", "Capacity Building & Evaluation"), + ), +) +def test_collection_for_maps_classes_to_curated_sectors(class_name, expected): + assert collection_for(class_name) == expected + + +def test_collection_for_normalizes_top_level_paths(): + assert normalize_class("Top/Health/Diseases/Bacterial diseases/Tuberculosis") == ( + "tuberculosis" + ) + assert collection_for("Top/Health/Diseases/Bacterial diseases/Tuberculosis") == ( + "Health" + ) + + +def test_collection_for_falls_back_to_thesaurus_subjects(): + assert collection_for(None, ["Education"]) == "Education" + assert collection_for("", ["Food security"]) == "Agriculture & Food Security" + + +def test_collection_for_uses_other_when_nothing_matches(): + assert collection_for("Some unmapped label") == OTHER_COLLECTION + assert collection_for(None, ["Unknown descriptor"]) == OTHER_COLLECTION + + +def test_curated_taxonomy_stays_in_the_expected_range(): + assert 18 <= len(COLLECTION_NAMES) <= 22 + assert len(set(COLLECTION_NAMES)) == len(COLLECTION_NAMES) + + +@pytest.mark.parametrize( + ("name", "expected"), + ( + ("Education", "education"), + ("Agriculture & Food Security", "agriculture-and-food-security"), + ("HIV/AIDS", "hiv-aids"), + ("Other", "other"), + ), +) +def test_collection_slug_is_url_friendly(name, expected): + assert collection_slug(name) == expected diff --git a/scraper/tests/test_usaiddec_metadata.py b/scraper/tests/test_usaiddec_metadata.py new file mode 100644 index 00000000..453ce742 --- /dev/null +++ b/scraper/tests/test_usaiddec_metadata.py @@ -0,0 +1,116 @@ +"""Tests for USAID DEC Archive metadata conversion.""" + +from papers2zim.core.ports import WorkRef +from papers2zim.sources.usaiddec.metadata import UsaidDecMetadata + + +def test_fetch_builds_work_from_catalog_ref(): + metadata = UsaidDecMetadata() + ref = WorkRef( + id="PA00JMRT", + source="usaiddec", + extra={ + "title": "Evaluation report", + "description": "Summary", + "languages": ["en"], + "authors": ["Doe, Jane"], + "date_published": "2020-01", + "pdf_url": "https://docs.aiddata.org/ad4/pdfs/usaid-archive/PA00JMRT.pdf", + "source_url": "https://dec.usaid.gov/example", + "class_name": "Education (General)", + "subjects": ["Education"], + "geographies": ["Kenya"], + }, + ) + + (work,) = metadata.fetch([ref]) + + assert work.id == "PA00JMRT" + assert work.source == "usaiddec" + assert work.title == "Evaluation report" + assert work.description == "Summary" + assert work.languages == ["en"] + assert [creator.display_name for creator in work.creators] == ["Doe, Jane"] + assert work.creators[0].sort_name == "Doe" + assert work.creators[0].id == "doe-jane-8fa4c16c" + assert work.published == "2020-01" + assert work.formats[0].name == "pdf" + assert work.formats[0].url is not None + assert work.formats[0].url.endswith("/PA00JMRT.pdf") + assert [collection.name for collection in work.collections] == ["Education"] + assert work.collections[0].id == "education" + + +def test_fetch_skips_blank_authors_and_uses_one_curated_collection(): + ref = WorkRef( + id="PA00JMRT", + source="usaiddec", + extra={ + "title": "Evaluation report", + "authors": ["Doe, Jane", " "], + "class_name": "Basic education", + "subjects": ["Education", "education", "Health", "Agriculture"], + "geographies": ["Kenya", "Uganda"], + }, + ) + + (work,) = UsaidDecMetadata().fetch([ref]) + + assert [creator.display_name for creator in work.creators] == ["Doe, Jane"] + assert [collection.name for collection in work.collections] == ["Education"] + + +def test_author_ids_and_sort_names_are_deterministic(): + ref = WorkRef( + id="PA00JMRT", + source="usaiddec", + extra={"title": "Evaluation report", "authors": ["USAID", "Doe, Jane"]}, + ) + + first = next(UsaidDecMetadata().fetch([ref])) + second = next(UsaidDecMetadata().fetch([ref])) + + assert [creator.id for creator in first.creators] == [ + creator.id for creator in second.creators + ] + assert [creator.id for creator in first.creators] == [ + "usaid-eb1461da", + "doe-jane-8fa4c16c", + ] + assert [creator.sort_name for creator in first.creators] == ["USAID", "Doe"] + + +def test_fetch_ignores_authors_that_are_not_a_list(): + ref = WorkRef( + id="PA00JMRT", + source="usaiddec", + extra={"title": "Evaluation report", "authors": "Doe, Jane"}, + ) + + (work,) = UsaidDecMetadata().fetch([ref]) + + assert work.creators == [] + + +def test_fetch_preserves_all_report_languages(): + ref = WorkRef( + id="PA00JMRT", + source="usaiddec", + extra={"title": "Evaluation report", "languages": ["fr", "en"]}, + ) + + (work,) = UsaidDecMetadata().fetch([ref]) + + assert work.languages == ["fr", "en"] + + +def test_fetch_without_a_pdf_url_yields_a_work_with_no_formats(): + ref = WorkRef( + id="PA00JMRT", source="usaiddec", extra={"title": "Evaluation report"} + ) + + (work,) = UsaidDecMetadata().fetch([ref]) + + assert work.formats == [] + assert work.title == "Evaluation report" + assert work.languages == ["en"] diff --git a/scraper/tests/test_usaiddec_pipeline.py b/scraper/tests/test_usaiddec_pipeline.py new file mode 100644 index 00000000..6af1e968 --- /dev/null +++ b/scraper/tests/test_usaiddec_pipeline.py @@ -0,0 +1,161 @@ +"""Tests for USAID DEC Archive per-work processing.""" + +from unittest.mock import MagicMock, patch + +import requests + +from papers2zim.core.models import Format, Work +from papers2zim.core.ports import WorkRef +from papers2zim.core.progress import ScraperProgress +from papers2zim.core.work_store import WorkStore +from papers2zim.sources.usaiddec.pipeline import UsaidDecPipeline + + +def _work(**overrides) -> Work: + defaults = { + "id": "PA00JMRT", + "source": "usaiddec", + "title": "Evaluation report", + "formats": [ + Format( + name="pdf", + media_type="application/pdf", + url="https://docs.aiddata.org/ad4/pdfs/usaid-archive/PA00JMRT.pdf", + ) + ], + } + defaults.update(overrides) + return Work(**defaults) + + +def _pipeline( + work: Work, engine: MagicMock, assembler: MagicMock, formats=None, works=None +): + metadata = MagicMock() + metadata.fetch.return_value = [work] if works is None else works + return UsaidDecPipeline( + metadata=metadata, + store=WorkStore(), + assembler=assembler, + progress=ScraperProgress(None), + concurrency=1, + formats=formats or ["pdf"], + zim_name="test", + source_slug="usaiddec", + display_name="USAID DEC Archive", + engine=engine, + ) + + +def test_process_ref_downloads_pdf_and_stores_the_work(): + work = _work() + engine = MagicMock() + engine.fetch_bytes.return_value = b"%PDF-1.7 pdf bytes" + assembler = MagicMock() + pipeline = _pipeline(work, engine, assembler) + + with patch( + "papers2zim.sources.usaiddec.pipeline.extract_cover", + return_value=None, + ): + pipeline.process_ref(WorkRef(id="PA00JMRT", source="usaiddec")) + + assert pipeline.store.get("usaiddec", "PA00JMRT") is work + assert work.extra["unsupported_formats"] == [] + assert work.extra["has_cover"] is False + added_paths = [ + call.kwargs["path"] for call in assembler.add_item_for.call_args_list + ] + assert "Evaluation report.PA00JMRT.pdf" in added_paths + + +def test_process_ref_drops_report_when_pdf_is_invalid(): + work = _work() + engine = MagicMock() + engine.fetch_bytes.return_value = b"not a pdf" + assembler = MagicMock() + pipeline = _pipeline(work, engine, assembler) + + with patch( + "papers2zim.sources.usaiddec.pipeline.is_valid_book_file", + return_value=False, + ): + pipeline.process_ref(WorkRef(id="PA00JMRT", source="usaiddec")) + + assert pipeline.store.get("usaiddec", "PA00JMRT") is None + assert work.extra["unsupported_formats"] == ["pdf"] + assembler.add_item_for.assert_not_called() + + +def test_process_ref_marks_unavailable_unsupported_formats(): + work = _work(formats=[]) + engine = MagicMock() + assembler = MagicMock() + pipeline = _pipeline(work, engine, assembler, formats=["epub", "pdf"]) + + pipeline.process_ref(WorkRef(id="PA00JMRT", source="usaiddec")) + + assert sorted(work.extra["unsupported_formats"]) == ["epub", "pdf"] + assembler.add_item_for.assert_not_called() + + +def test_process_ref_without_metadata_stores_nothing(): + engine = MagicMock() + assembler = MagicMock() + pipeline = _pipeline(_work(), engine, assembler, works=[]) + + pipeline.process_ref(WorkRef(id="PA00JMRT", source="usaiddec")) + + assembler.add_item_for.assert_not_called() + assert pipeline.store.get("usaiddec", "PA00JMRT") is None + + +def test_process_ref_drops_the_report_when_the_download_fails(): + work = _work() + engine = MagicMock() + engine.fetch_bytes.side_effect = requests.ConnectionError("connection reset") + assembler = MagicMock() + pipeline = _pipeline(work, engine, assembler) + + pipeline.process_ref(WorkRef(id="PA00JMRT", source="usaiddec")) + + assert work.extra["unsupported_formats"] == ["pdf"] + assembler.add_item_for.assert_not_called() + assert pipeline.store.get("usaiddec", "PA00JMRT") is None + + +def test_process_ref_gives_up_on_a_client_error(): + work = _work() + response = requests.Response() + response.status_code = 404 + engine = MagicMock() + engine.fetch_bytes.side_effect = requests.HTTPError(response=response) + pipeline = _pipeline(work, engine, MagicMock()) + + pipeline.process_ref(WorkRef(id="PA00JMRT", source="usaiddec")) + + assert work.extra["unsupported_formats"] == ["pdf"] + assert pipeline.store.get("usaiddec", "PA00JMRT") is None + + +def test_process_ref_adds_the_extracted_cover(): + work = _work() + engine = MagicMock() + engine.fetch_bytes.return_value = b"%PDF-1.7 pdf bytes" + assembler = MagicMock() + pipeline = _pipeline(work, engine, assembler) + + with patch( + "papers2zim.sources.usaiddec.pipeline.extract_cover", + return_value=b"webp bytes", + ): + pipeline.process_ref(WorkRef(id="PA00JMRT", source="usaiddec")) + + added = { + call.kwargs["path"]: call.kwargs["content"] + for call in assembler.add_item_for.call_args_list + } + assert added["covers/PA00JMRT_cover_image.webp"] == b"webp bytes" + assert work.extra["has_cover"] is True + assert work.cover is not None + assert pipeline.store.get("usaiddec", "PA00JMRT") is work diff --git a/scraper/tests/test_usaiddec_resolver.py b/scraper/tests/test_usaiddec_resolver.py new file mode 100644 index 00000000..1c417216 --- /dev/null +++ b/scraper/tests/test_usaiddec_resolver.py @@ -0,0 +1,40 @@ +"""Tests for USAID DEC Archive format resolution.""" + +from papers2zim.core.models import Format, Work +from papers2zim.sources.usaiddec.resolver import UsaidDecFormatResolver + + +def _work(*formats: Format) -> Work: + return Work( + id="PA00JMRT", + source="usaiddec", + title="Evaluation report", + formats=list(formats), + ) + + +def test_resolve_pdf_returns_the_catalog_url(): + work = _work( + Format(name="pdf", media_type="application/pdf", url="https://a.org/r.pdf") + ) + + request = UsaidDecFormatResolver().resolve(work, "pdf") + + assert request is not None + assert request.url == "https://a.org/r.pdf" + assert request.format_name == "pdf" + assert request.extra["media_type"] == "application/pdf" + + +def test_resolve_rejects_formats_the_source_does_not_offer(): + work = _work( + Format(name="pdf", media_type="application/pdf", url="https://a.org/r.pdf") + ) + + assert UsaidDecFormatResolver().resolve(work, "epub") is None + + +def test_resolve_without_any_pdf_returns_nothing(): + work = _work(Format(name="pdf", media_type="application/pdf")) + + assert UsaidDecFormatResolver().resolve(work, "pdf") is None diff --git a/ui/src/components/common/SubjectCollectionIcon.vue b/ui/src/components/common/SubjectCollectionIcon.vue index c09cfc4f..f892e235 100644 --- a/ui/src/components/common/SubjectCollectionIcon.vue +++ b/ui/src/components/common/SubjectCollectionIcon.vue @@ -1,24 +1,42 @@