diff --git a/DESCRIPTION b/DESCRIPTION index b8947d0..1d0c937 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: washopenresearch Title: Dataset about open research data information in Water, Sanitation, and Hygiene -Version: 0.2.0 +Version: 0.2.0.9000 Authors@R: c( person("Mian", "Zhong", , "mzhong@ethz.ch", role = c("aut", "cre"), comment = c(ORCID = "0009-0009-4546-7214")), @@ -13,8 +13,13 @@ Description: The goal of washopenresearch is to provide an overview of open rese License: CC BY 4.0 Encoding: UTF-8 Roxygen: list(markdown = TRUE) -Depends: +Depends: R (>= 3.5) +Imports: + dplyr +Suggests: + europepmc, + rcrossref LazyData: true Config/Needs/website: rmarkdown Date: 2026-07-08 diff --git a/NAMESPACE b/NAMESPACE index 6ae9268..383049b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,2 +1,3 @@ # Generated by roxygen2: do not edit by hand +export(score_fair) diff --git a/NEWS.md b/NEWS.md index 53afaf3..31adb5a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,32 @@ +# washopenresearch (development version) + +## New features + +- New scripted acquisition pipeline for a fourth dataset, `datapapers`, covering + WASH-related data papers in seven dedicated data journals (Scientific Data, + Data in Brief, Gates Open Research, F1000Research, GigaScience, GigaByte, + and Data (MDPI)) (#28). The pipeline lives in + `data-raw/01_datapapers_acquire.R` (Crossref/Europe PMC harvest with a + committed raw snapshot), `data-raw/02_datapapers_screen.R` (relevance + screening captured in a committed decision sheet keyed on DOI), and + `data-raw/03_datapapers_process.R` (harmonisation to the shared schema and + export). The dataset itself is added once the first harvest and screening + round are complete. + +## Minor improvements and fixes + +- Expired pre-signed CDN links in `washdev$supp_url` are rewritten to stable DOI + URLs, and Google Scholar alert redirects in `uncnewsletter$paper_url` are + decoded to their target URLs (#10). The 343 Silverchair links carried a + January 2024 expiry, and the high-entropy signature tokens tripped secret + scanners; the article DOI is recovered from the link path, so no re-collection + is needed. Two helpers in `data-raw/helpers.R`, `canonicalize_silverchair_url()` + and `decode_scholar_redirect()`, do the rewrites reproducibly. +- The list-column collapsing helper and shared country-cleaning steps moved to + `data-raw/helpers.R`, sourced by all processing scripts. +- `data-raw/README.md` documents the run order and provenance of every + committed snapshot and decision sheet. + # washopenresearch 0.2.0 ## New features diff --git a/R/datapapers.R b/R/datapapers.R new file mode 100644 index 0000000..6fc6559 --- /dev/null +++ b/R/datapapers.R @@ -0,0 +1,41 @@ +#' Dataset about WASH data papers published in dedicated data journals +#' +#' Candidate WASH-related data papers harvested from Crossref and Europe PMC +#' for seven data journals (Scientific Data, Data in Brief, Gates Open +#' Research, F1000Research, GigaScience, GigaByte, and Data), screened for +#' relevance with a committed decision sheet. Data papers describe a shared +#' dataset, so the repository link takes the role that the data availability +#' statement variables play in `washdev` and `uncnewsletter`. +#' +#' @format ## `datapapers` +#' +#' \describe{ +#' \item{paperid}{ID number of the paper within this dataset} +#' \item{doi}{DOI of the data paper} +#' \item{paper_url}{Official url of the paper (DOI resolver link)} +#' \item{url_source}{Publisher website of the paper} +#' \item{journal}{Full name of the journal} +#' \item{title}{Title of the paper} +#' \item{published_year}{Year of publication} +#' \item{num_authors}{Number of the authors} +#' \item{first_author_name}{Name of the first author} +#' \item{first_author_affiliation}{Academic affiliation of the first author} +#' \item{first_author_affiliation_country}{Country of the first author parsed from first_author_affiliation variable encoded with United Nations names} +#' \item{data_repo_url}{Website urls of the repository holding the dataset the paper describes, separated by "; " when there are multiple} +#' \item{data_repo}{Name of the data repository (e.g. Zenodo, Dryad, Figshare, OSF, Dataverse) parsed from data_repo_url} +#' \item{license}{License url of the paper from Crossref metadata} +#' \item{related_paper_doi}{DOI of a linked research article, if any (e.g. Data in Brief co-submissions), separated by "; " when there are multiple} +#' \item{abstract}{Abstract of the paper as provided by the metadata source} +#' \item{query_term}{WASH search term(s) that retrieved the paper, separated by "; "} +#' \item{retrieval_date}{Date the paper metadata was harvested from the API} +#' } +#' @source Crossref () and Europe PMC +#' (); see `data-raw/README.md` for the pipeline. +#' +#' @docType data +#' @keywords datasets +#' @name datapapers +# Documented via @name/NULL rather than the "datapapers" string sentinel so +# devtools::document() runs before data/datapapers.rda exists; the dataset +# lands once the issue #28 screening sheet is filled and processed. +NULL diff --git a/R/score_fair.R b/R/score_fair.R new file mode 100644 index 0000000..28c7c12 --- /dev/null +++ b/R/score_fair.R @@ -0,0 +1,127 @@ +#' Score a paper's data management against the FAIR principles +#' +#' Scores each paper on the four FAIR dimensions (Findable, Accessible, +#' Interoperable, Reusable) from the data-sharing fields the package already +#' records. Each dimension is scored 0, 1, or 2, and `fair_total` is their sum +#' (0 to 8). The rubric is deliberately simple and documented so the scoring is +#' reproducible and open to criticism (issue #19); it is a screening instrument, +#' not a certified FAIR assessment. +#' +#' @details +#' The four dimensions are scored as follows. +#' +#' \strong{Findable} (persistent identifier and registered location): +#' \itemize{ +#' \item 2: the data is in a registered repository (`das_repo_name` is set) +#' and a repository link or dataset DOI is present. +#' \item 1: a data location is stated (`das_type` is "available in online +#' repository", or a `das_repo_url` is present) but without a recognised +#' repository, or the data is in the paper or its supplement. +#' \item 0: no data location (no DAS, "on request", or "not shareable"). +#' } +#' +#' \strong{Accessible} (can a reader get the data without a barrier): +#' \itemize{ +#' \item 2: a repository or supplement link is present (`das_repo_url` or +#' `supp_url`), so the data is directly retrievable. +#' \item 1: the data is stated to be in the paper or supplement but no link is +#' recorded. +#' \item 0: "on request", "not shareable", or no DAS. +#' } +#' +#' \strong{Interoperable} (open, machine-readable shared formats), from +#' `supp_file_type`: +#' \itemize{ +#' \item 2: any open machine-readable format (csv, txt, tsv, json, xml). +#' \item 1: structured but proprietary formats only (xlsx, docx, sav, dta). +#' \item 0: unstructured only (pdf, images), or no shared files. +#' } +#' +#' \strong{Reusable} (license and repository metadata). Because a license column +#' is not yet collected for these datasets, this dimension is scored from the +#' repository signal as a lower bound: +#' \itemize{ +#' \item 2: data in a recognised repository (`das_repo_name` set), which +#' normally carries a license and rich metadata. +#' \item 1: a data location is stated but not in a recognised repository. +#' \item 0: no shared data. +#' } +#' When a `license` column is added (see issue #19), raise this dimension to use +#' it directly. +#' +#' Note that Accessible scores the shared files, not the authors' intent. A +#' paper whose `das_type` is "on request" or "not shareable" can still score 2 +#' on Accessible if it ships a supplement with a `supp_url`, because that +#' supplement is directly retrievable. The `das_type` value stays visible +#' alongside the score, so a restricted-data paper that still shares a +#' supplement is distinguishable from a fully open one. +#' +#' @param data A data frame with the columns `has_das`, `das_type`, +#' `das_repo_url`, `das_repo_name`, and `supp_file_type`. The washdev, +#' uncnewsletter, and ploswater datasets all carry these. `das_repo_name` is +#' optional; when absent, the repository signal falls back to `das_repo_url`. +#' +#' @return `data` with five integer columns added: `fair_findable`, +#' `fair_accessible`, `fair_interoperable`, `fair_reusable`, and `fair_total`. +#' +#' @examples +#' scored <- score_fair(ploswater) +#' table(scored$fair_total) +#' +#' @export +score_fair <- function(data) { + needed <- c("has_das", "das_type", "supp_file_type") + missing <- setdiff(needed, names(data)) + if (length(missing)) { + stop("score_fair() needs column(s): ", paste(missing, collapse = ", "), + call. = FALSE) + } + + das_type <- as.character(data$das_type) + repo_url <- if ("das_repo_url" %in% names(data)) data$das_repo_url else NA_character_ + repo_name <- if ("das_repo_name" %in% names(data)) data$das_repo_name else NA_character_ + supp_url <- if ("supp_url" %in% names(data)) data$supp_url else NA_character_ + supp_type <- data$supp_file_type + + has_repo_name <- !is.na(repo_name) & nzchar(repo_name) + has_repo_url <- !is.na(repo_url) & nzchar(repo_url) + has_supp_url <- !is.na(supp_url) & nzchar(supp_url) + in_repo_das <- !is.na(das_type) & das_type == "available in online repository" + in_paper_das <- !is.na(das_type) & das_type == "in paper" + barrier_das <- !is.na(das_type) & das_type %in% c("on request", "not shareable") + + findable <- dplyr::case_when( + has_repo_name & (has_repo_url | in_repo_das) ~ 2L, + in_repo_das | has_repo_url | in_paper_das ~ 1L, + TRUE ~ 0L + ) + + accessible <- dplyr::case_when( + has_repo_url | has_supp_url ~ 2L, + in_paper_das ~ 1L, + barrier_das | is.na(das_type) ~ 0L, + TRUE ~ 0L + ) + + open_fmt <- "\\b(csv|txt|tsv|json|xml)\\b" + struct_fmt <- "\\b(xlsx|xls|docx|doc|sav|dta|rds|parquet)\\b" + supp_lc <- tolower(ifelse(is.na(supp_type), "", supp_type)) + interoperable <- dplyr::case_when( + grepl(open_fmt, supp_lc) ~ 2L, + grepl(struct_fmt, supp_lc) ~ 1L, + TRUE ~ 0L + ) + + reusable <- dplyr::case_when( + has_repo_name ~ 2L, + in_repo_das | has_repo_url | in_paper_das ~ 1L, + TRUE ~ 0L + ) + + data$fair_findable <- findable + data$fair_accessible <- accessible + data$fair_interoperable <- interoperable + data$fair_reusable <- reusable + data$fair_total <- findable + accessible + interoperable + reusable + data +} diff --git a/R/uncnewsletter.R b/R/uncnewsletter.R index d7b8a0c..3f90f17 100644 --- a/R/uncnewsletter.R +++ b/R/uncnewsletter.R @@ -3,7 +3,6 @@ #' @format ## `uncnewsletter` #' #' \describe{ -#' \item{url_source}{Publisher website of the paper} #' \item{paperid}{ID number of the paper on the journal website} #' \item{issue_url}{Volume number of the journal} #' \item{paper_url}{Official website url of the paper} @@ -31,5 +30,6 @@ #' \item{das_type}{Type of the data availability statement including in paper(data in full paper scope like supplementary material or appendix or main content) on request(data available on request to the authors) available in online repository(data is shared in a public online repository) not shareable(data is not shareable). NA if it does not have a data availability statement.} #' \item{das_repo_url}{Website urls of the data if the relevant data of the paper is shared on a public repository, separated by "; " when there are multiple} #' \item{keywords}{Keywords of the paper, separated by "; "} +#' \item{doi}{DOI of the paper, backfilled via a Crossref title search (issue #20); NA where no match cleared the title-similarity threshold.} #' } "uncnewsletter" diff --git a/R/washdev.R b/R/washdev.R index 0b7ab01..36e4c8c 100644 --- a/R/washdev.R +++ b/R/washdev.R @@ -31,6 +31,6 @@ #' \item{das_repo_url}{Website urls of the data if the relevant data of the paper is shared on a public repository, separated by "; " when there are multiple} #' \item{keywords}{Keywords of the paper, separated by "; "} #' \item{url_source}{Publisher website of the paper} -#' \item{doi}{DOI of the paper. Collected since the R port of the scraper; NA for articles scraped earlier, to be backfilled via Crossref.} +#' \item{doi}{DOI of the paper. Collected by the R scraper for recent articles and backfilled via Crossref for legacy rows (issue #20); NA where no Crossref match was found.} #' } "washdev" diff --git a/README.Rmd b/README.Rmd index 8d5acec..cc2a037 100644 --- a/README.Rmd +++ b/README.Rmd @@ -38,7 +38,7 @@ library(wordcloud2) The goal of washopenresearch is to provide an overview of open research data related to Water Sanitation and Hygiene (WASH). The current version -contains three datasets from the following sources: +contains four datasets from the following sources: - `washdev`: Open access journal [*Journal of Water, Sanitation and Hygiene for Development*](https://iwaponline.com/washdev) @@ -47,6 +47,16 @@ contains three datasets from the following sources: News](https://waterinstitute.unc.edu/our-work/nc-water-news-newsletter) - `ploswater`: Open access journal [*PLOS Water*](https://journals.plos.org/water/) +- `datapapers`: WASH-related data papers in seven dedicated data + journals ([Scientific + Data](https://www.nature.com/sdata/), [Data in + Brief](https://www.sciencedirect.com/journal/data-in-brief), [Gates + Open Research](https://gatesopenresearch.org), + [F1000Research](https://f1000research.com), + [GigaScience](https://academic.oup.com/gigascience), + [GigaByte](https://gigabytejournal.com), and + [Data](https://www.mdpi.com/journal/data)), harvested from Crossref + and Europe PMC ![Word cloud of the most frequent keywords in articles of the Journal of Water, Sanitation and Hygiene for Development, with water, sanitation, and hygiene appearing largest](man/figures/washdev_wordcloud.png){width="515"} @@ -81,12 +91,12 @@ read_csv("data-raw/dictionary.csv") |> ## Data -The package provides access to three datasets `washdev`, -`uncnewsletter`, and `ploswater`. Each dataset collects information on -scientific articles about (1) article metadata (e.g. title, first -author, correspondence author), (2) supplementary material information, -(3) data availability statement, and (4) semantic information (e.g. -keywords). +The package provides access to four datasets `washdev`, +`uncnewsletter`, `ploswater`, and `datapapers`. Each dataset collects +information on scientific articles about (1) article metadata (e.g. +title, first author, correspondence author), (2) supplementary material +information, (3) data availability statement or linked data repository, +and (4) semantic information (e.g. keywords or abstract). ```{r} library(washopenresearch) @@ -174,6 +184,34 @@ readr::read_csv("data-raw/dictionary.csv") |> kableExtra::scroll_box(height = "200px") ``` +### datapapers + +The dataset `datapapers` contains WASH-related data papers published in +seven dedicated data journals, identified from Crossref and Europe PMC +metadata and screened for relevance (see `data-raw/README.md` for the +pipeline). It has `r nrow(datapapers)` observations. Because a data +paper exists to describe a shared dataset, `data_repo_url` and +`data_repo` take the role that the data availability statement variables +play in the other two datasets. + +```{r} +datapapers |> + head(3) |> + gt::gt() |> + gt::as_raw_html() +``` + +For an overview of the variable descriptions, see the following table. + +```{r echo=FALSE, message=FALSE, warning=FALSE} +readr::read_csv("data-raw/dictionary.csv") |> + dplyr::filter(file_name == "datapapers.rda") |> + dplyr::select(variable_name:description) |> + knitr::kable() |> + kableExtra::kable_styling("striped") |> + kableExtra::scroll_box(height = "200px") +``` + ## Example ### washdev @@ -261,6 +299,20 @@ uncnewsletter |> theme_classic() ``` +### datapapers + +1. How many papers per journal, and how many resolve to a data + repository? + +```{r} +datapapers |> + group_by(journal) |> + summarise(papers = n(), + with_repository_link = sum(!is.na(data_repo_url))) |> + arrange(desc(papers)) |> + knitr::kable() +``` + ## Method We describe the raw data collection procedure of each dataset in this @@ -286,6 +338,17 @@ publication's html file using the publication url. The retrieval is rule-based to find the relevant fields (e.g. supplementary materials) and extract the value. +### datapapers + +The collection of `datapapers` is fully scripted in R. Crossref is +queried by journal ISSN and Europe PMC by journal name (for the +F1000-platform journals) with a fixed list of WASH search terms; the +harvest is committed as a raw snapshot with the retrieval date and +matching query terms recorded per row. Relevance screening and country +corrections are captured in committed CSV decision sheets keyed on DOI, +so the pipeline runs end-to-end non-interactively. See +`data-raw/README.md` for the run order. + ### uncnewsletter The collection of `uncnewsletter` is a combination of web scraping and diff --git a/data-raw/01_datapapers_acquire.R b/data-raw/01_datapapers_acquire.R new file mode 100644 index 0000000..83d3e3c --- /dev/null +++ b/data-raw/01_datapapers_acquire.R @@ -0,0 +1,212 @@ +# Acquire candidate WASH data papers from data journals (issue #28). +# +# Queries Crossref (by journal ISSN) and Europe PMC (for the F1000-platform +# journals, whose versioned articles have patchy Crossref coverage) with the +# WASH term list from helpers.R, unions the results, deduplicates on DOI, and +# writes the committed raw snapshot data-raw/datapapers_raw.csv. +# +# Run non-interactively from the package root: +# Rscript data-raw/01_datapapers_acquire.R +# +# Re-running overwrites the snapshot; the git diff shows what changed since +# the last harvest. Requires network access to api.crossref.org and +# www.ebi.ac.uk (Europe PMC). + +library(dplyr) +library(purrr) +library(stringr) +library(readr) +library(tidyr) +library(rcrossref) +library(europepmc) + +source("data-raw/helpers.R") + +journals <- datapapers_journals() +search_terms <- datapapers_search_terms() +retrieval_date <- format(Sys.Date()) + +# Crossref --------------------------------------------------------------------- + +# One query per journal x term; Crossref relevance-ranks `query` matches, so +# a high `limit` with cursor paging retrieves the full match set for the ISSN. +query_crossref <- function(issn, journal, term_label, term) { + message("Crossref: ", journal, " / ", term) + res <- tryCatch( + rcrossref::cr_works( + filter = c(issn = issn, type = "journal-article"), + query = term, + cursor = "*", + cursor_max = 10000, + limit = 1000 + ), + error = function(e) { + warning("Crossref query failed for ", journal, " / ", term, ": ", + conditionMessage(e), call. = FALSE) + NULL + } + ) + works <- purrr::pluck(res, "data") + if (is.null(works) || nrow(works) == 0) { + return(NULL) + } + tibble( + doi = works$doi, + title = works$title, + journal = journal, + published_year = suppressWarnings( + as.integer(str_sub(dplyr::coalesce(works$published.print, + works$published.online, + works$issued), 1, 4)) + ), + crossref_type = works$type, + abstract = if ("abstract" %in% names(works)) works$abstract else NA_character_, + license = purrr::map_chr( + if ("license" %in% names(works)) works$license else vector("list", nrow(works)), + \(x) purrr::pluck(x, "URL", 1, .default = NA_character_) + ), + author = purrr::map( + if ("author" %in% names(works)) works$author else vector("list", nrow(works)), + identity + ), + relation = purrr::map( + if ("relation" %in% names(works)) works$relation else vector("list", nrow(works)), + identity + ), + query_term = term_label, + retrieval_date = retrieval_date + ) +} + +# Europe PMC ------------------------------------------------------------------- + +query_europepmc <- function(journal, term_label, term) { + message("Europe PMC: ", journal, " / ", term) + query <- sprintf('JOURNAL:"%s" AND (%s)', journal, term) + res <- tryCatch( + europepmc::epmc_search(query = query, limit = 10000, verbose = FALSE), + error = function(e) { + warning("Europe PMC query failed for ", journal, " / ", term, ": ", + conditionMessage(e), call. = FALSE) + NULL + } + ) + if (is.null(res) || nrow(res) == 0) { + return(NULL) + } + tibble( + doi = res$doi, + title = res$title, + journal = journal, + published_year = suppressWarnings(as.integer(res$pubYear)), + crossref_type = NA_character_, + abstract = NA_character_, + license = NA_character_, + author_string = res$authorString, + num_authors_epmc = purrr::map_int( + str_split(res$authorString, ",\\s*"), + \(x) length(x[!is.na(x) & x != ""]) + ), + query_term = term_label, + retrieval_date = retrieval_date + ) +} + +# Harvest ---------------------------------------------------------------------- + +crossref_journals <- journals |> filter(api == "crossref") +epmc_journals <- journals |> filter(api == "europepmc") + +crossref_raw <- purrr::pmap( + tidyr::expand_grid( + crossref_journals |> select(issn, journal), + tibble(term_label = names(search_terms), term = unname(search_terms)) + ), + \(issn, journal, term_label, term) query_crossref(issn, journal, term_label, term) +) |> + purrr::compact() |> + bind_rows() + +epmc_raw <- purrr::pmap( + tidyr::expand_grid( + epmc_journals |> select(journal), + tibble(term_label = names(search_terms), term = unname(search_terms)) + ), + \(journal, term_label, term) query_europepmc(journal, term_label, term) +) |> + purrr::compact() |> + bind_rows() + +# Flatten the Crossref list-columns to the snapshot schema --------------------- + +flatten_crossref <- function(data) { + if (nrow(data) == 0) { + return(data) + } + data |> + mutate( + num_authors = purrr::map_int(author, \(a) if (is.null(a)) NA_integer_ else nrow(a)), + first_author_name = purrr::map_chr(author, \(a) { + if (is.null(a) || nrow(a) == 0) return(NA_character_) + first <- if ("sequence" %in% names(a)) { + a[which(a$sequence == "first")[1], ] + } else { + a[1, ] + } + if (nrow(first) == 0 || is.na(first$family[1])) return(NA_character_) + str_squish(paste(dplyr::coalesce(first$given[1], ""), first$family[1])) + }), + first_author_affiliation = purrr::map_chr(author, \(a) { + if (is.null(a) || nrow(a) == 0 || !"affiliation.name" %in% names(a)) { + return(NA_character_) + } + first <- if ("sequence" %in% names(a)) { + a[which(a$sequence == "first")[1], ] + } else { + a[1, ] + } + purrr::pluck(first, "affiliation.name", 1, .default = NA_character_) + }), + # Crossref relation metadata links a data paper to its dataset + # (isSupplementedBy) and to a related research article (isSupplementTo). + data_repo_doi = purrr::map_chr(relation, \(r) { + ids <- purrr::pluck(r, "is-supplemented-by", "id", .default = NULL) + if (is.null(ids)) NA_character_ else paste(unique(ids), collapse = "; ") + }), + related_paper_doi = purrr::map_chr(relation, \(r) { + ids <- purrr::pluck(r, "is-supplement-to", "id", .default = NULL) + if (is.null(ids)) NA_character_ else paste(unique(ids), collapse = "; ") + }), + author = NULL, + relation = NULL + ) +} + +crossref_flat <- flatten_crossref(crossref_raw) + +epmc_flat <- epmc_raw |> + mutate( + first_author_name = str_extract(author_string, "^[^,]+"), + first_author_affiliation = NA_character_, + data_repo_doi = NA_character_, + related_paper_doi = NA_character_ + ) |> + rename(num_authors = num_authors_epmc) |> + select(-author_string) + +# Union, deduplicate on DOI, record every matching query term ------------------ + +datapapers_raw <- bind_rows(crossref_flat, epmc_flat) |> + filter(!is.na(doi)) |> + mutate(doi = str_to_lower(doi)) |> + group_by(doi) |> + summarise( + across(-query_term, \(x) dplyr::first(x[!is.na(x)], default = dplyr::first(x))), + query_term = paste(sort(unique(query_term)), collapse = "; "), + .groups = "drop" + ) |> + arrange(journal, published_year, doi) + +message("Harvested ", nrow(datapapers_raw), " unique candidate papers.") + +readr::write_csv(datapapers_raw, "data-raw/datapapers_raw.csv") diff --git a/data-raw/02_datapapers_screen.R b/data-raw/02_datapapers_screen.R new file mode 100644 index 0000000..e5d1e3c --- /dev/null +++ b/data-raw/02_datapapers_screen.R @@ -0,0 +1,88 @@ +# Screen harvested data-paper candidates for WASH relevance (issue #28). +# +# Reads data-raw/datapapers_raw.csv, computes an automatic relevance flag from +# title/abstract keyword matches, and updates the committed decision sheet +# data-raw/datapapers_screening.csv. New candidates are appended with +# include = NA; existing human decisions are never overwritten, so the manual +# screening effort is captured in git and the pipeline stays non-interactive. +# +# Run from the package root: +# Rscript data-raw/02_datapapers_screen.R +# +# A human then fills `include` (TRUE/FALSE) and `reason` for rows where +# `auto_relevant` alone is not conclusive. Decisions key on DOI, so the sheet +# can be split among contributors and merged. + +library(dplyr) +library(stringr) +library(readr) + +source("data-raw/helpers.R") + +raw_path <- "data-raw/datapapers_raw.csv" +screening_path <- "data-raw/datapapers_screening.csv" + +if (!file.exists(raw_path)) { + stop("Missing ", raw_path, ". Run data-raw/01_datapapers_acquire.R first.", + call. = FALSE) +} + +datapapers_raw <- readr::read_csv(raw_path, show_col_types = FALSE) + +# Automatic relevance: does any WASH search term appear in the title or +# abstract? Word-boundary matching keeps "wash" from matching "washer" etc. +term_pattern <- datapapers_search_terms() |> + unname() |> + str_replace_all(fixed(" AND "), ".*") |> + (\(x) paste0("\\b(", paste(x, collapse = "|"), ")\\b"))() + +candidates <- datapapers_raw |> + mutate( + auto_relevant = str_detect( + str_to_lower(paste(coalesce(title, ""), coalesce(abstract, ""))), + str_to_lower(term_pattern) + ) + ) |> + select(doi, title, journal, published_year, auto_relevant) + +# Merge with existing decisions: never overwrite a filled `include`. ---------- + +if (file.exists(screening_path)) { + existing <- readr::read_csv( + screening_path, + col_types = cols( + doi = col_character(), + title = col_character(), + journal = col_character(), + published_year = col_integer(), + auto_relevant = col_logical(), + include = col_logical(), + reason = col_character() + ) + ) +} else { + existing <- tibble( + doi = character(), title = character(), journal = character(), + published_year = integer(), auto_relevant = logical(), + include = logical(), reason = character() + ) +} + +screening <- candidates |> + left_join(existing |> select(doi, include, reason), by = "doi") |> + bind_rows(existing |> anti_join(candidates, by = "doi")) |> + arrange(journal, published_year, doi) + +readr::write_csv(screening, screening_path, na = "") + +n_pending <- sum(is.na(screening$include)) +message( + nrow(screening), " candidates in ", screening_path, "; ", + sum(screening$include %in% TRUE), " included, ", + sum(screening$include %in% FALSE), " excluded, ", + n_pending, " pending a decision." +) +if (n_pending > 0) { + message("Fill `include` (TRUE/FALSE) and `reason` for the pending rows, ", + "then commit the sheet.") +} diff --git a/data-raw/03_datapapers_process.R b/data-raw/03_datapapers_process.R new file mode 100644 index 0000000..8ad6694 --- /dev/null +++ b/data-raw/03_datapapers_process.R @@ -0,0 +1,105 @@ +# Process screened data papers into the `datapapers` dataset (issue #28). +# +# Joins the raw harvest with the committed screening decisions, harmonizes the +# columns to the washdev/uncnewsletter schema (see data-raw/dictionary.csv) +# plus data-paper-specific fields, cleans affiliation countries with the +# committed fixes sheet, and writes data/datapapers.rda and the +# inst/extdata exports. +# +# Run from the package root: +# Rscript data-raw/03_datapapers_process.R + +library(dplyr) +library(stringr) +library(readr) +library(countries) + +source("data-raw/helpers.R") + +raw_path <- "data-raw/datapapers_raw.csv" +screening_path <- "data-raw/datapapers_screening.csv" +fixes_path <- "data-raw/datapapers_country_fixes.csv" + +for (path in c(raw_path, screening_path)) { + if (!file.exists(path)) { + stop("Missing ", path, ". Run the earlier data-raw/0*_datapapers_*.R ", + "scripts first.", call. = FALSE) + } +} + +datapapers_raw <- readr::read_csv(raw_path, show_col_types = FALSE) +screening <- readr::read_csv(screening_path, show_col_types = FALSE) +country_fixes <- readr::read_csv( + fixes_path, + col_types = cols(doi = col_character(), + first_author_affiliation_country = col_character()) +) + +n_pending <- sum(is.na(screening$include)) +if (n_pending > 0) { + message(n_pending, " screening decisions still pending; those papers are ", + "excluded from this build.") +} + +# Keep included papers only, harmonized to the shared schema ------------------- + +journals <- datapapers_journals() + +datapapers <- datapapers_raw |> + inner_join(screening |> filter(include %in% TRUE) |> select(doi), + by = "doi") |> + left_join(journals |> select(journal, url_source), by = "journal") |> + mutate( + paper_url = paste0("https://doi.org/", doi), + # Data papers exist to describe a shared dataset, so the linked + # repository DOI/URL plays the role of das_repo_url in the other + # datasets. Rows without relation metadata get their repository link at + # screening or in issue #27's download step. + data_repo_url = if_else( + !is.na(data_repo_doi) & !str_detect(data_repo_doi, "^https?://"), + str_replace_all(data_repo_doi, "(^|; )(10\\.)", "\\1https://doi.org/\\2"), + data_repo_doi + ), + data_repo = parse_repo_name(data_repo_url) + ) |> + # Clean affiliation countries: automatic standardisation, then the committed + # fixes sheet for the residual NAs (no hard-coded ID vectors). + mutate( + first_author_affiliation_country = str_extract( + first_author_affiliation, "[^,]+$") |> str_squish() |> to_un_country_name() + ) |> + apply_country_fixes(country_fixes, key = "doi", + value_col = "first_author_affiliation_country") |> + arrange(journal, published_year, doi) |> + mutate(paperid = row_number()) |> + select( + paperid, doi, paper_url, url_source, journal, title, published_year, + num_authors, first_author_name, first_author_affiliation, + first_author_affiliation_country, + data_repo_url, data_repo, license, related_paper_doi, + abstract, query_term, retrieval_date + ) |> + mutate( + across(c(paperid, published_year, num_authors), as.integer) + ) + +# Report residual NA countries so the fixes sheet can be extended -------------- + +residual <- datapapers |> + filter(is.na(first_author_affiliation_country), + !is.na(first_author_affiliation)) |> + select(doi, first_author_affiliation) +if (nrow(residual) > 0) { + message(nrow(residual), " papers have an affiliation but no standardised ", + "country. Add rows for them to ", fixes_path, ":") + print(residual, n = nrow(residual)) +} + +stopifnot(!any(purrr::map_lgl(datapapers, is.list))) + +# Write to R data object and flat-file exports --------------------------------- + +usethis::use_data(datapapers, overwrite = TRUE) + +readr::write_csv(datapapers, here::here("inst", "extdata", "datapapers.csv")) +openxlsx::write.xlsx(datapapers, here::here("inst", "extdata", "datapapers.xlsx")) diff --git a/data-raw/README.md b/data-raw/README.md new file mode 100644 index 0000000..8faedf7 --- /dev/null +++ b/data-raw/README.md @@ -0,0 +1,65 @@ +# data-raw: pipeline and provenance + +This directory contains everything needed to rebuild the package datasets +from their sources. All scripts are run **from the package root** and are +non-interactive: + +``` sh +Rscript data-raw/