From bb656b0337a211e4903690ab37e74fa4b9055381 Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Mon, 13 Jul 2026 10:21:55 +0200 Subject: [PATCH 01/13] fix toolFixWeight: sub -> gsub --- R/toolFixWeight.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R/toolFixWeight.R b/R/toolFixWeight.R index 2bd01792..43282dcc 100644 --- a/R/toolFixWeight.R +++ b/R/toolFixWeight.R @@ -55,10 +55,10 @@ toolFixWeight <- function(weight, map, dim) { } else { # merge all subdims into one dim, and replace back at the end using extramap originalMap2 <- map[[2]] - map[[1]] <- sub(".", "p", map[[1]], fixed = TRUE) - map[[2]] <- sub(".", "p", map[[2]], fixed = TRUE) + map[[1]] <- gsub(".", "p", map[[1]], fixed = TRUE) + map[[2]] <- gsub(".", "p", map[[2]], fixed = TRUE) extramap <- stats::setNames(nm = map[[2]], originalMap2) - getItems(weight, dim, full = TRUE) <- sub(".", "p", getItems(weight, dim, full = TRUE), fixed = TRUE) + getItems(weight, dim, full = TRUE) <- gsub(".", "p", getItems(weight, dim, full = TRUE), fixed = TRUE) } dim <- dim + 0.1 stopifnot(dim %in% c(1.1, 2.1, 3.1)) From 3f9d0170c2bf55cc7cf79fcf0bf0f439e67636ed Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Mon, 13 Jul 2026 10:28:06 +0200 Subject: [PATCH 02/13] remove stats (setNames) from toolFixWeight --- R/toolFixWeight.R | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/R/toolFixWeight.R b/R/toolFixWeight.R index 43282dcc..590c6798 100644 --- a/R/toolFixWeight.R +++ b/R/toolFixWeight.R @@ -57,7 +57,8 @@ toolFixWeight <- function(weight, map, dim) { originalMap2 <- map[[2]] map[[1]] <- gsub(".", "p", map[[1]], fixed = TRUE) map[[2]] <- gsub(".", "p", map[[2]], fixed = TRUE) - extramap <- stats::setNames(nm = map[[2]], originalMap2) + extramap <- originalMap2 + names(extramap) <- map[[2]] getItems(weight, dim, full = TRUE) <- gsub(".", "p", getItems(weight, dim, full = TRUE), fixed = TRUE) } dim <- dim + 0.1 @@ -65,7 +66,9 @@ toolFixWeight <- function(weight, map, dim) { } stopifnot(dim >= 1, dim < 4) mainDim <- floor(dim) - map <- stats::setNames(nm = map[[2]], object = map[[1]]) + mapnames <- map[[2]] + map <- map[[1]] + names(map) <- mapnames # append subdim for coarse items according to map weight <- add_dimension(weight, dim + 0.1, "placeholder_dimname", From e374f7253439b2abc05ac0b227afa0eabe47dd93 Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Mon, 13 Jul 2026 10:32:46 +0200 Subject: [PATCH 03/13] adapt test to also cover fixed case --- tests/testthat/test-toolFixWeight.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testthat/test-toolFixWeight.R b/tests/testthat/test-toolFixWeight.R index 939781c6..78b08c0c 100644 --- a/tests/testthat/test-toolFixWeight.R +++ b/tests/testthat/test-toolFixWeight.R @@ -44,8 +44,8 @@ test_that("toolFixWeight works with subdims", { }) test_that("toolFixWeight works with multi subdims map", { - map <- data.frame(c("X.CC", "X.CC", "X.CC", "X.DD", "X.DD"), - c("X.CC1", "X.CC2", "X.CC3", "X.DD1", "X.DD2")) + map <- data.frame(c("X.CC.U", "X.CC.U", "X.CC.U", "X.DD.U", "X.DD.U"), + c("X.CC1.U", "X.CC2.U", "X.CC3.U", "X.DD1.U", "X.DD2.U")) weight <- new.magpie(c("A", "B"), map[[2]], c("EEE", "FFF"), fill = 0) weight["B", "CC3", "EEE"] <- 1 From c5b42a46854bfee2a828682d8fc4f65ea5c3dbf1 Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:42:06 +0200 Subject: [PATCH 04/13] fix toolMapFromRel --- R/toolAggregate.R | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/R/toolAggregate.R b/R/toolAggregate.R index f621e75a..17bc8b7d 100644 --- a/R/toolAggregate.R +++ b/R/toolAggregate.R @@ -257,7 +257,8 @@ toolZeroWeight <- function(weight, rel, from, to, dim, wdim, partrel, zeroWeight } else if (zeroWeight == "setNA") { weightSum[weightSum == 0] <- NA } else if (zeroWeight == "fix") { - weight <- toolFixWeight(weight, toolMapFromRel(rel, from, to), dim) + # will only end up here if we are disaggregating, thus coarse = from and fine = to is ok + weight <- toolFixWeight(weight, toolMapFromRel(rel, coarse = from, fine = to), dim) weightSum <- toolAggregate(weight, rel, from = from, to = to, dim = wdim, partrel = partrel, verbosity = 10) if (any(weightSum == 0, na.rm = TRUE)) { warning("Critical: toolAggregate zeroWeight = fix did not fix the weight! Please contact a madrat developer.") @@ -527,15 +528,27 @@ toolExpandRel <- function(rel, x, dim) { return(newRel[toItems, names, drop = FALSE]) } -toolMapFromRel <- function(rel, from, to) { +toolMapFromRel <- function(rel, coarse, fine) { + if (!is.character(coarse) || is.na(coarse)) { + coarse <- "coarse" + } + if (!is.character(fine) || is.na(fine)) { + fine <- "fine" + } if (is.data.frame(rel)) { - map <- rel[, c(from, to)] + map <- rel[, c(coarse, fine)] } else { map <- do.call(rbind, lapply(colnames(rel), function(i) { return(data.frame(coarse = i, fine = names(which(rel[, i] == 1)))) })) - colnames(map) <- c(from, to) + colnames(map) <- c(coarse, fine) + } + # flip coarse and fine if necessary + if (length(unique(map[[coarse]])) > length(unique(map[[fine]]))) { + map <- map[, 2:1] + names(map) <- names(map)[2:1] } + stopifnot(length(unique(map[[coarse]])) <= length(unique(map[[fine]]))) return(map) } From f93065fcc6edbb96cc7129a28334a3ec398918ff Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:43:17 +0200 Subject: [PATCH 05/13] dev version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index b72e6280..c445e4ad 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Type: Package Package: madrat Title: May All Data be Reproducible and Transparent (MADRaT) * -Version: 3.37.2 +Version: 3.37.2.9001 Date: 2026-07-02 Authors@R: c( person("Jan Philipp", "Dietrich", , "dietrich@pik-potsdam.de", role = c("aut", "cre"), From 63b14d4e8a48ee45fbd4a4ba2e2dff49da0fb7c5 Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:02:54 +0200 Subject: [PATCH 06/13] fix linter multiline import --- NAMESPACE | 15 --------------- R/calcOutput.R | 10 +++------- R/getISOlist.R | 2 +- R/getMadratInfo.R | 2 -- R/readSource.R | 2 +- R/regionscode.R | 2 +- R/toolCountry2isocode.R | 4 ++-- R/toolCountryFill.R | 6 +++--- R/toolGetMapping.R | 2 +- R/toolISOhistorical.R | 5 +++-- tests/testthat/test-toolGetMapping.R | 2 +- 11 files changed, 16 insertions(+), 36 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index a3cfa32d..0696cb1f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -59,25 +59,17 @@ import(magclass) importFrom(callr,r) importFrom(igraph,"V<-") importFrom(igraph,V) -importFrom(igraph,as_undirected) -importFrom(igraph,cluster_edge_betweenness) -importFrom(igraph,components) importFrom(igraph,ego) importFrom(igraph,graph_from_data_frame) importFrom(igraph,make_ego_graph) -importFrom(igraph,membership) importFrom(igraph,subcomponent) importFrom(igraph,union) importFrom(magclass,"getCells<-") -importFrom(magclass,"getComment<-") -importFrom(magclass,"getYears<-") importFrom(magclass,as.magpie) -importFrom(magclass,clean_magpie) importFrom(magclass,collapseNames) importFrom(magclass,dimExists) importFrom(magclass,dimSums) importFrom(magclass,getCells) -importFrom(magclass,getComment) importFrom(magclass,getItems) importFrom(magclass,getNames) importFrom(magclass,getYears) @@ -86,15 +78,11 @@ importFrom(magclass,magpie_expand) importFrom(magclass,mbind) importFrom(magclass,ncells) importFrom(magclass,new.magpie) -importFrom(magclass,nregions) -importFrom(magclass,nyears) importFrom(magclass,read.magpie) importFrom(magclass,setCells) importFrom(magclass,setYears) importFrom(magclass,time_interpolate) importFrom(magclass,where) -importFrom(magclass,write.magpie) -importFrom(magclass,write.report) importFrom(methods,formalArgs) importFrom(pkgload,is_dev_package) importFrom(renv,activate) @@ -115,10 +103,7 @@ importFrom(utils,capture.output) importFrom(utils,download.file) importFrom(utils,installed.packages) importFrom(utils,modifyList) -importFrom(utils,packageDescription) importFrom(utils,person) -importFrom(utils,read.csv) -importFrom(utils,read.csv2) importFrom(utils,sessionInfo) importFrom(utils,str) importFrom(utils,tail) diff --git a/R/calcOutput.R b/R/calcOutput.R index 3a247bf5..2575e8ff 100644 --- a/R/calcOutput.R +++ b/R/calcOutput.R @@ -104,10 +104,6 @@ #' a <- calcOutput(type = "TauTotal") #' } #' -#' @importFrom magclass nyears nregions getComment<- getComment getYears clean_magpie write.report write.magpie -#' getCells getYears<- is.magpie dimSums -#' @importFrom utils packageDescription read.csv2 read.csv -#' @importFrom withr defer local_dir #' @export calcOutput <- function(type, aggregate = TRUE, file = NULL, years = NULL, # nolint round = NULL, signif = NULL, supplementary = FALSE, @@ -205,7 +201,7 @@ calcOutput <- function(type, aggregate = TRUE, file = NULL, years = NULL, # noli if (x$isocountries) { .countrycheck <- function(datacountries, name) { datacountries <- robustSort(datacountries) - isoCountry <- read.csv2(system.file("extdata", "iso_country.csv", package = "madrat"), row.names = NULL) + isoCountry <- utils::read.csv2(system.file("extdata", "iso_country.csv", package = "madrat"), row.names = NULL) isoCountry1 <- as.vector(isoCountry[, "x"]) names(isoCountry1) <- isoCountry[, "X"] isocountries <- robustSort(isoCountry1) @@ -268,11 +264,11 @@ calcOutput <- function(type, aggregate = TRUE, file = NULL, years = NULL, # noli } startinfo <- toolstartmessage(callString, "+") - defer({ + withr::defer({ toolendmessage(startinfo, "-") }) - local_dir(getConfig("outputfolder")) + withr::local_dir(getConfig("outputfolder")) functionname <- prepFunctionName(type = type, prefix = "calc", ignore = ifelse(is.null(years), "years", NA)) extraArgs <- sapply(attr(functionname, "formals"), function(x) return(eval(parse(text = x))), simplify = FALSE) # nolint diff --git a/R/getISOlist.R b/R/getISOlist.R index 5cef4639..836a2f02 100644 --- a/R/getISOlist.R +++ b/R/getISOlist.R @@ -25,7 +25,7 @@ #' @importFrom magclass ncells #' @export getISOlist <- function(type = "all", threshold = 1) { - isoCountry <- read.csv2(system.file("extdata", "iso_country.csv", package = "madrat"), row.names = NULL) + isoCountry <- utils::read.csv2(system.file("extdata", "iso_country.csv", package = "madrat"), row.names = NULL) isoCountry1 <- as.vector(isoCountry[, "x"]) names(isoCountry1) <- isoCountry[, "X"] ref <- robustSort(isoCountry1) diff --git a/R/getMadratInfo.R b/R/getMadratInfo.R index 0da0703d..6089775a 100644 --- a/R/getMadratInfo.R +++ b/R/getMadratInfo.R @@ -15,8 +15,6 @@ #' @author Jan Philipp Dietrich #' @seealso \code{\link{getCalculations}}, \code{\link{getMadratGraph}} #' @family dependency graph -#' @importFrom igraph graph_from_data_frame components V -#' cluster_edge_betweenness as_undirected membership #' @export getMadratInfo <- function(graph = NULL, cutoff = 5, extended = FALSE, ...) { diff --git a/R/readSource.R b/R/readSource.R index bc4b8e5e..39649a20 100644 --- a/R/readSource.R +++ b/R/readSource.R @@ -112,7 +112,7 @@ readSource <- function(type, subtype = NULL, subset = NULL, # nolint: cyclocomp_ } .testISO <- function(x, functionname = "function") { - isoCountry <- read.csv2(system.file("extdata", "iso_country.csv", package = "madrat"), row.names = NULL) + isoCountry <- utils::read.csv2(system.file("extdata", "iso_country.csv", package = "madrat"), row.names = NULL) isoCountry1 <- as.vector(isoCountry[, "x"]) names(isoCountry1) <- isoCountry[, "X"] isocountries <- robustSort(isoCountry1) diff --git a/R/regionscode.R b/R/regionscode.R index 648d0beb..27cff701 100644 --- a/R/regionscode.R +++ b/R/regionscode.R @@ -49,7 +49,7 @@ regionscode <- function(mapping = NULL, label = FALSE, strict = TRUE) { if (ncol(mapping) >= 3) mapping[[1]] <- NULL # read list of ISO-countries - isCountry <- read.csv2(system.file("extdata", "iso_country.csv", package = "madrat"), row.names = NULL) + isCountry <- utils::read.csv2(system.file("extdata", "iso_country.csv", package = "madrat"), row.names = NULL) isCountry1 <- as.vector(isCountry[, "x"]) names(isCountry1) <- isCountry[, "X"] isocountries <- robustSort(isCountry1) diff --git a/R/toolCountry2isocode.R b/R/toolCountry2isocode.R index aa322486..d263a7d9 100644 --- a/R/toolCountry2isocode.R +++ b/R/toolCountry2isocode.R @@ -25,8 +25,8 @@ #' @export toolCountry2isocode <- function(country, warn = TRUE, ignoreCountries = NULL, type = NULL, mapping = NULL) { if (!is.null(type)) warning("\"type\" argument is deprecated and will be ignored! Don't use it!") - country2iso <- read.csv2(system.file("extdata", "country2iso.csv", package = "madrat"), - row.names = NULL, encoding = "UTF-8") + country2iso <- utils::read.csv2(system.file("extdata", "country2iso.csv", package = "madrat"), + row.names = NULL, encoding = "UTF-8") country2iso1 <- as.vector(country2iso[, "x"]) names(country2iso1) <- country2iso[, "X"] mapping <- c(country2iso1, mapping) diff --git a/R/toolCountryFill.R b/R/toolCountryFill.R index 21bd7349..5f921f52 100644 --- a/R/toolCountryFill.R +++ b/R/toolCountryFill.R @@ -44,7 +44,7 @@ toolCountryFill <- function(x, fill = NA, no_remove_warning = NULL, overwrite = countrylist = NULL, ...) { comment <- getComment(x) if (is.null(countrylist)) { - isoCountry <- read.csv2(system.file("extdata", "iso_country.csv", package = "madrat"), row.names = NULL) + isoCountry <- utils::read.csv2(system.file("extdata", "iso_country.csv", package = "madrat"), row.names = NULL) countrylist <- as.vector(isoCountry[, "x"]) names(countrylist) <- isoCountry[, "X"] } @@ -57,8 +57,8 @@ toolCountryFill <- function(x, fill = NA, no_remove_warning = NULL, overwrite = # warn only for countries which were not explicitly mentioned in argument "remove" countries2warn <- setdiff(additionalCountries, no_remove_warning) if (length(countries2warn) > 0) { - historicalCountries <- unique(read.csv2(system.file("extdata", "ISOhistorical.csv", package = "madrat"), - stringsAsFactors = FALSE)[["fromISO"]]) + historicalCountries <- unique(utils::read.csv2(system.file("extdata", "ISOhistorical.csv", package = "madrat"), + stringsAsFactors = FALSE)[["fromISO"]]) removedHistoricalCountries <- intersect(countries2warn, historicalCountries) historicalHint <- if (length(removedHistoricalCountries) > 0) { paste(" - By using madrat::toolISOhistorical the data for the following countries can usually still be used:", diff --git a/R/toolGetMapping.R b/R/toolGetMapping.R index b2e1be25..11237db9 100644 --- a/R/toolGetMapping.R +++ b/R/toolGetMapping.R @@ -129,7 +129,7 @@ toolGetMapping <- function(name, type = NULL, where = NULL, filetype <- tolower(file_ext(fname)) if (filetype == "csv") { sep <- if (grepl(pattern = ";", x = readLines(fname, 1))) ";" else "," - return(read.csv(fname, sep = sep, stringsAsFactors = FALSE, comment.char = "*")) + return(utils::read.csv(fname, sep = sep, stringsAsFactors = FALSE, comment.char = "*")) } else if (filetype == "rda") { data <- NULL load(fname) diff --git a/R/toolISOhistorical.R b/R/toolISOhistorical.R index 9a42d6a0..81a6de15 100644 --- a/R/toolISOhistorical.R +++ b/R/toolISOhistorical.R @@ -39,9 +39,10 @@ toolISOhistorical <- function(m, mapping = NULL, additional_mapping = NULL, over # mapping of historical countries and regions to the standard ISO-Country-List # and last year of existence of the historical countries if (is.null(mapping)) { - mapping <- read.csv2(system.file("extdata", "ISOhistorical.csv", package = "madrat"), stringsAsFactors = FALSE) + mapping <- utils::read.csv2(system.file("extdata", "ISOhistorical.csv", package = "madrat"), + stringsAsFactors = FALSE) } else if (is.character(mapping)) { - mapping <- read.csv(mapping, sep = ";", as.is = TRUE) + mapping <- utils::read.csv(mapping, sep = ";", as.is = TRUE) } # add additional mapping, if provided if (!is.null(additional_mapping)) { diff --git a/tests/testthat/test-toolGetMapping.R b/tests/testthat/test-toolGetMapping.R index c6ab6b67..42864cdb 100644 --- a/tests/testthat/test-toolGetMapping.R +++ b/tests/testthat/test-toolGetMapping.R @@ -10,7 +10,7 @@ test_that("toolGetMapping works", { grep("csv", dir(as.character(getConfig("mappingfolder"))), value = TRUE))[1] fullPath <- paste0(system.file(package = "madrat"), "/extdata/", fname) # try to find that mapping as if you did not know where it is - expect_identical(toolGetMapping(fname), read.csv(fullPath, header = TRUE, sep = ";", colClasses = "character")) + expect_identical(toolGetMapping(fname), utils::read.csv(fullPath, header = TRUE, sep = ";", colClasses = "character")) expect_identical(toolGetMapping(fullPath, type = "csv"), toolGetMapping(fname)) expect_identical(head(toolGetMapping("regionmappingH12.csv", where = "madrat")), expected) expect_identical(toolGetMapping("bla", where = "local", returnPathOnly = TRUE, error.missing = FALSE), "bla") From 2feb23745d9dfbe311bca5969e55f4a6aa514db9 Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:03:33 +0200 Subject: [PATCH 07/13] build --- .buildlibrary | 2 +- CITATION.cff | 4 ++-- DESCRIPTION | 4 ++-- README.md | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildlibrary b/.buildlibrary index 27f66234..0a9208d3 100644 --- a/.buildlibrary +++ b/.buildlibrary @@ -1,4 +1,4 @@ -ValidationKey: '69584592' +ValidationKey: '69642331' AcceptedWarnings: - 'Warning: package ''.*'' was built under R version' - 'Warning: namespace ''.*'' is not available and has been replaced' diff --git a/CITATION.cff b/CITATION.cff index 57f219e3..83fa33a6 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,8 +2,8 @@ cff-version: 1.2.0 message: If you use this software, please cite it using the metadata from this file. type: software title: 'madrat: May All Data be Reproducible and Transparent (MADRaT) *' -version: 3.37.2 -date-released: '2026-07-02' +version: 3.37.3 +date-released: '2026-07-13' abstract: Provides a framework which should improve reproducibility and transparency in data processing. It provides functionality such as automatic meta data creation and management, rudimentary quality management, data caching, work-flow management diff --git a/DESCRIPTION b/DESCRIPTION index c445e4ad..ca5c70c6 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Type: Package Package: madrat Title: May All Data be Reproducible and Transparent (MADRaT) * -Version: 3.37.2.9001 -Date: 2026-07-02 +Version: 3.37.3 +Date: 2026-07-13 Authors@R: c( person("Jan Philipp", "Dietrich", , "dietrich@pik-potsdam.de", role = c("aut", "cre"), comment = c(affiliation = "Potsdam Institute for Climate Impact Research", ORCID = "0000-0002-4309-6431")), diff --git a/README.md b/README.md index 4c6bd7b3..833e74dc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # May All Data be Reproducible and Transparent (MADRaT) * -R package **madrat**, version **3.37.2** +R package **madrat**, version **3.37.3** [![CRAN status](https://www.r-pkg.org/badges/version/madrat)](https://cran.r-project.org/package=madrat) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1115490.svg)](https://doi.org/10.5281/zenodo.1115490) [![R build status](https://github.com/pik-piam/madrat/workflows/check/badge.svg)](https://github.com/pik-piam/madrat/actions) [![codecov](https://codecov.io/gh/pik-piam/madrat/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/madrat) [![r-universe](https://pik-piam.r-universe.dev/badges/madrat)](https://pik-piam.r-universe.dev/builds) @@ -55,7 +55,7 @@ In case of questions / problems please contact Jan Philipp Dietrich , Version: 3.37.2, . +Dietrich J, Sauer P, Baumstark L, Wirth S, Giannousakis A, Rodrigues R, Bodirsky B, Leip D, Crawford M, Kreidenweis U, Klein D, Rein P (2026). "madrat: May All Data be Reproducible and Transparent (MADRaT) *." doi:10.5281/zenodo.1115490 . Version: 3.37.3, . A BibTeX entry for LaTeX users is @@ -64,9 +64,9 @@ A BibTeX entry for LaTeX users is title = {madrat: May All Data be Reproducible and Transparent (MADRaT) *}, author = {Jan Philipp Dietrich and Pascal Sauer and Lavinia Baumstark and Stephen Wirth and Anastasis Giannousakis and Renato Rodrigues and Benjamin Leon Bodirsky and Debbora Leip and Michael Crawford and Ulrich Kreidenweis and David Klein and Patrick Rein}, doi = {10.5281/zenodo.1115490}, - date = {2026-07-02}, + date = {2026-07-13}, year = {2026}, url = {https://github.com/pik-piam/madrat}, - note = {Version: 3.37.2}, + note = {Version: 3.37.3}, } ``` From 6c5ce5aa788214f884d7911927785738271ba839 Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:00:06 +0200 Subject: [PATCH 08/13] reset toolMapFromRel --- R/toolAggregate.R | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/R/toolAggregate.R b/R/toolAggregate.R index 17bc8b7d..f621e75a 100644 --- a/R/toolAggregate.R +++ b/R/toolAggregate.R @@ -257,8 +257,7 @@ toolZeroWeight <- function(weight, rel, from, to, dim, wdim, partrel, zeroWeight } else if (zeroWeight == "setNA") { weightSum[weightSum == 0] <- NA } else if (zeroWeight == "fix") { - # will only end up here if we are disaggregating, thus coarse = from and fine = to is ok - weight <- toolFixWeight(weight, toolMapFromRel(rel, coarse = from, fine = to), dim) + weight <- toolFixWeight(weight, toolMapFromRel(rel, from, to), dim) weightSum <- toolAggregate(weight, rel, from = from, to = to, dim = wdim, partrel = partrel, verbosity = 10) if (any(weightSum == 0, na.rm = TRUE)) { warning("Critical: toolAggregate zeroWeight = fix did not fix the weight! Please contact a madrat developer.") @@ -528,27 +527,15 @@ toolExpandRel <- function(rel, x, dim) { return(newRel[toItems, names, drop = FALSE]) } -toolMapFromRel <- function(rel, coarse, fine) { - if (!is.character(coarse) || is.na(coarse)) { - coarse <- "coarse" - } - if (!is.character(fine) || is.na(fine)) { - fine <- "fine" - } +toolMapFromRel <- function(rel, from, to) { if (is.data.frame(rel)) { - map <- rel[, c(coarse, fine)] + map <- rel[, c(from, to)] } else { map <- do.call(rbind, lapply(colnames(rel), function(i) { return(data.frame(coarse = i, fine = names(which(rel[, i] == 1)))) })) - colnames(map) <- c(coarse, fine) - } - # flip coarse and fine if necessary - if (length(unique(map[[coarse]])) > length(unique(map[[fine]]))) { - map <- map[, 2:1] - names(map) <- names(map)[2:1] + colnames(map) <- c(from, to) } - stopifnot(length(unique(map[[coarse]])) <= length(unique(map[[fine]]))) return(map) } From bcee56372f041a640ce9efb571fb5359e5dfeeab Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:52:57 +0200 Subject: [PATCH 09/13] fix toolFixWeight for weighted mean aggregation --- R/toolAggregate.R | 15 +++++++++------ R/toolFixWeight.R | 10 +++++++--- tests/testthat/test-toolAggregate.R | 17 ++++++++++++++++- tests/testthat/test-toolFixWeight.R | 1 - 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/R/toolAggregate.R b/R/toolAggregate.R index f621e75a..518a90ee 100644 --- a/R/toolAggregate.R +++ b/R/toolAggregate.R @@ -219,8 +219,9 @@ toolAggregateWeighted <- function(x, rel, weight, from, to, dim, wdim, partrel, } weightAndWeightSum <- toolZeroWeight(weight, rel, from, to, dim, wdim, partrel, zeroWeight) - weight <- weightAndWeightSum$weight - weight2 <- 1 / (weightAndWeightSum$weightSum + 10^-100) + weight <- weightAndWeightSum$weight # fine resolution + weight2 <- 1 / (weightAndWeightSum$weightSum + 10^-100) # coarse resolution + stopifnot(length(weight) >= length(weight2)) if (mixedAggregation) { weight2[is.na(weight2)] <- 1 @@ -233,15 +234,17 @@ toolAggregateWeighted <- function(x, rel, weight, from, to, dim, wdim, partrel, } if (setequal(getItems(weight, dim = wdim), getItems(x, dim = dim))) { + # weighted mean aggregation out <- toolAggregate(x * weight, rel, from = from, to = to, dim = dim, partrel = partrel) * weight2 } else { + # weighted disaggregation out <- toolAggregate(x * weight2, rel, from = from, to = to, dim = dim, partrel = partrel) * weight + if (zeroWeight == "fix" && !isTRUE(all.equal(sum(out), sum(x)))) { + warning("total sum is not the same after toolAggregate despite zeroWeight = 'fix', ", + "please contact a madrat developer.") + } } - if (zeroWeight == "fix" && !isTRUE(all.equal(sum(out), sum(x)))) { - warning("total sum is not the same after toolAggregate despite zeroWeight = 'fix', ", - "please contact a madrat developer.") - } getComment(out) <- c(xComment, paste0("Data aggregated (toolAggregate): ", date())) return(out) diff --git a/R/toolFixWeight.R b/R/toolFixWeight.R index 590c6798..550e5ba6 100644 --- a/R/toolFixWeight.R +++ b/R/toolFixWeight.R @@ -8,8 +8,8 @@ #' @param weight magclass object containing weights to be used for a weighted #' (dis)aggregation. The provided weight does not need to be normalized, any #' number >= 0 is allowed. -#' @param map a data frame where the first column contains coarse resolution items -#' and the second column contains fine resolution items; fine resolution items +#' @param map a 2-column data frame where one column contains coarse resolution items +#' and the other column contains fine resolution items; fine resolution items #' must match items in weight #' @param dim which dim to fix (e.g. 1 or 3.2 or "region") #' @return weight, with weights set to 10^-30 only where otherwise the total @@ -44,7 +44,11 @@ toolFixWeight <- function(weight, map, dim) { dim <- dimCode(dim, weight) stopifnot(length(dim) == 1, weight >= 0, - setequal(map[[2]], getItems(weight, dim))) + ncol(map) == 2) + if (!setequal(map[[2]], getItems(weight, dim))) { + map <- map[, 2:1] + } + stopifnot(setequal(map[[2]], getItems(weight, dim))) originalDimnames <- dimnames(weight) extramap <- NULL diff --git a/tests/testthat/test-toolAggregate.R b/tests/testthat/test-toolAggregate.R index 8c7715bb..7b6d1a47 100644 --- a/tests/testthat/test-toolAggregate.R +++ b/tests/testthat/test-toolAggregate.R @@ -260,7 +260,7 @@ test_that("trivial renaming works with weight", { expect_silent(toolAggregate(pm, mapping, from = "a", to = "b", dim = 3, weight = weight)) }) -test_that("zeroWeight = fix works", { +test_that("zeroWeight = fix works for disaggregation", { x <- new.magpie(c("A", "B"), fill = 100) rel <- data.frame(c("A", "A", "B", "B"), c("A1", "A2", "B1", "B2")) @@ -283,6 +283,21 @@ test_that("zeroWeight = fix works", { expect_true(as.vector(y["B2", , ]) == 0) # and this is also still 0 }) +test_that("zeroWeight = fix works for weighted mean aggregation", { + rel <- data.frame(c("A", "A", "B", "B"), + c("A.A1", "A.A2", "B.B1", "B.B2")) + x <- new.magpie(rel[[2]], fill = 100) + weight <- new.magpie(rel[[2]], fill = 0) + weight["B1", , ] <- 1 + + expect_warning(y <- toolAggregate(x, rel, weight), + "Weight sum is 0") + expect_true(as.vector(y["A", , ]) == 0) + + expect_silent(y <- toolAggregate(x, rel, weight, zeroWeight = "fix")) + expect_true(as.vector(y["A", , ]) == 100) +}) + test_that("empty cells in a to column do not result in aggregated data", { empty <- "" diff --git a/tests/testthat/test-toolFixWeight.R b/tests/testthat/test-toolFixWeight.R index 78b08c0c..73638fe8 100644 --- a/tests/testthat/test-toolFixWeight.R +++ b/tests/testthat/test-toolFixWeight.R @@ -58,7 +58,6 @@ test_that("toolFixWeight works with multi subdims map", { }) test_that("toolFixWeight can handle large objects", { - skip("to save time skip test whether toolFixWeight can handle large objects") to <- Reduce(x = 1:26, init = NULL, f = function(total, i) { return(c(total, paste0(LETTERS[i], seq_len(1000 * i)))) }) From fab059d3fc13405403bbad088bf7f7d71d76e06a Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:53:07 +0200 Subject: [PATCH 10/13] dev version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index ca5c70c6..9c74b321 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Type: Package Package: madrat Title: May All Data be Reproducible and Transparent (MADRaT) * -Version: 3.37.3 +Version: 3.37.3.9001 Date: 2026-07-13 Authors@R: c( person("Jan Philipp", "Dietrich", , "dietrich@pik-potsdam.de", role = c("aut", "cre"), From 9c1d1a750649959a72da1c26f8b45325db188c5f Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Wed, 15 Jul 2026 12:16:36 +0200 Subject: [PATCH 11/13] build --- .buildlibrary | 2 +- CITATION.cff | 4 ++-- DESCRIPTION | 4 ++-- R/toolAggregate.R | 9 ++++----- README.md | 8 ++++---- man/toolFixWeight.Rd | 4 ++-- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.buildlibrary b/.buildlibrary index 0a9208d3..2b4133bd 100644 --- a/.buildlibrary +++ b/.buildlibrary @@ -1,4 +1,4 @@ -ValidationKey: '69642331' +ValidationKey: '69669726' AcceptedWarnings: - 'Warning: package ''.*'' was built under R version' - 'Warning: namespace ''.*'' is not available and has been replaced' diff --git a/CITATION.cff b/CITATION.cff index 83fa33a6..7c6c5bbc 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,8 +2,8 @@ cff-version: 1.2.0 message: If you use this software, please cite it using the metadata from this file. type: software title: 'madrat: May All Data be Reproducible and Transparent (MADRaT) *' -version: 3.37.3 -date-released: '2026-07-13' +version: 3.37.4 +date-released: '2026-07-15' abstract: Provides a framework which should improve reproducibility and transparency in data processing. It provides functionality such as automatic meta data creation and management, rudimentary quality management, data caching, work-flow management diff --git a/DESCRIPTION b/DESCRIPTION index 9c74b321..c901d58b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Type: Package Package: madrat Title: May All Data be Reproducible and Transparent (MADRaT) * -Version: 3.37.3.9001 -Date: 2026-07-13 +Version: 3.37.4 +Date: 2026-07-15 Authors@R: c( person("Jan Philipp", "Dietrich", , "dietrich@pik-potsdam.de", role = c("aut", "cre"), comment = c(affiliation = "Potsdam Institute for Climate Impact Research", ORCID = "0000-0002-4309-6431")), diff --git a/R/toolAggregate.R b/R/toolAggregate.R index 518a90ee..3c4982e4 100644 --- a/R/toolAggregate.R +++ b/R/toolAggregate.R @@ -240,8 +240,8 @@ toolAggregateWeighted <- function(x, rel, weight, from, to, dim, wdim, partrel, # weighted disaggregation out <- toolAggregate(x * weight2, rel, from = from, to = to, dim = dim, partrel = partrel) * weight if (zeroWeight == "fix" && !isTRUE(all.equal(sum(out), sum(x)))) { - warning("total sum is not the same after toolAggregate despite zeroWeight = 'fix', ", - "please contact a madrat developer.") + warning("total sum is not the same after toolAggregate's weighted disaggregation ", + "despite zeroWeight = 'fix', please contact a madrat developer.") } } @@ -254,8 +254,7 @@ toolZeroWeight <- function(weight, rel, from, to, dim, wdim, partrel, zeroWeight weightSum <- toolAggregate(weight, rel, from = from, to = to, dim = wdim, partrel = partrel, verbosity = 10) if (zeroWeight != "allow" && any(weightSum == 0, na.rm = TRUE)) { if (zeroWeight == "warn") { - warning("Weight sum is 0, so cannot normalize and will return 0 for some ", - "aggregation targets. This changes the total sum of the magpie object! ", + warning("Weight sum is 0, so cannot normalize and will return 0 for some aggregation targets. ", 'If this is really intended set zeroWeight = "allow", or "setNA" to return NA.') } else if (zeroWeight == "setNA") { weightSum[weightSum == 0] <- NA @@ -266,7 +265,7 @@ toolZeroWeight <- function(weight, rel, from, to, dim, wdim, partrel, zeroWeight warning("Critical: toolAggregate zeroWeight = fix did not fix the weight! Please contact a madrat developer.") } } else { - stop("Weight sum is 0, so cannot normalize. This changes the total sum of the magpie object!") + stop("Weight sum is 0, so cannot normalize weights!") } } return(list(weight = weight, weightSum = weightSum)) diff --git a/README.md b/README.md index 833e74dc..556f29d8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # May All Data be Reproducible and Transparent (MADRaT) * -R package **madrat**, version **3.37.3** +R package **madrat**, version **3.37.4** [![CRAN status](https://www.r-pkg.org/badges/version/madrat)](https://cran.r-project.org/package=madrat) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1115490.svg)](https://doi.org/10.5281/zenodo.1115490) [![R build status](https://github.com/pik-piam/madrat/workflows/check/badge.svg)](https://github.com/pik-piam/madrat/actions) [![codecov](https://codecov.io/gh/pik-piam/madrat/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/madrat) [![r-universe](https://pik-piam.r-universe.dev/badges/madrat)](https://pik-piam.r-universe.dev/builds) @@ -55,7 +55,7 @@ In case of questions / problems please contact Jan Philipp Dietrich . Version: 3.37.3, . +Dietrich J, Sauer P, Baumstark L, Wirth S, Giannousakis A, Rodrigues R, Bodirsky B, Leip D, Crawford M, Kreidenweis U, Klein D, Rein P (2026). "madrat: May All Data be Reproducible and Transparent (MADRaT) *." doi:10.5281/zenodo.1115490 . Version: 3.37.4, . A BibTeX entry for LaTeX users is @@ -64,9 +64,9 @@ A BibTeX entry for LaTeX users is title = {madrat: May All Data be Reproducible and Transparent (MADRaT) *}, author = {Jan Philipp Dietrich and Pascal Sauer and Lavinia Baumstark and Stephen Wirth and Anastasis Giannousakis and Renato Rodrigues and Benjamin Leon Bodirsky and Debbora Leip and Michael Crawford and Ulrich Kreidenweis and David Klein and Patrick Rein}, doi = {10.5281/zenodo.1115490}, - date = {2026-07-13}, + date = {2026-07-15}, year = {2026}, url = {https://github.com/pik-piam/madrat}, - note = {Version: 3.37.3}, + note = {Version: 3.37.4}, } ``` diff --git a/man/toolFixWeight.Rd b/man/toolFixWeight.Rd index 88dc476a..b03e9f55 100644 --- a/man/toolFixWeight.Rd +++ b/man/toolFixWeight.Rd @@ -11,8 +11,8 @@ toolFixWeight(weight, map, dim) (dis)aggregation. The provided weight does not need to be normalized, any number >= 0 is allowed.} -\item{map}{a data frame where the first column contains coarse resolution items -and the second column contains fine resolution items; fine resolution items +\item{map}{a 2-column data frame where one column contains coarse resolution items +and the other column contains fine resolution items; fine resolution items must match items in weight} \item{dim}{which dim to fix (e.g. 1 or 3.2 or "region")} From 99d3277db405c1818b75be0a00bc5fd19c49c484 Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:28:05 +0200 Subject: [PATCH 12/13] dev version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index c901d58b..fd583b54 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Type: Package Package: madrat Title: May All Data be Reproducible and Transparent (MADRaT) * -Version: 3.37.4 +Version: 3.37.4.9001 Date: 2026-07-15 Authors@R: c( person("Jan Philipp", "Dietrich", , "dietrich@pik-potsdam.de", role = c("aut", "cre"), From b265994aa08bf7055c7eebb3490eab035eb5db18 Mon Sep 17 00:00:00 2001 From: Pascal Sauer <156898545+pascal-sauer@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:32:31 +0200 Subject: [PATCH 13/13] undo dev version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index fd583b54..c901d58b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Type: Package Package: madrat Title: May All Data be Reproducible and Transparent (MADRaT) * -Version: 3.37.4.9001 +Version: 3.37.4 Date: 2026-07-15 Authors@R: c( person("Jan Philipp", "Dietrich", , "dietrich@pik-potsdam.de", role = c("aut", "cre"),