read in latest IEA data

.buildlibrary

@@ -1,4 +1,4 @@
-ValidationKey: '34918422'
+ValidationKey: '34964460'
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'
 - 'Warning: namespace ''.*'' is not available and has been replaced'

.github/workflows/pkgdown.yaml

@@ -28,7 +28,7 @@ jobs:
         run: |
           options(repos = c(pikpiam = 'https://pik-piam.r-universe.dev',
                             CRAN = Sys.getenv('RSPM')))
-          pak::pak(".")
+          pak::pak(".", dependencies = 'all')
           pak::pkg_install("tidyverse/tidytemplate")
 
       - name: Build site

.lintr

@@ -1,2 +1,2 @@
-linters: lucode2::lintrRules()
+linters: lucode2::lintrRules(modification = list(pipe_consistency_linter = pipe_consistency_linter(pipe = c("auto"))))
 encoding: "UTF-8"

CITATION.cff

@@ -2,8 +2,8 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'mrcommons: MadRat commons Input Data Library'
-version: 1.69.4
-date-released: '2026-06-09'
+version: 1.69.5
+date-released: '2026-06-24'
 abstract: Provides useful functions and a common structure to all the input data required
   to run models like MAgPIE and REMIND of model input data.
 authors:

DESCRIPTION

@@ -1,8 +1,8 @@
 Type: Package
 Package: mrcommons
 Title: MadRat commons Input Data Library
-Version: 1.69.4
-Date: 2026-06-09
+Version: 1.69.5
+Date: 2026-06-24
 Authors@R: c(
     person("Benjamin Leon", "Bodirsky", , "bodirsky@pik-potsdam.de", role = "aut"),
     person("Kristine", "Karstens", role = "aut"),

NAMESPACE

@@ -36,7 +36,6 @@ import(mrfaocore)
 import(mrlandcore)
 import(mstools)
 importFrom(countrycode,countrycode)
-importFrom(data.table,":=")
 importFrom(data.table,as.data.table)
 importFrom(data.table,fread)
 importFrom(dplyr,"%>%")

R/convertIEA.R

@@ -12,16 +12,73 @@
 #' @importFrom tidyr unite
 #'
 convertIEA <- function(x, subtype) {
+  if (subtype == "EnergyBalances-2026") {
+    stop("convert = T not supported for this subtype")
+  }
+
+  if (subtype == "EnergyBalances-latest") {
+
+    x <- x["GLO", , , invert = TRUE]
+
+    # map to old product and flow names
+    # this will be removed once we completely switch to the new format and adjust
+    # all the IEA mappings accordingly
+
+    ieaShortNameMapping <- toolGetMapping("ieaShortNameMapping.csv", type = "sectoral", where = "mrcommons")
+
+    flowMap <- ieaShortNameMapping %>%
+      filter(.data$type == "FLOW") %>%
+      select(-"type")
+
+    flowMap <- rbind(
+      flowMap,
+      data.frame(
+        old_name = setdiff(getNames(x, dim = 2), flowMap$new_name),
+        new_name = setdiff(getNames(x, dim = 2), flowMap$new_name)
+      )
+    )
+
+    flowMap <- arrange(flowMap, .data$new_name)
+
+    getNames(x, dim = 2) <- flowMap[, "old_name"]
+
+    productMap <- ieaShortNameMapping %>%
+      filter(.data$type == "PRODUCT") %>%
+      select(-"type")
+
+
+    productMap <- rbind(
+      productMap,
+      data.frame(
+        old_name = setdiff(getNames(x, dim = 1), productMap$new_name),
+        new_name = setdiff(getNames(x, dim = 1), productMap$new_name)
+      )
+    )
+
+    productMap <- arrange(productMap, .data$new_name)
+
+    getNames(x, dim = 1) <- productMap[, "old_name"]
+  }
+
   if (grepl("EnergyBalances", subtype)) {
+
     # aggregate Kosovo to Serbia
     x1 <- x["KOS", , ]
-    getItems(x1, dim = 1) <- c("SRB")
+    getItems(x1, dim = 1) <- "SRB"
     x["SRB", , ] <- x["SRB", , ] + x1
-    x <- x[c("KOS"), , , invert = TRUE]
-
-    # convert electricity outputs (unit conversion between ktoe and GWh)
-    x[, , c("ELOUTPUT", "ELMAINE", "ELAUTOE", "ELMAINC", "ELAUTOC")] <- 0.0859845 *
-      x[, , c("ELOUTPUT", "ELMAINE", "ELAUTOE", "ELMAINC", "ELAUTOC")]
+    x <- x["KOS", , , invert = TRUE]
+
+    if (subtype == "EnergyBalances") {
+      # convert electricity outputs (unit conversion between ktoe and GWh, not
+      # needed in 2025 edition of data)
+      x[, , c("ELOUTPUT", "ELMAINE", "ELAUTOE", "ELMAINC", "ELAUTOC")] <- 0.0859845 *
+        x[, , c("ELOUTPUT", "ELMAINE", "ELAUTOE", "ELMAINC", "ELAUTOC")]
+    } else if (subtype == "EnergyBalances-latest") {
+      # just drop unit dimension, as all values are already in intended unit:
+      # GWh for "ELOUTPUT", "ELMAINE", "ELAUTOE", "ELMAINC", "ELAUTOC" and ktoe for
+      # everything else
+      x <- magclass::collapseDim(x, dim = 3.3)
+    }
 
     # calculate weight to be used for regional disaggregations
     wp <- calcOutput("Population", scenario = "SSP2", aggregate = FALSE)[, 2010, ]
@@ -30,29 +87,48 @@ convertIEA <- function(x, subtype) {
     wg <- wg / max(wg)
     w <- wp + wg
 
-    # disaggregating Other Africa (IAF),
-    # Other non-OECD Americas (ILA) and
-    # Other non-OECD Asia (IAS) regions to countries
+    # disaggregating regions
+    # - Other Africa (IAF)
+    # - Other non-OECD Americas (ILA)
+    # - Other non-OECD Asia (IAS)
+
     mappingfile <- toolGetMapping(
       type = "regional", name = "regionmappingIeaOther2016.csv",
       returnPathOnly = TRUE, where = "mrcommons"
     )
+
     mapping <- read.csv2(mappingfile, stringsAsFactors = TRUE) %>%
-      filter(!(!!sym("CountryCode") %in% getItems(x, dim = 1)))
-    xadd <- toolAggregate(x[levels(mapping[[2]]), , ], mapping, weight = w[as.vector(mapping[[1]]), , ])
-    x <- x[setdiff(getItems(x, dim = 1), as.vector(unique(mapping[[2]]))), , ]
+      filter(!(.data$CountryCode %in% getItems(x, dim = 1)))
+
+    xadd <- toolAggregate(
+      x = x[unique(mapping$RegionCode), , ],
+      rel = mapping,
+      weight = w[unique(mapping$CountryCode), , ]
+    )
+    x <- x[unique(mapping$RegionCode), , , invert = TRUE]
     x <- mbind(x, xadd)
 
-    # disaggregating extinct countries USSR (SUN) and Yugoslavia (YUG)
-    ISOhistorical <- read.csv2(system.file("extdata", "ISOhistorical.csv", package = "madrat"), stringsAsFactors = FALSE) # nolint
-    ISOhistorical <- ISOhistorical[!ISOhistorical$toISO == "SCG", ] # nolint
-    x <- toolISOhistorical(x,
-      mapping = ISOhistorical,
-      additional_weight = w[ISOhistorical[ISOhistorical$fromISO %in% c("YUG", "SUN"), "toISO"], , ]
+    # disaggregating extinct countries USSR (SUN) and Yugoslavia (YUG) ----
+
+    yugoslavia <- data.frame(
+      fromISO = "YUG",
+      toISO = c("SRB", "MNE", "SVN", "HRV", "MKD", "BIH"),
+      lastYear = "y1989"
+    )
+
+    sun <- toolGetMapping("ISOhistorical.csv", where = "madrat") %>%
+      filter(.data$fromISO == "SUN") %>%
+      mutate("lastYear" = "y1989")
+
+    histMap <- rbind(yugoslavia, sun)
+
+    x <- madrat::toolISOhistorical(x,
+      mapping = histMap, overwrite = TRUE,
+      additional_weight = w[unique(histMap$toISO), , ]
     )
     x[is.na(x)] <- 0
 
-    # filling missing country data
+    # filling missing country data ----
     x <- toolCountryFill(x, 0, verbosity = 2)
 
     # These changes may reduce the amount of CHP plants to below what is actually
@@ -79,9 +155,9 @@ convertIEA <- function(x, subtype) {
     x <- add_columns(x, addnm = missingFlows, dim = 3, fill = 0)
 
     d <- x[, , c("ELMAINE", "ELMAINC", "HEMAINC")]
-    tmp <- mcalc(d, ELMAINE ~ ifelse(HEMAINC > 0, ELMAINE, ELMAINC + ELMAINE), append = FALSE)
+    tmp <- magclass::mcalc(d, ELMAINE ~ ifelse(HEMAINC > 0, ELMAINE, ELMAINC + ELMAINE), append = FALSE)
     x[, , "ELMAINE"] <- tmp
-    tmp <- mcalc(d, ELMAINC ~ ifelse(HEMAINC > 0, ELMAINC, 0), append = FALSE)
+    tmp <- magclass::mcalc(d, ELMAINC ~ ifelse(HEMAINC > 0, ELMAINC, 0), append = FALSE)
     x[, , "ELMAINC"] <- tmp
 
     # for each product: check if the flow "HEAUTOC" > 0, if yes, do nothing;
@@ -100,9 +176,9 @@ convertIEA <- function(x, subtype) {
     x <- add_columns(x, addnm = missingFlows, dim = 3, fill = 0)
 
     d <- x[, , c("ELAUTOE", "HEAUTOC", "ELAUTOC")]
-    tmp <- mcalc(d, ELAUTOE ~ ifelse(HEAUTOC > 0, ELAUTOE, ELAUTOC + ELAUTOE), append = FALSE)
+    tmp <- magclass::mcalc(d, ELAUTOE ~ ifelse(HEAUTOC > 0, ELAUTOE, ELAUTOC + ELAUTOE), append = FALSE)
     x[, , "ELAUTOE"] <- tmp
-    tmp <- mcalc(d, ELAUTOC ~ ifelse(HEAUTOC > 0, ELAUTOC, 0), append = FALSE)
+    tmp <- magclass::mcalc(d, ELAUTOC ~ ifelse(HEAUTOC > 0, ELAUTOC, 0), append = FALSE)
     x[, , "ELAUTOC"] <- tmp
 
 
@@ -112,7 +188,7 @@ convertIEA <- function(x, subtype) {
     # in 2005 as there is no MARBUNK demand at all for REF regions.
 
     x["RUS", seq(1990, 2010, 1), "NONBIODIES.MARBUNK"] <-
-      x["RUS", c(1990, 2010), "NONBIODIES.MARBUNK"] |> time_interpolate(seq(1990, 2010, 1))
+      x["RUS", c(1990, 2010), "NONBIODIES.MARBUNK"] %>% time_interpolate(seq(1990, 2010, 1))
 
     # Adjust totals
     x["RUS", seq(1991, 2009, 1), "TOTAL.MARBUNK"] <-

R/readIEA.R

@@ -5,35 +5,17 @@
 #' @param subtype data subtype. Either "EnergyBalances", "EnergyBalances-latest", or "Emissions".
 #' - "EnergyBalances": IEA energy balances until 2022 (2023 incomplete), data updated in Sep 2024,
 #' the current default for REMIND input data
-#' - "EnergyBalances-latest": IEA energy balances until 2022 (2023 incomplete), data updated in Sep 2024,
-#' currently same as default
+#' - "EnergyBalances-latest": IEA energy balances until 2023 (2024 incomplete), data updated in Aug 2025,
+#' - "EnergyBalances-2026": incomplete IEA energy balances update from Apr 2026,
 #' @return magpie object of the IEA
 #' @author Anastasis Giannousakis, Lavinia Baumstark, Renato Rodrigues, Falk Benke
-#' @seealso [madrat::readSource()]
-#' @examples
-#' \dontrun{
-#' a <- readSource(type = "IEA", subtype = "EnergyBalances")
-#' }
-#'
-#' @importFrom data.table fread :=
-#' @importFrom dplyr %>%
-#' @importFrom madrat toolCountry2isocode
 #'
 readIEA <- function(subtype) {
+  if (subtype == "EnergyBalances") {
+    energyBalancesFile <- "IEA-Energy-Balances-2024/worldbig.csv"
+    incomplete <- 2023
 
-  if (grepl("EnergyBalances", subtype)) {
-
-    if (subtype == "EnergyBalances") {
-      energyBalancesFile <- "IEA-Energy-Balances-2024/worldbig.csv"
-      incomplete <- 2023
-    } else if (subtype == "EnergyBalances-latest") {
-      energyBalancesFile <- "IEA-Energy-Balances-2024/worldbig.csv"
-      incomplete <- 2023
-    } else {
-      stop("Invalid subtype!")
-    }
-
-    data <- fread(
+    data <- data.table::fread(
       file = energyBalancesFile,
       col.names = c("COUNTRY", "PRODUCT", "FLOW", "TIME", "ktoe"),
       colClasses = c("character", "character", "character", "numeric", "character"),
@@ -43,13 +25,100 @@ readIEA <- function(subtype) {
     data$COUNTRY <- toolCountry2isocode(data$COUNTRY, warn = FALSE)
 
     data <- data %>%
-      filter(!is.na(!!sym("ktoe")),
-             !is.na(!!sym("COUNTRY")),
-             !!sym("TIME") !=  incomplete) %>% # exclude latest year with incomplete data
-      mutate(!!sym("ktoe") := as.numeric(!!sym("ktoe")))
+      filter(
+        !is.na(.data$ktoe),
+        !is.na(.data$COUNTRY),
+        .data$TIME != incomplete
+      ) %>% # exclude latest year with incomplete data
+      mutate("ktoe" = as.numeric(.data$ktoe))
+
+    mdata <- as.magpie(data,
+      datacol = dim(data)[2], spatial = which(colnames(data) == "COUNTRY"),
+      temporal = which(colnames(data) == "TIME")
+    )
+  } else if (subtype == "EnergyBalances-latest") {
+    data <- NULL
+    incomplete <- 2024
+
+    for (i in seq(1, 3)) {
+      energyBalancesFile <- file.path("IEA-Energy-Balances-2025", paste0("WBIG", i, ".zip"))
+
+      tmp <- data.table::fread(
+        file = energyBalancesFile,
+        col.names = c("COUNTRY", "PRODUCT", "FLOW", "TIME", "UNIT", "VALUE"),
+        stringsAsFactors = FALSE,
+        na.strings = c("x", "..", "c", ""),
+        skip = 0,
+        showProgress = FALSE,
+        select = c(1, 2, 4, 3, 5, 6)
+      ) %>%
+        filter(
+          .data$UNIT %in% c("KTOE", "GWH"),
+          .data$TIME != incomplete # exclude latest year with incomplete data
+        ) %>%
+        mutate("VALUE" = as.numeric(.data$VALUE))
+
+
+      tmp$COUNTRY <- toolCountry2isocode(tmp$COUNTRY, warn = FALSE, mapping = c(
+        "BURKINAFASO" = "BFA",
+        "CONGO_DRC" = "COD",
+        "CONGO_REPUB" = "COG",
+        "DOMINICANREP" = "DOM",
+        "NEWZEALAND" = "NZL",
+        "SAUDIARABIA" = "SAU",
+        "SOUTHAFRICA" = "ZAF",
+        "WORLD" = "GLO",
+        "F_USSR" = "SUN",
+        "F_YUGOSLAVIA" = "YUG",
+        "OTH_NON_OECDAFR" = "IAF",
+        "OTH_NON_OECDAM" = "ILA",
+        "OTH_NON_OECDAO" = "IAS"
+      ))
 
-    mdata <- as.magpie(data, datacol = dim(data)[2], spatial = which(colnames(data) == "COUNTRY"),
-                       temporal = which(colnames(data) == "TIME"))
+      tmp <- tmp %>%
+        filter(!is.na(.data$COUNTRY), !is.na(.data$VALUE))
+
+      data <- rbind(data, tmp)
+      rm(tmp)
+    }
+
+    mdata <- as.magpie(data, spatial = 1, temporal = 4, datacol = 6)
+
+  } else if (subtype == "EnergyBalances-2026") {
+    data <- NULL
+    incomplete <- 2025
+
+    files <- list.files("IEA-Energy-Balances-2026-Apr", pattern = "*.csv", full.names = TRUE)
+
+    for (f in files) {
+      tmp <- data.table::fread(
+        file = f,
+        select = c(3, 7, 5, 11, 17, 15),
+        col.names = c("COUNTRY", "PRODUCT", "FLOW", "TIME", "UNIT", "VALUE"),
+        na.strings = c("x", "..", "c", ""),
+        skip = 0,
+        showProgress = FALSE
+      ) %>%
+        filter(
+          .data$UNIT %in% c("KTOE", "GWH"),
+          .data$TIME != incomplete # exclude latest year with incomplete data
+        ) %>%
+        mutate("VALUE" = as.numeric(.data$VALUE))
+
+      tmp$COUNTRY <- madrat::toolCountry2isocode(tmp$COUNTRY,
+        warn = TRUE,
+        mapping = c(
+          "NEWZEALAND" = "NZL",
+          "SOUTHAFRICA" = "ZAF"
+        )
+      )
+      tmp <- tmp %>%
+        filter(!is.na(.data$COUNTRY), !is.na(.data$VALUE))
+      data <- rbind(data, tmp)
+      rm(tmp)
+    }
+
+    mdata <- as.magpie(data, spatial = 1, temporal = 4, datacol = 6)
 
   } else if (subtype == "Emissions") {
     data <- read.csv("emissions2013.csv")
@@ -59,9 +128,8 @@ readIEA <- function(subtype) {
     if (names(data)[[5]] == "MtCO2") data <- data[!is.na(data$MtCO2), ]
     if (names(data)[[5]] == "MtCO2") data$MtCO2 <- suppressWarnings(as.numeric(data$MtCO2)) # nolint
     mdata <- as.magpie(data, datacol = dim(data)[2])
-
   } else {
     stop("Invalid subtype!")
   }
-  return(mdata)
+  return(magclass::magpiesort(mdata))
 }