From ff1714c488b581aaefd514a7fe2f6602ae77a3a6 Mon Sep 17 00:00:00 2001 From: J Wokaty Date: Wed, 5 Feb 2025 16:33:53 -0500 Subject: [PATCH 01/16] Generate html manuals with OUTGOING, copy html manuals for 3.21 --- BBS-make-OUTGOING.py | 16 ++++++++++++++++ propagation/3.21/updateReposPkgs-bioc.sh | 7 +++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/BBS-make-OUTGOING.py b/BBS-make-OUTGOING.py index 8bad53a7..4d47ad4b 100755 --- a/BBS-make-OUTGOING.py +++ b/BBS-make-OUTGOING.py @@ -11,6 +11,7 @@ import os import time import shutil +import subprocess import bbs.fileutils import bbs.parse @@ -120,6 +121,7 @@ def copy_outgoing_pkgs(products_in_subdir, source_node): if BBSvars.buildtype in ['workflows', 'books', 'bioc-mac-arm64']: pass elif source_node: + # pdf manuals pdf_file = os.path.join(BBSvars.products_in_rdir.path, BBSutils.getSourceNode(), 'checksrc', @@ -132,6 +134,20 @@ def copy_outgoing_pkgs(products_in_subdir, source_node): os.link(pdf_file, dst) # create hard link to avoid making a copy else: print("BBS> [stage6b] SKIPPED (file %s doesn't exist)" % pdf_file) + # html manuals + rel_link = "../../%s/man/%s.html" + hooks = f"hooks <- list(pkg_href = function(pkg) sprintf('{rel_link}', pkg, pkg));" + html_file = os.path.join(BBSvars.Central_rdir.path, + "OUTGOING/manuals", + '%s.html' % pkg) + Rexpr = hooks + f"tools::pkg2HTML('{pkg_path}',out='{html_file}',hooks=hooks)" + cmd = BBSbase.Rexpr2syscmd(Rexpr) + try: + print(Rexpr) + subprocess.run(cmd, stdout=None, stderr=subprocess.STDOUT, + shell=True, check=True) + except subprocess.CalledProcessError as e: + print("BBS> [stage6b] SKIPPED (could not generate %s)" % html_file) print('BBS> [stage6b] END copying outgoing packages from %s.' % srcdir) return diff --git a/propagation/3.21/updateReposPkgs-bioc.sh b/propagation/3.21/updateReposPkgs-bioc.sh index 63648e99..b7a10838 100755 --- a/propagation/3.21/updateReposPkgs-bioc.sh +++ b/propagation/3.21/updateReposPkgs-bioc.sh @@ -67,10 +67,13 @@ echo "" MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/bioc repo with reference manuals..." -for i in `ls $MANUALS_SRC`; do +for i in `ls $MANUALS_SRC/*.pdf`; do pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man - cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man + cp --update --verbose $MANUALS_SRC/$i.pdf $MANUALS_DEST/$pkg/man + if [ -f $i.html ]; then + cp --update --verbose $MANUALS_SRC/$i.html $MANUALS_DEST/$pkg/man + fi done echo "DONE." From 3ed07a2431f7f1f75dba13a19e7bdcf6ede11b49 Mon Sep 17 00:00:00 2001 From: J Wokaty Date: Wed, 5 Feb 2025 21:24:42 -0500 Subject: [PATCH 02/16] Fix copying manuals --- propagation/3.21/updateReposPkgs-bioc.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/propagation/3.21/updateReposPkgs-bioc.sh b/propagation/3.21/updateReposPkgs-bioc.sh index b7a10838..f275f7a3 100755 --- a/propagation/3.21/updateReposPkgs-bioc.sh +++ b/propagation/3.21/updateReposPkgs-bioc.sh @@ -67,13 +67,10 @@ echo "" MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/bioc repo with reference manuals..." -for i in `ls $MANUALS_SRC/*.pdf`; do - pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` +for i in `ls $MANUALS_SRC`; do + pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man - cp --update --verbose $MANUALS_SRC/$i.pdf $MANUALS_DEST/$pkg/man - if [ -f $i.html ]; then - cp --update --verbose $MANUALS_SRC/$i.html $MANUALS_DEST/$pkg/man - fi + cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done echo "DONE." From ddaa3ae74670ff3ab54a808eae3a6fe49b7b72d7 Mon Sep 17 00:00:00 2001 From: J Wokaty Date: Wed, 5 Feb 2025 21:29:50 -0500 Subject: [PATCH 03/16] Copy 3.21 html manuals for annotations, experiments --- propagation/3.21/updateReposPkgs-data-annotation.sh | 2 +- propagation/3.21/updateReposPkgs-data-experiment.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/propagation/3.21/updateReposPkgs-data-annotation.sh b/propagation/3.21/updateReposPkgs-data-annotation.sh index 6a3fffdf..8746393d 100755 --- a/propagation/3.21/updateReposPkgs-data-annotation.sh +++ b/propagation/3.21/updateReposPkgs-data-annotation.sh @@ -54,7 +54,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/data/annotation repo with reference manuals..." for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` + pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done diff --git a/propagation/3.21/updateReposPkgs-data-experiment.sh b/propagation/3.21/updateReposPkgs-data-experiment.sh index 8693ff9f..bd90840a 100755 --- a/propagation/3.21/updateReposPkgs-data-experiment.sh +++ b/propagation/3.21/updateReposPkgs-data-experiment.sh @@ -54,7 +54,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/data/experiment repo with reference manuals..." for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` + pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done From 545bd2b9a3b94b9dcadb7c7a179612277d767dea Mon Sep 17 00:00:00 2001 From: J Wokaty Date: Fri, 7 Feb 2025 12:58:06 -0500 Subject: [PATCH 04/16] Make Meta dbs --- 3.20/bioc/bbscentral2/postrun.sh | 1 + 3.21/bioc/bbscentral1/postrun.sh | 1 + BBS-make-meta-dbs.py | 55 ++++++++++++++++++++++++++++++++ utils/makeMetaDbs.R | 29 +++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100755 BBS-make-meta-dbs.py create mode 100644 utils/makeMetaDbs.R diff --git a/3.20/bioc/bbscentral2/postrun.sh b/3.20/bioc/bbscentral2/postrun.sh index 66e8ecd6..3a19ecb9 100755 --- a/3.20/bioc/bbscentral2/postrun.sh +++ b/3.20/bioc/bbscentral2/postrun.sh @@ -32,6 +32,7 @@ if [ -n "$BBS_OUTGOING_MAP" ]; then find OUTGOING -type d -exec chmod 755 {} \; $BBS_PYTHON_CMD $BBS_HOME/BBS-make-PROPAGATION_STATUS_DB.py chmod -R +r . + $BBS_PYTHON_CMD $BBS_HOME/BBS-make-meta.py fi # Generate the HTML report diff --git a/3.21/bioc/bbscentral1/postrun.sh b/3.21/bioc/bbscentral1/postrun.sh index 66e8ecd6..3a19ecb9 100755 --- a/3.21/bioc/bbscentral1/postrun.sh +++ b/3.21/bioc/bbscentral1/postrun.sh @@ -32,6 +32,7 @@ if [ -n "$BBS_OUTGOING_MAP" ]; then find OUTGOING -type d -exec chmod 755 {} \; $BBS_PYTHON_CMD $BBS_HOME/BBS-make-PROPAGATION_STATUS_DB.py chmod -R +r . + $BBS_PYTHON_CMD $BBS_HOME/BBS-make-meta.py fi # Generate the HTML report diff --git a/BBS-make-meta-dbs.py b/BBS-make-meta-dbs.py new file mode 100755 index 00000000..0c6c0500 --- /dev/null +++ b/BBS-make-meta-dbs.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import sys +import os +import time +import subprocess + +import BBSutils +import BBSvars +import BBSbase + +def make_meta(): + ## Prepare Rexpr (must be a single string with no spaces). + Rscript_path = os.path.join(BBSvars.BBS_home, + 'utils', + 'generateMetaDbs.R') + Rfun = 'generateMetaDbs' + OUTGOING_dir = 'OUTGOING' + db_filepath = 'PROPAGATION_STATUS_DB.txt' + Rfuncall = f"{Rfun}('{OUTGOING_dir}','{db_filepath}')" + Rexpr = f"source('{Rscript_path}');{Rfuncall}" + + ## Turn Rexpr into a system command. + cmd = BBSbase.Rexpr2syscmd(Rexpr) + + try: + ## Nasty things (that I don't really understand) can happen with + ## subprocess.run() if this code is runned by the Task Scheduler + ## on Windows (the child process tends to almost always return an + ## error). Apparently using 'stderr=subprocess.STDOUT' fixes this pb. + subprocess.run(cmd, stdout=None, stderr=subprocess.STDOUT, shell=True, + check=True) + except subprocess.CalledProcessError as e: + print(f'BBS> [generateMetaDbs] Failed at {time.asctime()} . . .') + print(f'BBS> [generateMetaDbs] Error: {e} . . .') + return + + +############################################################################## +### MAIN SECTION +############################################################################## + +if __name__ == "__main__": + if not os.path.isfile('PROPAGATION_STATUS_DB.txt'): + msg = ( + f'Make sure to be in {BBSvars.Central_rdir.path} and run ' + f'BBS-make-PROPAGATION_STATUS_DB.py before BBS-make-meta.py.' + ) + sys.exit('=> EXIT.') + print('BBS> ==============================================================') + print(f'BBS> [generateMetaDbs] STARTING on {time.asctime()} ...') + sys.stdout.flush() + generate_meta_dbs() + print(f'BBS> [generateMetaDbs] DONE on {time.asctime()} ...') + diff --git a/utils/makeMetaDbs.R b/utils/makeMetaDbs.R new file mode 100644 index 00000000..f10c50c5 --- /dev/null +++ b/utils/makeMetaDbs.R @@ -0,0 +1,29 @@ +generateMetaDbs <- function(outgoing_path, db_filepath) { + + meat_dir <- Sys.getenv("BBS_MEAT_PATH") + bbs_central_rdir <- Sys.getenv("BBS_CENTRAL_RDIR") + prop_status <- read.dcf(file.path(bbs_central_rdir, db_filepath)) + + pkgs <- c() + for (i in 1:dim(prop_status)[2]) { + pkg_type_stage <- strsplit(colnames(prop_status)[i], "#")[[1]] + if ("source" %in% pkg_type_stage && + strsplit(prop_status[i], ",")[[1]][1] %in% c("YES", "UNNEEDED")) { + pkgs <- c(pkgs, pkg_type_stage[1]) + } + } + + # Because the location is always recreated, we always create the Meta + # directory and always create all the databases + web_dir <- file.path(bbs_central_rdir, "web", "packages") + meta_dir <- file.path(bbs_central_rdir, "OUTGOING", "Meta") + dir.create(meta_dir, recursive = TRUE) + + biocViews::build_db_from_source(file.path(meat_dir, pkgs), bbs_central_rdir) + + aliases_db_file <- file.path(meta_dir, "aliases.rds") + biocViews::build_meta_aliases_db(web_dir, aliases_db_file, TRUE) + + rdxrefs_db_file <- file.path(meta_dir, "rdxrefs.rds") + biocViews::build_meta_rdxrefs_db(web_dir, rdxrefs_db_file, TRUE) +} From f9a37302ea8621ccef663e9adef9f1cee5bcf315 Mon Sep 17 00:00:00 2001 From: J Wokaty Date: Fri, 7 Feb 2025 15:55:44 -0500 Subject: [PATCH 05/16] Fix names for makeMetaDbs --- 3.20/bioc/bbscentral2/postrun.sh | 2 +- 3.21/bioc/bbscentral1/postrun.sh | 2 +- BBS-make-meta-dbs.py | 12 ++++++------ utils/makeMetaDbs.R | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/3.20/bioc/bbscentral2/postrun.sh b/3.20/bioc/bbscentral2/postrun.sh index 3a19ecb9..5072f3b4 100755 --- a/3.20/bioc/bbscentral2/postrun.sh +++ b/3.20/bioc/bbscentral2/postrun.sh @@ -32,7 +32,7 @@ if [ -n "$BBS_OUTGOING_MAP" ]; then find OUTGOING -type d -exec chmod 755 {} \; $BBS_PYTHON_CMD $BBS_HOME/BBS-make-PROPAGATION_STATUS_DB.py chmod -R +r . - $BBS_PYTHON_CMD $BBS_HOME/BBS-make-meta.py + $BBS_PYTHON_CMD $BBS_HOME/BBS-make-meta-dbs.py fi # Generate the HTML report diff --git a/3.21/bioc/bbscentral1/postrun.sh b/3.21/bioc/bbscentral1/postrun.sh index 3a19ecb9..5072f3b4 100755 --- a/3.21/bioc/bbscentral1/postrun.sh +++ b/3.21/bioc/bbscentral1/postrun.sh @@ -32,7 +32,7 @@ if [ -n "$BBS_OUTGOING_MAP" ]; then find OUTGOING -type d -exec chmod 755 {} \; $BBS_PYTHON_CMD $BBS_HOME/BBS-make-PROPAGATION_STATUS_DB.py chmod -R +r . - $BBS_PYTHON_CMD $BBS_HOME/BBS-make-meta.py + $BBS_PYTHON_CMD $BBS_HOME/BBS-make-meta-dbs.py fi # Generate the HTML report diff --git a/BBS-make-meta-dbs.py b/BBS-make-meta-dbs.py index 0c6c0500..39e9ee23 100755 --- a/BBS-make-meta-dbs.py +++ b/BBS-make-meta-dbs.py @@ -9,12 +9,12 @@ import BBSvars import BBSbase -def make_meta(): +def make_meta_dbs(): ## Prepare Rexpr (must be a single string with no spaces). Rscript_path = os.path.join(BBSvars.BBS_home, 'utils', - 'generateMetaDbs.R') - Rfun = 'generateMetaDbs' + 'makeMetaDbs.R') + Rfun = 'makeMetaDbs' OUTGOING_dir = 'OUTGOING' db_filepath = 'PROPAGATION_STATUS_DB.txt' Rfuncall = f"{Rfun}('{OUTGOING_dir}','{db_filepath}')" @@ -48,8 +48,8 @@ def make_meta(): ) sys.exit('=> EXIT.') print('BBS> ==============================================================') - print(f'BBS> [generateMetaDbs] STARTING on {time.asctime()} ...') + print(f'BBS> [makeMetaDbs] STARTING on {time.asctime()} ...') sys.stdout.flush() - generate_meta_dbs() - print(f'BBS> [generateMetaDbs] DONE on {time.asctime()} ...') + make_meta_dbs() + print(f'BBS> [makeMetaDbs] DONE on {time.asctime()} ...') diff --git a/utils/makeMetaDbs.R b/utils/makeMetaDbs.R index f10c50c5..9da96081 100644 --- a/utils/makeMetaDbs.R +++ b/utils/makeMetaDbs.R @@ -1,4 +1,4 @@ -generateMetaDbs <- function(outgoing_path, db_filepath) { +makeMetaDbs <- function(outgoing_path, db_filepath) { meat_dir <- Sys.getenv("BBS_MEAT_PATH") bbs_central_rdir <- Sys.getenv("BBS_CENTRAL_RDIR") From b2ec3ce6ddd9af8eae4031c63d8ee2fa566f3974 Mon Sep 17 00:00:00 2001 From: J Wokaty Date: Fri, 7 Feb 2025 17:07:21 -0500 Subject: [PATCH 06/16] Fix makeMetaDbs.R --- utils/makeMetaDbs.R | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/utils/makeMetaDbs.R b/utils/makeMetaDbs.R index 9da96081..9f16ee4f 100644 --- a/utils/makeMetaDbs.R +++ b/utils/makeMetaDbs.R @@ -17,13 +17,22 @@ makeMetaDbs <- function(outgoing_path, db_filepath) { # directory and always create all the databases web_dir <- file.path(bbs_central_rdir, "web", "packages") meta_dir <- file.path(bbs_central_rdir, "OUTGOING", "Meta") - dir.create(meta_dir, recursive = TRUE) + if (!dir.exists(meta_dir)) { + dir.create(meta_dir, recursive = TRUE) + } - biocViews::build_db_from_source(file.path(meat_dir, pkgs), bbs_central_rdir) + pkg_paths <- file.path(meat_dir, pkgs) + for (pkg_path in pkg_paths) { + biocViews::build_db_from_source(pkg_path, bbs_central_rdir) + } aliases_db_file <- file.path(meta_dir, "aliases.rds") - biocViews::build_meta_aliases_db(web_dir, aliases_db_file, TRUE) + meta_aliases_db <- biocViews::build_meta_aliases_db(web_dir, aliases_db_file, + TRUE) + saveRDS(meta_aliases_db, aliases_db_file, version = 2) rdxrefs_db_file <- file.path(meta_dir, "rdxrefs.rds") - biocViews::build_meta_rdxrefs_db(web_dir, rdxrefs_db_file, TRUE) + meta_rdxrefs_db <- biocViews::build_meta_rdxrefs_db(web_dir, rdxrefs_db_file, + TRUE) + saveRDS(meta_rdxrefs_db, rdxrefs_db_file, version = 2) } From b77a6bb736077d1d6edc32d04b1bc33da8c12eee Mon Sep 17 00:00:00 2001 From: J Wokaty Date: Sat, 8 Feb 2025 00:24:56 -0500 Subject: [PATCH 07/16] Copy Meta to the source repository --- propagation/3.20/updateReposPkgs-bioc.sh | 4 ++++ propagation/3.21/updateReposPkgs-bioc.sh | 4 ++++ utils/makeMetaDbs.R | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/propagation/3.20/updateReposPkgs-bioc.sh b/propagation/3.20/updateReposPkgs-bioc.sh index e7c410e8..b2e84179 100755 --- a/propagation/3.20/updateReposPkgs-bioc.sh +++ b/propagation/3.20/updateReposPkgs-bioc.sh @@ -73,5 +73,9 @@ for i in `ls $MANUALS_SRC`; do cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done +echo "Updating $SRC_CONTRIB with aliases and cross references dbs ..." +META_SRC="$BBS_OUTGOING_DIR/Meta" +cp --recursive --update --verbose $META_SRC $SRC_CONTRIB + echo "DONE." exit 0 diff --git a/propagation/3.21/updateReposPkgs-bioc.sh b/propagation/3.21/updateReposPkgs-bioc.sh index f275f7a3..a113d231 100755 --- a/propagation/3.21/updateReposPkgs-bioc.sh +++ b/propagation/3.21/updateReposPkgs-bioc.sh @@ -73,5 +73,9 @@ for i in `ls $MANUALS_SRC`; do cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done +echo "Updating $SRC_CONTRIB with aliases and cross references dbs ..." +META_SRC="$BBS_OUTGOING_DIR/Meta" +cp --recursive --update --verbose $META_SRC $SRC_CONTRIB + echo "DONE." exit 0 diff --git a/utils/makeMetaDbs.R b/utils/makeMetaDbs.R index 9f16ee4f..e42208e5 100644 --- a/utils/makeMetaDbs.R +++ b/utils/makeMetaDbs.R @@ -7,7 +7,7 @@ makeMetaDbs <- function(outgoing_path, db_filepath) { pkgs <- c() for (i in 1:dim(prop_status)[2]) { pkg_type_stage <- strsplit(colnames(prop_status)[i], "#")[[1]] - if ("source" %in% pkg_type_stage && + if ("source" %in% pkg_type_stage && strsplit(prop_status[i], ",")[[1]][1] %in% c("YES", "UNNEEDED")) { pkgs <- c(pkgs, pkg_type_stage[1]) } From c3011b405cae948d11adde91c6cef2238092b491 Mon Sep 17 00:00:00 2001 From: J Wokaty Date: Mon, 10 Feb 2025 12:51:57 -0500 Subject: [PATCH 08/16] Add comments, copy html manuals for 3.20 --- BBS-make-meta-dbs.py | 7 +------ propagation/3.20/updateReposPkgs-data-annotation.sh | 2 +- propagation/3.20/updateReposPkgs-data-experiment.sh | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/BBS-make-meta-dbs.py b/BBS-make-meta-dbs.py index 39e9ee23..59fa7062 100755 --- a/BBS-make-meta-dbs.py +++ b/BBS-make-meta-dbs.py @@ -5,11 +5,11 @@ import time import subprocess -import BBSutils import BBSvars import BBSbase def make_meta_dbs(): + ## Following example of BBS-make-OUTGOING.py ## Prepare Rexpr (must be a single string with no spaces). Rscript_path = os.path.join(BBSvars.BBS_home, 'utils', @@ -20,14 +20,9 @@ def make_meta_dbs(): Rfuncall = f"{Rfun}('{OUTGOING_dir}','{db_filepath}')" Rexpr = f"source('{Rscript_path}');{Rfuncall}" - ## Turn Rexpr into a system command. cmd = BBSbase.Rexpr2syscmd(Rexpr) try: - ## Nasty things (that I don't really understand) can happen with - ## subprocess.run() if this code is runned by the Task Scheduler - ## on Windows (the child process tends to almost always return an - ## error). Apparently using 'stderr=subprocess.STDOUT' fixes this pb. subprocess.run(cmd, stdout=None, stderr=subprocess.STDOUT, shell=True, check=True) except subprocess.CalledProcessError as e: diff --git a/propagation/3.20/updateReposPkgs-data-annotation.sh b/propagation/3.20/updateReposPkgs-data-annotation.sh index 7c25cc47..d00505aa 100755 --- a/propagation/3.20/updateReposPkgs-data-annotation.sh +++ b/propagation/3.20/updateReposPkgs-data-annotation.sh @@ -54,7 +54,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/data/annotation repo with reference manuals..." for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` + pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done diff --git a/propagation/3.20/updateReposPkgs-data-experiment.sh b/propagation/3.20/updateReposPkgs-data-experiment.sh index 2fd83f2e..cb69c899 100755 --- a/propagation/3.20/updateReposPkgs-data-experiment.sh +++ b/propagation/3.20/updateReposPkgs-data-experiment.sh @@ -54,7 +54,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/data/experiment repo with reference manuals..." for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` + pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done From 61e09d5c64c4e662a31d37304970d32cac202e0e Mon Sep 17 00:00:00 2001 From: J Wokaty Date: Mon, 10 Feb 2025 15:23:51 -0500 Subject: [PATCH 09/16] Move rds creation to propagation step --- 3.20/bioc/bbscentral2/postrun.sh | 1 - 3.21/bioc/bbscentral1/postrun.sh | 1 - BBS-make-meta-dbs.py | 50 ------------------------ propagation/3.20/updateReposPkgs-bioc.sh | 13 +++++- propagation/3.21/updateReposPkgs-bioc.sh | 14 ++++++- utils/makeMetaDbs.R | 19 ++++----- 6 files changed, 31 insertions(+), 67 deletions(-) delete mode 100755 BBS-make-meta-dbs.py diff --git a/3.20/bioc/bbscentral2/postrun.sh b/3.20/bioc/bbscentral2/postrun.sh index 5072f3b4..66e8ecd6 100755 --- a/3.20/bioc/bbscentral2/postrun.sh +++ b/3.20/bioc/bbscentral2/postrun.sh @@ -32,7 +32,6 @@ if [ -n "$BBS_OUTGOING_MAP" ]; then find OUTGOING -type d -exec chmod 755 {} \; $BBS_PYTHON_CMD $BBS_HOME/BBS-make-PROPAGATION_STATUS_DB.py chmod -R +r . - $BBS_PYTHON_CMD $BBS_HOME/BBS-make-meta-dbs.py fi # Generate the HTML report diff --git a/3.21/bioc/bbscentral1/postrun.sh b/3.21/bioc/bbscentral1/postrun.sh index 5072f3b4..66e8ecd6 100755 --- a/3.21/bioc/bbscentral1/postrun.sh +++ b/3.21/bioc/bbscentral1/postrun.sh @@ -32,7 +32,6 @@ if [ -n "$BBS_OUTGOING_MAP" ]; then find OUTGOING -type d -exec chmod 755 {} \; $BBS_PYTHON_CMD $BBS_HOME/BBS-make-PROPAGATION_STATUS_DB.py chmod -R +r . - $BBS_PYTHON_CMD $BBS_HOME/BBS-make-meta-dbs.py fi # Generate the HTML report diff --git a/BBS-make-meta-dbs.py b/BBS-make-meta-dbs.py deleted file mode 100755 index 59fa7062..00000000 --- a/BBS-make-meta-dbs.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import time -import subprocess - -import BBSvars -import BBSbase - -def make_meta_dbs(): - ## Following example of BBS-make-OUTGOING.py - ## Prepare Rexpr (must be a single string with no spaces). - Rscript_path = os.path.join(BBSvars.BBS_home, - 'utils', - 'makeMetaDbs.R') - Rfun = 'makeMetaDbs' - OUTGOING_dir = 'OUTGOING' - db_filepath = 'PROPAGATION_STATUS_DB.txt' - Rfuncall = f"{Rfun}('{OUTGOING_dir}','{db_filepath}')" - Rexpr = f"source('{Rscript_path}');{Rfuncall}" - - cmd = BBSbase.Rexpr2syscmd(Rexpr) - - try: - subprocess.run(cmd, stdout=None, stderr=subprocess.STDOUT, shell=True, - check=True) - except subprocess.CalledProcessError as e: - print(f'BBS> [generateMetaDbs] Failed at {time.asctime()} . . .') - print(f'BBS> [generateMetaDbs] Error: {e} . . .') - return - - -############################################################################## -### MAIN SECTION -############################################################################## - -if __name__ == "__main__": - if not os.path.isfile('PROPAGATION_STATUS_DB.txt'): - msg = ( - f'Make sure to be in {BBSvars.Central_rdir.path} and run ' - f'BBS-make-PROPAGATION_STATUS_DB.py before BBS-make-meta.py.' - ) - sys.exit('=> EXIT.') - print('BBS> ==============================================================') - print(f'BBS> [makeMetaDbs] STARTING on {time.asctime()} ...') - sys.stdout.flush() - make_meta_dbs() - print(f'BBS> [makeMetaDbs] DONE on {time.asctime()} ...') - diff --git a/propagation/3.20/updateReposPkgs-bioc.sh b/propagation/3.20/updateReposPkgs-bioc.sh index b2e84179..c06b25b1 100755 --- a/propagation/3.20/updateReposPkgs-bioc.sh +++ b/propagation/3.20/updateReposPkgs-bioc.sh @@ -58,6 +58,16 @@ update_repo "$MAC_BIG_SUR_arm64_CONTRIB" "mac.binary.big-sur-arm64" "tgz" echo "" +echo "" +echo "========================================================================" +/bin/date +echo "------------------------------------------------------------------------" + +$Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$REPOS_ROOT', '$META_SRC'))" + +echo "" + + ## FIXME: Why aren't manuals propagated based on the same criteria as source ## packages? Looks like the former are propagated based on their timestamps ## only (see below) while for source packages we use the more refined @@ -68,13 +78,12 @@ MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/bioc repo with reference manuals..." for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` + pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done echo "Updating $SRC_CONTRIB with aliases and cross references dbs ..." -META_SRC="$BBS_OUTGOING_DIR/Meta" cp --recursive --update --verbose $META_SRC $SRC_CONTRIB echo "DONE." diff --git a/propagation/3.21/updateReposPkgs-bioc.sh b/propagation/3.21/updateReposPkgs-bioc.sh index a113d231..22f1d0b1 100755 --- a/propagation/3.21/updateReposPkgs-bioc.sh +++ b/propagation/3.21/updateReposPkgs-bioc.sh @@ -17,6 +17,9 @@ WIN_CONTRIB="$REPOS_ROOT/bin/windows/contrib/$R_VERSION" MAC_BIG_SUR_x86_64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-x86_64/contrib/$R_VERSION" MAC_BIG_SUR_arm64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-arm64/contrib/$R_VERSION" +META_SRC="$SRC_CONTRIB/Meta" +META_R_EXPR="source('/home/biocbuild/BBS/utils/makeMetaDbs.R')" + if [ ! -f "$PROPAGATION_DB_FILE" ]; then echo "ERROR: $PROPAGATION_DB_FILE not found. Did postrun.sh run?" exit 1 @@ -58,6 +61,16 @@ update_repo "$MAC_BIG_SUR_arm64_CONTRIB" "mac.binary.big-sur-arm64" "tgz" echo "" +echo "" +echo "========================================================================" +/bin/date +echo "------------------------------------------------------------------------" + +$Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$REPOS_ROOT', '$META_SRC'))" + +echo "" + + ## FIXME: Why aren't manuals propagated based on the same criteria as source ## packages? Looks like the former are propagated based on their timestamps ## only (see below) while for source packages we use the more refined @@ -74,7 +87,6 @@ for i in `ls $MANUALS_SRC`; do done echo "Updating $SRC_CONTRIB with aliases and cross references dbs ..." -META_SRC="$BBS_OUTGOING_DIR/Meta" cp --recursive --update --verbose $META_SRC $SRC_CONTRIB echo "DONE." diff --git a/utils/makeMetaDbs.R b/utils/makeMetaDbs.R index e42208e5..652cfdbe 100644 --- a/utils/makeMetaDbs.R +++ b/utils/makeMetaDbs.R @@ -1,8 +1,7 @@ -makeMetaDbs <- function(outgoing_path, db_filepath) { +makeMetaDbs <- function(db_filepath, repos_root, meta_path) { meat_dir <- Sys.getenv("BBS_MEAT_PATH") - bbs_central_rdir <- Sys.getenv("BBS_CENTRAL_RDIR") - prop_status <- read.dcf(file.path(bbs_central_rdir, db_filepath)) + prop_status <- read.dcf(db_filepath) pkgs <- c() for (i in 1:dim(prop_status)[2]) { @@ -13,26 +12,22 @@ makeMetaDbs <- function(outgoing_path, db_filepath) { } } - # Because the location is always recreated, we always create the Meta - # directory and always create all the databases - web_dir <- file.path(bbs_central_rdir, "web", "packages") - meta_dir <- file.path(bbs_central_rdir, "OUTGOING", "Meta") + web_dir <- file.path(repos_root "web", "packages") + meta_dir <- file.path(meta_path) if (!dir.exists(meta_dir)) { dir.create(meta_dir, recursive = TRUE) } pkg_paths <- file.path(meat_dir, pkgs) for (pkg_path in pkg_paths) { - biocViews::build_db_from_source(pkg_path, bbs_central_rdir) + biocViews::build_db_from_source(pkg_path, repos_root) } aliases_db_file <- file.path(meta_dir, "aliases.rds") - meta_aliases_db <- biocViews::build_meta_aliases_db(web_dir, aliases_db_file, - TRUE) + meta_aliases_db <- biocViews::build_meta_aliases_db(web_dir, aliases_db_file) saveRDS(meta_aliases_db, aliases_db_file, version = 2) rdxrefs_db_file <- file.path(meta_dir, "rdxrefs.rds") - meta_rdxrefs_db <- biocViews::build_meta_rdxrefs_db(web_dir, rdxrefs_db_file, - TRUE) + meta_rdxrefs_db <- biocViews::build_meta_rdxrefs_db(web_dir, rdxrefs_db_file) saveRDS(meta_rdxrefs_db, rdxrefs_db_file, version = 2) } From 34621e66f35365d4c695c2f99b6910b084e80e23 Mon Sep 17 00:00:00 2001 From: J Wokaty Date: Mon, 10 Feb 2025 16:37:53 -0500 Subject: [PATCH 10/16] Fix meta creation --- propagation/3.20/updateReposPkgs-bioc.sh | 7 ++++--- propagation/3.21/updateReposPkgs-bioc.sh | 8 +++----- utils/makeMetaDbs.R | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/propagation/3.20/updateReposPkgs-bioc.sh b/propagation/3.20/updateReposPkgs-bioc.sh index c06b25b1..516381c7 100755 --- a/propagation/3.20/updateReposPkgs-bioc.sh +++ b/propagation/3.20/updateReposPkgs-bioc.sh @@ -17,6 +17,9 @@ WIN_CONTRIB="$REPOS_ROOT/bin/windows/contrib/$R_VERSION" MAC_BIG_SUR_x86_64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-x86_64/contrib/$R_VERSION" MAC_BIG_SUR_arm64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-arm64/contrib/$R_VERSION" +META_SRC="$SRC_CONTRIB/Meta" +META_R_EXPR="source('/home/biocbuild/BBS/utils/makeMetaDbs.R')" + if [ ! -f "$PROPAGATION_DB_FILE" ]; then echo "ERROR: $PROPAGATION_DB_FILE not found. Did postrun.sh run?" exit 1 @@ -63,6 +66,7 @@ echo "========================================================================" /bin/date echo "------------------------------------------------------------------------" +echo "Updating $META_SRC with aliases and cross refences dbs..." $Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$REPOS_ROOT', '$META_SRC'))" echo "" @@ -83,8 +87,5 @@ for i in `ls $MANUALS_SRC`; do cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done -echo "Updating $SRC_CONTRIB with aliases and cross references dbs ..." -cp --recursive --update --verbose $META_SRC $SRC_CONTRIB - echo "DONE." exit 0 diff --git a/propagation/3.21/updateReposPkgs-bioc.sh b/propagation/3.21/updateReposPkgs-bioc.sh index 22f1d0b1..143be9fc 100755 --- a/propagation/3.21/updateReposPkgs-bioc.sh +++ b/propagation/3.21/updateReposPkgs-bioc.sh @@ -1,5 +1,4 @@ #!/bin/bash - set -e # exit immediately if a simple command returns a non-zero status cd "$HOME/propagation/3.21" @@ -19,6 +18,7 @@ MAC_BIG_SUR_arm64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-arm64/contrib/$R_VERSI META_SRC="$SRC_CONTRIB/Meta" META_R_EXPR="source('/home/biocbuild/BBS/utils/makeMetaDbs.R')" +MEAT_PATH="/home/biocbuild/bbs-$BIOC_VERSION-bioc/meat" if [ ! -f "$PROPAGATION_DB_FILE" ]; then echo "ERROR: $PROPAGATION_DB_FILE not found. Did postrun.sh run?" @@ -66,7 +66,8 @@ echo "========================================================================" /bin/date echo "------------------------------------------------------------------------" -$Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$REPOS_ROOT', '$META_SRC'))" +echo "Updating $META_SRC with aliases and cross refences dbs..." +$Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$MEAT_PATH', '$REPOS_ROOT', '$META_SRC'))" echo "" @@ -86,8 +87,5 @@ for i in `ls $MANUALS_SRC`; do cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done -echo "Updating $SRC_CONTRIB with aliases and cross references dbs ..." -cp --recursive --update --verbose $META_SRC $SRC_CONTRIB - echo "DONE." exit 0 diff --git a/utils/makeMetaDbs.R b/utils/makeMetaDbs.R index 652cfdbe..ec0926a1 100644 --- a/utils/makeMetaDbs.R +++ b/utils/makeMetaDbs.R @@ -12,7 +12,7 @@ makeMetaDbs <- function(db_filepath, repos_root, meta_path) { } } - web_dir <- file.path(repos_root "web", "packages") + web_dir <- file.path(repos_root, "web", "packages") meta_dir <- file.path(meta_path) if (!dir.exists(meta_dir)) { dir.create(meta_dir, recursive = TRUE) From aedb57713af79abf8c3eb8fdf8a7d19062457e6c Mon Sep 17 00:00:00 2001 From: A Wokaty Date: Thu, 1 May 2025 16:25:01 -0400 Subject: [PATCH 11/16] Remove changes from BBS-make-OUTGOING.py and 3.20 propagation files --- BBS-make-OUTGOING.py | 16 ---------------- propagation/3.20/updateReposPkgs-bioc.sh | 16 +--------------- .../3.20/updateReposPkgs-data-annotation.sh | 2 +- .../3.20/updateReposPkgs-data-experiment.sh | 2 +- 4 files changed, 3 insertions(+), 33 deletions(-) diff --git a/BBS-make-OUTGOING.py b/BBS-make-OUTGOING.py index 4d47ad4b..8bad53a7 100755 --- a/BBS-make-OUTGOING.py +++ b/BBS-make-OUTGOING.py @@ -11,7 +11,6 @@ import os import time import shutil -import subprocess import bbs.fileutils import bbs.parse @@ -121,7 +120,6 @@ def copy_outgoing_pkgs(products_in_subdir, source_node): if BBSvars.buildtype in ['workflows', 'books', 'bioc-mac-arm64']: pass elif source_node: - # pdf manuals pdf_file = os.path.join(BBSvars.products_in_rdir.path, BBSutils.getSourceNode(), 'checksrc', @@ -134,20 +132,6 @@ def copy_outgoing_pkgs(products_in_subdir, source_node): os.link(pdf_file, dst) # create hard link to avoid making a copy else: print("BBS> [stage6b] SKIPPED (file %s doesn't exist)" % pdf_file) - # html manuals - rel_link = "../../%s/man/%s.html" - hooks = f"hooks <- list(pkg_href = function(pkg) sprintf('{rel_link}', pkg, pkg));" - html_file = os.path.join(BBSvars.Central_rdir.path, - "OUTGOING/manuals", - '%s.html' % pkg) - Rexpr = hooks + f"tools::pkg2HTML('{pkg_path}',out='{html_file}',hooks=hooks)" - cmd = BBSbase.Rexpr2syscmd(Rexpr) - try: - print(Rexpr) - subprocess.run(cmd, stdout=None, stderr=subprocess.STDOUT, - shell=True, check=True) - except subprocess.CalledProcessError as e: - print("BBS> [stage6b] SKIPPED (could not generate %s)" % html_file) print('BBS> [stage6b] END copying outgoing packages from %s.' % srcdir) return diff --git a/propagation/3.20/updateReposPkgs-bioc.sh b/propagation/3.20/updateReposPkgs-bioc.sh index 516381c7..e7c410e8 100755 --- a/propagation/3.20/updateReposPkgs-bioc.sh +++ b/propagation/3.20/updateReposPkgs-bioc.sh @@ -17,9 +17,6 @@ WIN_CONTRIB="$REPOS_ROOT/bin/windows/contrib/$R_VERSION" MAC_BIG_SUR_x86_64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-x86_64/contrib/$R_VERSION" MAC_BIG_SUR_arm64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-arm64/contrib/$R_VERSION" -META_SRC="$SRC_CONTRIB/Meta" -META_R_EXPR="source('/home/biocbuild/BBS/utils/makeMetaDbs.R')" - if [ ! -f "$PROPAGATION_DB_FILE" ]; then echo "ERROR: $PROPAGATION_DB_FILE not found. Did postrun.sh run?" exit 1 @@ -61,17 +58,6 @@ update_repo "$MAC_BIG_SUR_arm64_CONTRIB" "mac.binary.big-sur-arm64" "tgz" echo "" -echo "" -echo "========================================================================" -/bin/date -echo "------------------------------------------------------------------------" - -echo "Updating $META_SRC with aliases and cross refences dbs..." -$Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$REPOS_ROOT', '$META_SRC'))" - -echo "" - - ## FIXME: Why aren't manuals propagated based on the same criteria as source ## packages? Looks like the former are propagated based on their timestamps ## only (see below) while for source packages we use the more refined @@ -82,7 +68,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/bioc repo with reference manuals..." for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` + pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done diff --git a/propagation/3.20/updateReposPkgs-data-annotation.sh b/propagation/3.20/updateReposPkgs-data-annotation.sh index d00505aa..7c25cc47 100755 --- a/propagation/3.20/updateReposPkgs-data-annotation.sh +++ b/propagation/3.20/updateReposPkgs-data-annotation.sh @@ -54,7 +54,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/data/annotation repo with reference manuals..." for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` + pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done diff --git a/propagation/3.20/updateReposPkgs-data-experiment.sh b/propagation/3.20/updateReposPkgs-data-experiment.sh index cb69c899..2fd83f2e 100755 --- a/propagation/3.20/updateReposPkgs-data-experiment.sh +++ b/propagation/3.20/updateReposPkgs-data-experiment.sh @@ -54,7 +54,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/data/experiment repo with reference manuals..." for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` + pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done From 95552dbb531c511254ccecf63c9f8f274906ce1c Mon Sep 17 00:00:00 2001 From: A Wokaty Date: Fri, 2 May 2025 11:44:31 -0400 Subject: [PATCH 12/16] Set bioc, experiments to use extractManuals; copy html manuals --- propagation/3.22/prepareRepos-bioc.config | 2 +- .../3.22/prepareRepos-data-experiment.config | 2 +- propagation/3.22/updateReposPkgs-bioc.sh | 17 ++++++++++++++++- .../3.22/updateReposPkgs-data-annotation.sh | 2 +- .../3.22/updateReposPkgs-data-experiment.sh | 2 +- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/propagation/3.22/prepareRepos-bioc.config b/propagation/3.22/prepareRepos-bioc.config index 79c50c0b..389684db 100644 --- a/propagation/3.22/prepareRepos-bioc.config +++ b/propagation/3.22/prepareRepos-bioc.config @@ -7,7 +7,7 @@ MEAT0_PATH="/home/biocbuild/bbs-3.22-bioc/MEAT0" # Manuals were already copied from # /home/biocbuild/public_html/BBS/3.22/bioc/OUTGOING/manuals # by updateReposPkgs-bioc.sh so don't need to be extracted. -EXTRACT_MANUALS="FALSE" +EXTRACT_MANUALS="TRUE" REPOS_ROOT="$VIEW_ROOT/bioc" REPOS_URL="$VIEW_URL/bioc" STATS_URL="https://bioconductor.org/packages/stats/bioc" diff --git a/propagation/3.22/prepareRepos-data-experiment.config b/propagation/3.22/prepareRepos-data-experiment.config index 55b17afe..f0b66899 100644 --- a/propagation/3.22/prepareRepos-data-experiment.config +++ b/propagation/3.22/prepareRepos-data-experiment.config @@ -7,7 +7,7 @@ MEAT0_PATH="/home/biocbuild/bbs-3.22-data-experiment/MEAT0" # Manuals were already copied from # /home/biocbuild/public_html/BBS/3.22/data-experiment/OUTGOING/manuals # by updateReposPkgs-data-experiment.sh so don't need to be extracted. -EXTRACT_MANUALS="FALSE" +EXTRACT_MANUALS="TRUE" REPOS_ROOT="$VIEW_ROOT/data/experiment" REPOS_URL="$VIEW_URL/data/experiment" STATS_URL="" diff --git a/propagation/3.22/updateReposPkgs-bioc.sh b/propagation/3.22/updateReposPkgs-bioc.sh index ba4aeccd..875e8e0e 100755 --- a/propagation/3.22/updateReposPkgs-bioc.sh +++ b/propagation/3.22/updateReposPkgs-bioc.sh @@ -17,6 +17,10 @@ WIN_CONTRIB="$REPOS_ROOT/bin/windows/contrib/$R_VERSION" MAC_BIG_SUR_x86_64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-x86_64/contrib/$R_VERSION" MAC_BIG_SUR_arm64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-arm64/contrib/$R_VERSION" +META_SRC="$SRC_CONTRIB/Meta" +META_R_EXPR="source('/home/biocbuild/BBS/utils/makeMetaDbs.R')" +MEAT_PATH="/home/biocbuild/bbs-$BIOC_VERSION-bioc/meat" + if [ ! -f "$PROPAGATION_DB_FILE" ]; then echo "ERROR: $PROPAGATION_DB_FILE not found. Did postrun.sh run?" exit 1 @@ -58,6 +62,17 @@ update_repo "$MAC_BIG_SUR_arm64_CONTRIB" "mac.binary.big-sur-arm64" "tgz" echo "" +echo "" +echo "========================================================================" +/bin/date +echo "------------------------------------------------------------------------" + +echo "Updating $META_SRC with aliases and cross refences dbs..." +$Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$MEAT_PATH', '$REPOS_ROOT', '$META_SRC'))" + +echo "" + + ## FIXME: Why aren't manuals propagated based on the same criteria as source ## packages? Looks like the former are propagated based on their timestamps ## only (see below) while for source packages we use the more refined @@ -68,7 +83,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/bioc repo with reference manuals..." for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` + pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done diff --git a/propagation/3.22/updateReposPkgs-data-annotation.sh b/propagation/3.22/updateReposPkgs-data-annotation.sh index da8d1316..a6b2884e 100755 --- a/propagation/3.22/updateReposPkgs-data-annotation.sh +++ b/propagation/3.22/updateReposPkgs-data-annotation.sh @@ -54,7 +54,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/data/annotation repo with reference manuals..." for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` + pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done diff --git a/propagation/3.22/updateReposPkgs-data-experiment.sh b/propagation/3.22/updateReposPkgs-data-experiment.sh index ffaa53ab..c9101207 100755 --- a/propagation/3.22/updateReposPkgs-data-experiment.sh +++ b/propagation/3.22/updateReposPkgs-data-experiment.sh @@ -54,7 +54,7 @@ MANUALS_DEST="$REPOS_ROOT/manuals" MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" echo "Updating $BIOC_VERSION/data/experiment repo with reference manuals..." for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".pdf"); print(a[1])}'` + pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` mkdir -p $MANUALS_DEST/$pkg/man cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man done From e6bf198d9872aba2b3d6e92b67f31305a145641e Mon Sep 17 00:00:00 2001 From: A Wokaty Date: Fri, 2 May 2025 13:57:07 -0400 Subject: [PATCH 13/16] Generate bioc, exp manuals via extractManuals --- propagation/3.21/prepareRepos-bioc.config | 2 +- propagation/3.21/prepareRepos-data-experiment.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/propagation/3.21/prepareRepos-bioc.config b/propagation/3.21/prepareRepos-bioc.config index 790c2fca..c5be9a9a 100644 --- a/propagation/3.21/prepareRepos-bioc.config +++ b/propagation/3.21/prepareRepos-bioc.config @@ -7,7 +7,7 @@ MEAT0_PATH="/home/biocbuild/bbs-3.21-bioc/MEAT0" # Manuals were already copied from # /home/biocbuild/public_html/BBS/3.21/bioc/OUTGOING/manuals # by updateReposPkgs-bioc.sh so don't need to be extracted. -EXTRACT_MANUALS="FALSE" +EXTRACT_MANUALS="TRUE" REPOS_ROOT="$VIEW_ROOT/bioc" REPOS_URL="$VIEW_URL/bioc" STATS_URL="https://bioconductor.org/packages/stats/bioc" diff --git a/propagation/3.21/prepareRepos-data-experiment.config b/propagation/3.21/prepareRepos-data-experiment.config index ce33caa8..52ad3b47 100644 --- a/propagation/3.21/prepareRepos-data-experiment.config +++ b/propagation/3.21/prepareRepos-data-experiment.config @@ -7,7 +7,7 @@ MEAT0_PATH="/home/biocbuild/bbs-3.21-data-experiment/MEAT0" # Manuals were already copied from # /home/biocbuild/public_html/BBS/3.21/data-experiment/OUTGOING/manuals # by updateReposPkgs-data-experiment.sh so don't need to be extracted. -EXTRACT_MANUALS="FALSE" +EXTRACT_MANUALS="TRUE" REPOS_ROOT="$VIEW_ROOT/data/experiment" REPOS_URL="$VIEW_URL/data/experiment" STATS_URL="" From 043a5c9e6b0de72ad88f999a9bb3b1eadf09dd7d Mon Sep 17 00:00:00 2001 From: A Wokaty Date: Fri, 2 May 2025 16:01:30 -0400 Subject: [PATCH 14/16] Remove functionality copying manuals in BBS-make-OUTGOING.py --- BBS-make-OUTGOING.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/BBS-make-OUTGOING.py b/BBS-make-OUTGOING.py index 8bad53a7..9e2028a5 100755 --- a/BBS-make-OUTGOING.py +++ b/BBS-make-OUTGOING.py @@ -85,14 +85,6 @@ def copy_outgoing_pkgs(products_in_subdir, source_node): errmsg = "Directory '%s' does not exist!\n\n" % srcdir + \ " %s is late or stopped building?" % node_hostname raise FileExistsError(errmsg) - ## Workflow and book packages do not have manuals/ because we do not run - ## `R CMD check`. - manuals_dir = '../manuals' - if BBSvars.buildtype in ['workflows', 'books', 'bioc-mac-arm64']: - pass - elif source_node: - print('BBS> [stage6b] mkdir %s' % manuals_dir) - os.mkdir(manuals_dir) print('BBS> [stage6b] BEGIN copying outgoing packages from %s.' % srcdir) node_Arch = BBSutils.getNodeSpec(node_hostname, 'Arch') node_pkgType = BBSutils.getNodeSpec(node_hostname, 'pkgType') @@ -116,22 +108,6 @@ def copy_outgoing_pkgs(products_in_subdir, source_node): os.link(pkg_path, pkg_file) # create hard link to avoid making a copy else: print("BBS> [stage6b] SKIPPED (file %s doesn't exist)" % pkg_path) - ## Get reference manual from pkg.Rcheck directory. - if BBSvars.buildtype in ['workflows', 'books', 'bioc-mac-arm64']: - pass - elif source_node: - pdf_file = os.path.join(BBSvars.products_in_rdir.path, - BBSutils.getSourceNode(), - 'checksrc', - '%s.Rcheck' % pkg, - '%s-manual.pdf' % pkg) - print('BBS> [stage6b] - copying %s to OUTGOING/manuals folder...' % pdf_file) - if os.path.exists(pdf_file): - dst = os.path.join(manuals_dir, '%s.pdf' % pkg) - #shutil.copy(pdf_file, dst) - os.link(pdf_file, dst) # create hard link to avoid making a copy - else: - print("BBS> [stage6b] SKIPPED (file %s doesn't exist)" % pdf_file) print('BBS> [stage6b] END copying outgoing packages from %s.' % srcdir) return From 900270c6e2bd90aeebb6fa1f245a59e1c0ffc2d3 Mon Sep 17 00:00:00 2001 From: A Wokaty Date: Tue, 6 May 2025 15:22:50 -0400 Subject: [PATCH 15/16] Remove copy manuals in updateReposPkgs*.sh --- propagation/3.21/updateReposPkgs-bioc.sh | 18 ------------------ .../3.21/updateReposPkgs-data-annotation.sh | 17 ----------------- .../3.21/updateReposPkgs-data-experiment.sh | 17 ----------------- propagation/3.22/updateReposPkgs-bioc.sh | 18 ------------------ .../3.22/updateReposPkgs-data-annotation.sh | 17 ----------------- .../3.22/updateReposPkgs-data-experiment.sh | 17 ----------------- 6 files changed, 104 deletions(-) diff --git a/propagation/3.21/updateReposPkgs-bioc.sh b/propagation/3.21/updateReposPkgs-bioc.sh index 143be9fc..f63b562b 100755 --- a/propagation/3.21/updateReposPkgs-bioc.sh +++ b/propagation/3.21/updateReposPkgs-bioc.sh @@ -69,23 +69,5 @@ echo "------------------------------------------------------------------------" echo "Updating $META_SRC with aliases and cross refences dbs..." $Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$MEAT_PATH', '$REPOS_ROOT', '$META_SRC'))" -echo "" - - -## FIXME: Why aren't manuals propagated based on the same criteria as source -## packages? Looks like the former are propagated based on their timestamps -## only (see below) while for source packages we use the more refined -## propagation criteria. This can easily lead to situations where the manual -## available on a package landing page doesn't match the version of the -## source package. Not good! -MANUALS_DEST="$REPOS_ROOT/manuals" -MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" -echo "Updating $BIOC_VERSION/bioc repo with reference manuals..." -for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` - mkdir -p $MANUALS_DEST/$pkg/man - cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man -done - echo "DONE." exit 0 diff --git a/propagation/3.21/updateReposPkgs-data-annotation.sh b/propagation/3.21/updateReposPkgs-data-annotation.sh index 8746393d..5c6d23f6 100755 --- a/propagation/3.21/updateReposPkgs-data-annotation.sh +++ b/propagation/3.21/updateReposPkgs-data-annotation.sh @@ -42,22 +42,5 @@ echo "" echo "Updating $BIOC_VERSION/data/annotation repo with source packages..." update_repo "$SRC_CONTRIB" "source" "tar.gz" -echo "" - -## FIXME: Why aren't manuals propagated based on the same criteria as source -## packages? Looks like the former are propagated based on their timestamps -## only (see below) while for source packages we use the more refined -## propagation criteria. This can easily lead to situations where the manual -## available on a package landing page doesn't match the version of the -## source package. Not good! -MANUALS_DEST="$REPOS_ROOT/manuals" -MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" -echo "Updating $BIOC_VERSION/data/annotation repo with reference manuals..." -for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` - mkdir -p $MANUALS_DEST/$pkg/man - cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man -done - echo "DONE." exit 0 diff --git a/propagation/3.21/updateReposPkgs-data-experiment.sh b/propagation/3.21/updateReposPkgs-data-experiment.sh index bd90840a..fe6aaf5a 100755 --- a/propagation/3.21/updateReposPkgs-data-experiment.sh +++ b/propagation/3.21/updateReposPkgs-data-experiment.sh @@ -42,22 +42,5 @@ echo "" echo "Updating $BIOC_VERSION/data/experiment repo with source packages..." update_repo "$SRC_CONTRIB" "source" "tar.gz" -echo "" - -## FIXME: Why aren't manuals propagated based on the same criteria as source -## packages? Looks like the former are propagated based on their timestamps -## only (see below) while for source packages we use the more refined -## propagation criteria. This can easily lead to situations where the manual -## available on a package landing page doesn't match the version of the -## source package. Not good! -MANUALS_DEST="$REPOS_ROOT/manuals" -MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" -echo "Updating $BIOC_VERSION/data/experiment repo with reference manuals..." -for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` - mkdir -p $MANUALS_DEST/$pkg/man - cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man -done - echo "DONE." exit 0 diff --git a/propagation/3.22/updateReposPkgs-bioc.sh b/propagation/3.22/updateReposPkgs-bioc.sh index 875e8e0e..daf12e99 100755 --- a/propagation/3.22/updateReposPkgs-bioc.sh +++ b/propagation/3.22/updateReposPkgs-bioc.sh @@ -70,23 +70,5 @@ echo "------------------------------------------------------------------------" echo "Updating $META_SRC with aliases and cross refences dbs..." $Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$MEAT_PATH', '$REPOS_ROOT', '$META_SRC'))" -echo "" - - -## FIXME: Why aren't manuals propagated based on the same criteria as source -## packages? Looks like the former are propagated based on their timestamps -## only (see below) while for source packages we use the more refined -## propagation criteria. This can easily lead to situations where the manual -## available on a package landing page doesn't match the version of the -## source package. Not good! -MANUALS_DEST="$REPOS_ROOT/manuals" -MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" -echo "Updating $BIOC_VERSION/bioc repo with reference manuals..." -for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` - mkdir -p $MANUALS_DEST/$pkg/man - cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man -done - echo "DONE." exit 0 diff --git a/propagation/3.22/updateReposPkgs-data-annotation.sh b/propagation/3.22/updateReposPkgs-data-annotation.sh index a6b2884e..96776e8a 100755 --- a/propagation/3.22/updateReposPkgs-data-annotation.sh +++ b/propagation/3.22/updateReposPkgs-data-annotation.sh @@ -42,22 +42,5 @@ echo "" echo "Updating $BIOC_VERSION/data/annotation repo with source packages..." update_repo "$SRC_CONTRIB" "source" "tar.gz" -echo "" - -## FIXME: Why aren't manuals propagated based on the same criteria as source -## packages? Looks like the former are propagated based on their timestamps -## only (see below) while for source packages we use the more refined -## propagation criteria. This can easily lead to situations where the manual -## available on a package landing page doesn't match the version of the -## source package. Not good! -MANUALS_DEST="$REPOS_ROOT/manuals" -MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" -echo "Updating $BIOC_VERSION/data/annotation repo with reference manuals..." -for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` - mkdir -p $MANUALS_DEST/$pkg/man - cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man -done - echo "DONE." exit 0 diff --git a/propagation/3.22/updateReposPkgs-data-experiment.sh b/propagation/3.22/updateReposPkgs-data-experiment.sh index c9101207..54417fb9 100755 --- a/propagation/3.22/updateReposPkgs-data-experiment.sh +++ b/propagation/3.22/updateReposPkgs-data-experiment.sh @@ -42,22 +42,5 @@ echo "" echo "Updating $BIOC_VERSION/data/experiment repo with source packages..." update_repo "$SRC_CONTRIB" "source" "tar.gz" -echo "" - -## FIXME: Why aren't manuals propagated based on the same criteria as source -## packages? Looks like the former are propagated based on their timestamps -## only (see below) while for source packages we use the more refined -## propagation criteria. This can easily lead to situations where the manual -## available on a package landing page doesn't match the version of the -## source package. Not good! -MANUALS_DEST="$REPOS_ROOT/manuals" -MANUALS_SRC="$BBS_OUTGOING_DIR/manuals" -echo "Updating $BIOC_VERSION/data/experiment repo with reference manuals..." -for i in `ls $MANUALS_SRC`; do - pkg=`echo $i| awk '{split($0,a,".(html|pdf)"); print(a[1])}'` - mkdir -p $MANUALS_DEST/$pkg/man - cp --update --verbose $MANUALS_SRC/$i $MANUALS_DEST/$pkg/man -done - echo "DONE." exit 0 From 4cf30a0c6b099e4068f1c9354c8160984379ec41 Mon Sep 17 00:00:00 2001 From: A Wokaty Date: Tue, 6 May 2025 16:18:20 -0400 Subject: [PATCH 16/16] Move metaDb to another PR --- propagation/3.21/updateReposPkgs-bioc.sh | 12 --------- propagation/3.22/updateReposPkgs-bioc.sh | 12 --------- utils/makeMetaDbs.R | 33 ------------------------ 3 files changed, 57 deletions(-) delete mode 100644 utils/makeMetaDbs.R diff --git a/propagation/3.21/updateReposPkgs-bioc.sh b/propagation/3.21/updateReposPkgs-bioc.sh index f63b562b..daee2e8f 100755 --- a/propagation/3.21/updateReposPkgs-bioc.sh +++ b/propagation/3.21/updateReposPkgs-bioc.sh @@ -16,10 +16,6 @@ WIN_CONTRIB="$REPOS_ROOT/bin/windows/contrib/$R_VERSION" MAC_BIG_SUR_x86_64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-x86_64/contrib/$R_VERSION" MAC_BIG_SUR_arm64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-arm64/contrib/$R_VERSION" -META_SRC="$SRC_CONTRIB/Meta" -META_R_EXPR="source('/home/biocbuild/BBS/utils/makeMetaDbs.R')" -MEAT_PATH="/home/biocbuild/bbs-$BIOC_VERSION-bioc/meat" - if [ ! -f "$PROPAGATION_DB_FILE" ]; then echo "ERROR: $PROPAGATION_DB_FILE not found. Did postrun.sh run?" exit 1 @@ -61,13 +57,5 @@ update_repo "$MAC_BIG_SUR_arm64_CONTRIB" "mac.binary.big-sur-arm64" "tgz" echo "" -echo "" -echo "========================================================================" -/bin/date -echo "------------------------------------------------------------------------" - -echo "Updating $META_SRC with aliases and cross refences dbs..." -$Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$MEAT_PATH', '$REPOS_ROOT', '$META_SRC'))" - echo "DONE." exit 0 diff --git a/propagation/3.22/updateReposPkgs-bioc.sh b/propagation/3.22/updateReposPkgs-bioc.sh index daf12e99..fc36ff0a 100755 --- a/propagation/3.22/updateReposPkgs-bioc.sh +++ b/propagation/3.22/updateReposPkgs-bioc.sh @@ -17,10 +17,6 @@ WIN_CONTRIB="$REPOS_ROOT/bin/windows/contrib/$R_VERSION" MAC_BIG_SUR_x86_64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-x86_64/contrib/$R_VERSION" MAC_BIG_SUR_arm64_CONTRIB="$REPOS_ROOT/bin/macosx/big-sur-arm64/contrib/$R_VERSION" -META_SRC="$SRC_CONTRIB/Meta" -META_R_EXPR="source('/home/biocbuild/BBS/utils/makeMetaDbs.R')" -MEAT_PATH="/home/biocbuild/bbs-$BIOC_VERSION-bioc/meat" - if [ ! -f "$PROPAGATION_DB_FILE" ]; then echo "ERROR: $PROPAGATION_DB_FILE not found. Did postrun.sh run?" exit 1 @@ -62,13 +58,5 @@ update_repo "$MAC_BIG_SUR_arm64_CONTRIB" "mac.binary.big-sur-arm64" "tgz" echo "" -echo "" -echo "========================================================================" -/bin/date -echo "------------------------------------------------------------------------" - -echo "Updating $META_SRC with aliases and cross refences dbs..." -$Rscript -e "$META_R_EXPR; try(makeMetaDbs('$PROPAGATION_DB_FILE', '$MEAT_PATH', '$REPOS_ROOT', '$META_SRC'))" - echo "DONE." exit 0 diff --git a/utils/makeMetaDbs.R b/utils/makeMetaDbs.R deleted file mode 100644 index ec0926a1..00000000 --- a/utils/makeMetaDbs.R +++ /dev/null @@ -1,33 +0,0 @@ -makeMetaDbs <- function(db_filepath, repos_root, meta_path) { - - meat_dir <- Sys.getenv("BBS_MEAT_PATH") - prop_status <- read.dcf(db_filepath) - - pkgs <- c() - for (i in 1:dim(prop_status)[2]) { - pkg_type_stage <- strsplit(colnames(prop_status)[i], "#")[[1]] - if ("source" %in% pkg_type_stage && - strsplit(prop_status[i], ",")[[1]][1] %in% c("YES", "UNNEEDED")) { - pkgs <- c(pkgs, pkg_type_stage[1]) - } - } - - web_dir <- file.path(repos_root, "web", "packages") - meta_dir <- file.path(meta_path) - if (!dir.exists(meta_dir)) { - dir.create(meta_dir, recursive = TRUE) - } - - pkg_paths <- file.path(meat_dir, pkgs) - for (pkg_path in pkg_paths) { - biocViews::build_db_from_source(pkg_path, repos_root) - } - - aliases_db_file <- file.path(meta_dir, "aliases.rds") - meta_aliases_db <- biocViews::build_meta_aliases_db(web_dir, aliases_db_file) - saveRDS(meta_aliases_db, aliases_db_file, version = 2) - - rdxrefs_db_file <- file.path(meta_dir, "rdxrefs.rds") - meta_rdxrefs_db <- biocViews::build_meta_rdxrefs_db(web_dir, rdxrefs_db_file) - saveRDS(meta_rdxrefs_db, rdxrefs_db_file, version = 2) -}