Pipeline cleanup

.bb/dependencies.txt

@@ -1 +1 @@
-https://github.com/biobricks-ai/pubchem-annotations#4bc6de375ba7dbdbd2986b53d952e41ace592551
+https://github.com/biobricks-ai/pubchem-annotations#3b1b417e43b17b6caf75a730f8c4119aa76f8c16

dvc.yaml

@@ -1,11 +1,17 @@
 stages:
 
   update_biobricks_dependencies:
-    cmd: stages/00_update_biobricks_dependencies.py
+    cmd: stages/00_update_biobricks_deps.sh
 
   process-annotations:
     cmd: stages/01_process.py
     deps:
       - .bb/dependencies.txt
     outs:
-      - brick/pubchem-annotations.hdt
+      - cache/process/combined_annotations.ttl
+  build:
+    cmd: stages/02_build.sh
+    deps:
+      - cache/process/combined_annotations.ttl
+    outs:
+      - brick/

stages/00_update_biobricks_deps.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/bash
+
+biobricks pull

stages/01_process.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import biobricks as bb
 import pandas as pd
 import pyarrow.parquet as pq

stages/02_build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Get local path
+localpath=$(pwd)
+echo "Local path: $localpath"
+
+# Get cache path
+cachepath="$localpath/cache"
+echo "Cache path: $cachepath"
+
+# Create brick directory
+brickpath="$localpath/brick"
+mkdir -p $brickpath
+echo "Brick path: $brickpath"
+
+export base_uri="http://rdf.ncbi.nlm.nih.gov/pubchem/annotations.hdt"
+
+rdf2hdt -i -p -B "$base_uri"  $cachepath/process/combined_annotations.ttl $brickpath/annotations.hdt

stages/02_test.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import rdflib
 import pathlib
 from rdflib_hdt import HDTStore