diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..3776dc9 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,271 @@ +{ + "permissions": { + "allow": [ + "Bash(git add *)", + "Bash(git check-ignore *)", + "Bash(curl -sL -A \"Mozilla/5.0\" -o bengio2021gflownet.pdf \"https://arxiv.org/pdf/2106.04399\")", + "Bash(curl -sL -A \"Mozilla/5.0\" -o koziarski2024rgfn.pdf \"https://arxiv.org/pdf/2406.08506\")", + "Bash(curl -sL -A \"Mozilla/5.0\" -o glueeval2024.pdf \"https://pubs.acs.org/doi/pdf/10.1021/acsomega.4c08049\")", + "WebSearch", + "Bash(pip install *)", + "Bash(python3 -c ' *)", + "Bash(cd *)", + "Bash(python3 -m pip install --user pdfminer.six -q)", + "Bash(python3)", + "Bash(grep -rniE \"glue-specific cooperativity|direct.{0,12}reward|oracle.{0,15}reward signal|trains? \\(against|on\\) the oracle|reward-maximiz|reinforcement\" docs README.md benchmarks models scripts Logs/README.md Logs/000_TEMPLATE.md)", + "Bash(grep -E \"__pycache__|\\\\.pyc$\")", + "Bash(git rm *)", + "Bash(grep -E \"\\\\.py$\")", + "Bash(conda env *)", + "Bash(git status *)", + "Bash(python *)", + "Bash(conda run *)", + "Bash(git mv *)", + "Bash(sed -i '' 's|Logs/RESEARCH_CONTEXT\\\\.md|docs/RESEARCH_CONTEXT.md|g' CLAUDE.md README.md benchmarks/README.md docs/REFACTOR_LOG.md experiments/active_learning_6td3/README.md glue/active_learning/__init__.py glue/active_learning/loop.py glue/oracles/base.py glue/oracles/docking_6td3_oracle.py glue/proxies/learned_proxy.py)", + "Bash(grep -rn \"[^/]RESEARCH_CONTEXT.md\\\\|^RESEARCH_CONTEXT.md\\\\|\\\\`RESEARCH_CONTEXT.md\" --include=\"*.md\" --include=\"*.py\" .)", + "Bash(sed -i '' 's|`RESEARCH_CONTEXT\\\\.md`|`docs/RESEARCH_CONTEXT.md`|g' Logs/references/README.md)", + "Bash(sed -i '' 's| RESEARCH_CONTEXT\\\\.md| docs/RESEARCH_CONTEXT.md|g' Logs/005_tier2-vina-roc-pr-curves.md)", + "Bash(pip show *)", + "Read(//scratch/markymoo/**)", + "Read(//scratch/markymoo/gnina/**)", + "Bash(/scratch/markymoo/gnina/run_gnina.sh --version)", + "Bash(pkill -f \"active_learning.py --cfg /scratch/markymoo/al_probe.gin\")", + "Bash(mkdir -p .claude/skills)", + "Bash(mv .claude/experiment-log .claude/skills/experiment-log)", + "Bash(bash -n research/preprocessing/pose_selection_ablation/submit_pose_ablation.sh)", + "Bash(bash -n research/preprocessing/pose_selection_ablation/collect_results.sh)", + "WebFetch(domain:ar5iv.labs.arxiv.org)", + "Bash(sinfo -o \"%P %a %l %D %G\")", + "Bash(squeue -u markymoo)", + "Bash(squeue -j 69425 -o \"%.10i %.2t %.50R\")", + "Bash(sinfo -p compute -o \"%P %a %D %T %C %G\")", + "Bash(scontrol show *)", + "Bash(sinfo -o \"%P %a %D %T %l\")", + "Bash(sinfo -o \"%.18P %.6a %.6D %.10T\")", + "Bash(echo \"EXIT=$?\")", + "Read(//home/markymoo/projects/RGFN_Fork/docking_6td3/**)", + "Read(//home/markymoo/projects/RGFN_Fork/test-data/**)", + "Bash(sinfo -p debug_full_node)", + "Bash(sinfo)", + "Bash(sbatch research/preprocessing/pose_selection_ablation/submit_pose_ablation.sh)", + "Bash(squeue *)", + "Bash(git fetch *)", + "Bash(git switch *)", + "Bash(python -c \"from rgfn.gfns.reaction_gfn.proxies.docking_proxy.docking_proxy import DockingMoleculeProxy; print\\('OK import'\\)\")", + "Bash(python -c \"import glue; from glue.oracles.docking_6td3_oracle import Docking6TD3Oracle; print\\('OK glue import'\\)\")", + "Bash(python -c \"import meeko; print\\('meeko OK'\\)\")", + "Bash(bash -n scripts/hpc/submit_al_seh.sh)", + "Bash(chmod +x scripts/hpc/submit_al_seh.sh scripts/make_seh_seed.py)", + "Bash(bash -n research/active_learning_6td3/submit_al_6td3_mini.sh)", + "Bash(chmod +x research/active_learning_6td3/submit_al_6td3_mini.sh)", + "Bash(sed -i 's#experiments/active_learning_6td3/seed_6td3.csv#experiments/active_learning/6td3/seed_6td3.csv#' configs/glue/active_learning_6td3.gin configs/glue/active_learning_6td3_inner.gin configs/glue/active_learning_6td3_mini.gin)", + "Bash(sed -i 's#experiments/active_learning_6td3/seed_mock.csv#experiments/active_learning/mock/seed_mock.csv#' configs/glue/active_learning_mock.gin)", + "Bash(sed -i 's#experiments/active_learning_seh/seed_seh.csv#experiments/active_learning/seh/seed_seh.csv#' configs/glue/active_learning_seh.gin)", + "Bash(sed -i 's#research/preprocessing/docking_6td3/#experiments/oracle_validation/docking_6td3/#' configs/glue/active_learning_6td3.gin)", + "Bash(sed -i 's#research/active_learning_seh/make_seh_seed.py#experiments/active_learning/seh/make_seh_seed.py#g' configs/glue/active_learning_seh.gin)", + "Bash(sed -i 's#os.path.join\\(HERE, \"..\", \"test-data\", \"DDB1_CDK12_Glues.csv\"\\)#os.path.join\\(HERE, \"..\", \"..\", \"..\", \"data\", \"validation-molecules\", \"DDB1_CDK12_Glues.csv\"\\)#' __TRACKED_VAR__/docking_6td3/dock_cluster.py)", + "Bash(sed -i 's#os.path.join\\(HERE, \"..\", \"test-data\", \"CRBN_GSPT1_Glues.csv\"\\)#os.path.join\\(HERE, \"..\", \"..\", \"..\", \"data\", \"validation-molecules\", \"CRBN_GSPT1_Glues.csv\"\\)#' __TRACKED_VAR__/docking_crbn/dock_cluster_crbn.py)", + "Bash(sed -i 's#DOCK6TD3 = os.path.join\\(HERE, \"..\", \"docking_6td3\"\\)#DOCK6TD3 = os.path.join\\(HERE, \"..\", \"..\", \"oracle_validation\", \"docking_6td3\"\\)#' experiments/ablations/pose_selection/dock_allposes.py)", + "Bash(sed -i 's#os.path.join\\(DOCK6TD3, \"..\", \"test-data\", \"DDB1_CDK12_Glues.csv\"\\)#os.path.join\\(HERE, \"..\", \"..\", \"..\", \"data\", \"validation-molecules\", \"DDB1_CDK12_Glues.csv\"\\)#' experiments/ablations/pose_selection/dock_allposes.py)", + "Bash(sed -i 's#os.path.join\\(HERE, \"docking_gnina\"\\)#os.path.join\\(HERE, \"docking_crbn\"\\)#' __TRACKED_VAR__/compare_systems.py)", + "Bash(sed -i 's#research/preprocessing/docking_6td3/dock_cluster.py#experiments/oracle_validation/docking_6td3/dock_cluster.py#; s#default to ``research/preprocessing/docking_6td3/``#default to ``experiments/oracle_validation/docking_6td3/``#' glue/oracles/docking_6td3_oracle.py)", + "Bash(sed -i 's#`research/preprocessing/test-data/`#`data/validation-molecules/`#' glue/datasets/__init__.py)", + "Bash(git restore *)", + "Bash(grep -n \"docking_gnina\\\\|test-data\\\\|models/\\\\|^models\\\\|cif in\\\\|in \\\\`models\" experiments/oracle_validation/README.md)", + "Bash(sed -i 's#research/active_learning_seh/submit_al_seh.sh#experiments/active_learning/seh/submit_al_seh.sh#g; s#research/preprocessing/pose_selection_ablation#experiments/ablations/pose_selection#g' docs/RESEARCH_CONTEXT.md)", + "Bash(sed -i 's#scripts/hpc/submit.sh#scripts/submit.sh#g' docs/PATCHES.md)", + "Bash(sed -i 's#research/active_learning_seh#experiments/active_learning/seh#g; s#experiments/active_learning_seh/seed_seh.csv#experiments/active_learning/seh/seed_seh.csv#g' Logs/010_seh-gpu-docking-oracle.md)", + "Bash(echo \"exit=$?\")", + "Bash(sinfo -N -o \"%N %t %P\")", + "Bash(sbatch --exclude=balam008 /home/markymoo/projects/RGFN_Fork/RGFN-Fork/experiments/active_learning/6td3/submit_al_6td3_mini.sh)", + "Bash(sacct -j 69445 --format=JobID,State,Elapsed,ExitCode)", + "Bash(sacct -j 69445 --format=JobID,JobName,State,Elapsed,ExitCode)", + "Bash(sacct -j 69445)", + "Bash(python3 -)", + "Bash(python3 -m py_compile glue/datasets/oracle_labeled.py)", + "Bash(python3 -m py_compile glue/oracles/step_timing.py glue/oracles/docking_6td3_oracle.py glue/active_learning/loop.py)", + "Bash(chmod +x experiments/active_learning/6td3/submit_al_6td3_pregpu.sh)", + "Bash(bash -n experiments/active_learning/6td3/submit_al_6td3_pregpu.sh)", + "Bash(sbatch experiments/active_learning/6td3/submit_al_6td3_pregpu.sh)", + "Bash(sacct -j 69450 --format=JobID,State,Start,Elapsed,End -n)", + "Bash(sacct -j 69450 --format=JobID,State,ExitCode,Reason%40,WorkDir%60 -n)", + "Bash(xargs ls -la)", + "Bash(sacct -j 69451 --format=JobID,State,Elapsed,NodeList -n)", + "Bash(timeout 20 ssh -o ConnectTimeout=12 -o BatchMode=yes balam 'echo BALAM_OK; squeue -u markymoo 2>/dev/null; echo \"--- job 69451 ---\"; sacct -j 69451 --format=JobID,JobName,State,Elapsed,End 2>/dev/null | head')", + "Bash(/home/markymoo/miniconda3/envs/rgfn/bin/python -c \"import meeko; print\\('meeko', meeko.__version__\\)\")", + "Bash(/home/markymoo/miniconda3/envs/rgfn/bin/obabel -V)", + "Bash(/home/markymoo/miniconda3/envs/rgfn/bin/python *)", + "Bash(sacct -j 69451 --format=JobID,State,Elapsed,ExitCode -n)", + "Bash(mkdir -p /scratch/markymoo/rgfn_runs/experiments/gpu_pose_gen)", + "Bash(bash -n experiments/ablations/gpu_pose_gen/submit_gpu_pose_gen.sh)", + "Bash(sbatch experiments/ablations/gpu_pose_gen/submit_gpu_pose_gen.sh)", + "Bash(awk -F, '{print $2,$4}')", + "Bash(scancel 69468)", + "Bash(mv gpu_pose_gen_results.csv gpu_pose_gen_results.computenode_failed.csv)", + "Bash(mv gpu_pose_gen_timing.csv gpu_pose_gen_timing.computenode_failed.csv)", + "Bash(bash -n experiments/ablations/gpu_pose_gen/run_login_loop.sh)", + "Bash(nohup bash experiments/ablations/gpu_pose_gen/run_login_loop.sh)", + "Bash(echo \"launched login loop, pid $!\")", + "Bash(cp /tmp/claude-3160042/-home-markymoo-projects-RGFN-Fork-RGFN-Fork/d92e50a2-376d-44f4-8693-cdeffde9db09/scratchpad/qv2_diag.sh /scratch/markymoo/qv2_diag.sh)", + "Bash(timeout 560 srun --account=ac-sdl --partition=compute --gpus-per-node=1 --time=00:12:00 bash /scratch/markymoo/qv2_diag.sh)", + "Bash(timeout 560 srun --account=ac-sdl --partition=compute --gpus-per-node=1 --time=00:10:00 bash /scratch/markymoo/qv2_diag2.sh)", + "Bash(timeout 560 srun --account=ac-sdl --partition=compute --gpus-per-node=1 --time=00:10:00 bash /scratch/markymoo/qv2_diag3.sh)", + "WebFetch(domain:docs.scinet.utoronto.ca)", + "Bash(timeout 560 srun --account=ac-sdl --partition=compute --gpus-per-node=1 --time=00:10:00 bash /scratch/markymoo/qv2_diag4.sh)", + "Bash(timeout 560 srun --account=ac-sdl --partition=compute --gpus-per-node=1 --time=00:06:00 bash /scratch/markymoo/cltest_run2.sh)", + "Bash(CUDA_VISIBLE_DEVICES=0 /scratch/markymoo/cltest)", + "Bash(timeout 560 srun --account=ac-sdl --partition=compute --gpus-per-node=1 --time=00:05:00 --exclude=balam008 bash -c 'echo \"NODE=$\\(hostname\\)\"; CUDA_VISIBLE_DEVICES=0 /scratch/markymoo/cltest')", + "Bash([ -f \"$OUT/discrimination_stats.csv\" ])", + "Bash(sacct -j 69477 --format=JobID,State,ExitCode,Elapsed)", + "Bash(awk -F, '{print $4}')", + "Bash(awk -F, '$2==\"qv2_dock\"{s+=$3;n+=$4} END{printf \"qv2_dock total %.1fs over %d mols = %.2f s/mol\\\\n\",s,n,s/n}' gpu_pose_gen_timing.csv)", + "Bash(awk -F, 'NR>1{s[$2]+=$3} END{for\\(k in s\\) printf \" %-14s %.1fs\\\\n\",k,s[k]}' gpu_pose_gen_timing.csv)", + "Bash(sed -n '/## Relevant Files/,/## Relevant Versions/p' Logs/012_pre-gpu-docking-substep-timing.md)", + "Bash(sed -n '/## Method/,/## Results/p' Logs/012_pre-gpu-docking-substep-timing.md)", + "Bash(pdftotext koziarski2024rgfn.pdf -)", + "Bash(pdftotext bengio2021gflownet.pdf -)", + "Bash(bash -n experiments/active_learning/6td3/submit_al_6td3_gpu.sh)", + "Bash(sinfo -o \"%P %a %l %D %t\")", + "Bash(sbatch *)", + "mcp__claude_ai_Claude_Code_Remote__send_later", + "Bash(python3 -c \"import pypdf; print\\('pypdf', pypdf.__version__\\)\")", + "Bash(python3 -c \"import PyPDF2; print\\('PyPDF2'\\)\")", + "Bash(python3 -c \"import pdfplumber; print\\('pdfplumber'\\)\")", + "Bash(python3 -c \"import fitz; print\\('pymupdf'\\)\")", + "WebFetch(domain:arxiv.org)", + "Bash(module spider *)", + "Bash(module avail *)", + "Read(//scinet/balam/**)", + "Bash(CUDA_VISIBLE_DEVICES=0 /scratch/markymoo/vina_gpu/opencl_healthcheck)", + "Bash(ldd /scratch/markymoo/vina_gpu/opencl_healthcheck)", + "Bash(ldd /scratch/markymoo/vina_gpu/Vina-GPU-2.1/QuickVina2-GPU-2.1/QuickVina2-GPU-2-1)", + "Bash(sacctmgr -n show assoc user=markymoo format=account%30,qos%40,maxwall)", + "WebFetch(domain:docs.alliancecan.ca)", + "Bash(echo \"=== run dir AL contents ===\")", + "Bash(ls -la /scratch/markymoo/rgfn_runs/experiments/active_learning/6td3_gpu/2026-06-29_16-31-16/active_learning/)", + "Bash(echo \"=== suggestions ===\")", + "Bash(ls -la /scratch/markymoo/rgfn_runs/experiments/active_learning/6td3_gpu/2026-06-29_16-31-16/active_learning/suggestions/)", + "Bash(echo \"=== job .out tail ===\")", + "Bash(awk -F, 'NR==1{for\\(i=1;i<=NF;i++\\)print i,$i}' /scratch/markymoo/rgfn_runs/experiments/active_learning/6td3_gpu/2026-06-29_16-31-16/active_learning/suggestions/candidates.csv)", + "Bash(awk -F, 'NR>1{print $27}' /scratch/markymoo/rgfn_runs/experiments/active_learning/6td3_gpu/2026-06-29_16-31-16/active_learning/suggestions/candidates.csv)", + "Bash(awk 'NR==1{min=$1} {a[NR]=$1; sum+=$1} END{print \"min\",min,\"max\",a[NR],\"mean\",int\\(sum/NR\\),\"n\",NR}')", + "Read(//home/markymoo/bin/**)", + "Bash(nvidia-smi -L)", + "Bash(awk -F, 'NR>1{print $10\"\\\\t\"$2}' /scratch/markymoo/rgfn_runs/experiments/active_learning/6td3_gpu/2026-06-29_16-31-16/active_learning/suggestions/candidates.csv)", + "Bash(python3 *)", + "WebFetch(domain:github.com)", + "Bash(timeout 15 git ls-remote --heads https://github.com/koziarskilab/SCENT.git)", + "Bash(bash -n external/setup_rxnflow.sh)", + "Bash(bash -n experiments/active_learning/rxnflow_6td3/submit_rxnflow_6td3.sh)", + "Bash(ls validation/configs/)", + "Bash(cat validation/configs/fraggfn_6td3.yaml)", + "Bash(cat experiments/active_learning/fraggfn_6td3/submit_fraggfn_6td3.sh)", + "Bash(head -3 experiments/active_learning/6td3/seed_6td3.csv)", + "Bash(wc -l experiments/active_learning/6td3/seed_6td3.csv)", + "Bash(bash -n external/setup_aizynthfinder.sh)", + "Bash(awk '{print $1, $NF}')", + "Bash(chmod +x external/setup_aizynthfinder.sh)", + "Bash(bash -n external/setup_scent.sh)", + "Bash(bash -n experiments/active_learning/scent_6td3/submit_scent_6td3.sh)", + "Bash(chmod +x external/setup_scent.sh experiments/active_learning/scent_6td3/submit_scent_6td3.sh)", + "Bash(bash -n experiments/active_learning/seh/submit_al_seh.sh)", + "Bash(sinfo -h -o \"%P %D %t %N\")", + "Bash(module load *)", + "Bash(cat)", + "Read(//tmp/**)", + "Bash(timeout 300 conda run -n scent python /tmp/scent_import_check.py)", + "Bash(nohup bash external/setup_aizynthfinder.sh)", + "Bash(echo \"launched setup PID $!\")", + "Bash(bash external/setup_aizynthfinder.sh 2>&1)", + "Bash(chmod +x /tmp/run_scent_smoke.sh)", + "Bash(nohup /tmp/run_scent_smoke.sh)", + "Bash(echo \"launched pid=$! ; log=/tmp/scent_smoke.log\")", + "Bash(pkill -f run_scent_al.py)", + "Bash(pkill -f run_scent_smoke)", + "Bash(rm -f /tmp/scent_smoke.log)", + "Bash(echo \"relaunched pid=$!\")", + "Bash(zcat data/building_blocks/zincfrag_10k.smi.gz)", + "Bash(ps -u markymoo -o pid,ppid,etime,rss,cmd --sort=start_time)", + "Bash(pkill -9 -f \"setup_aizynthfinder.sh\")", + "Bash(pkill -9 -f \"download_public_data\")", + "Bash(ps -u markymoo -o pid,ppid,etime,cmd)", + "Bash(scancel 69512)", + "Bash(rm -rf /scratch/markymoo/rgfn_runs/_seedtest)", + "Bash(rm -rf wandb/offline-run-20260630_194818-*)", + "Bash(grep -nE 'SCENT-AL\\\\] round [0-9]+: \\(fit M|sampled|\\\\|D\\\\|=\\)|score_batch\\\\] scored|SCENT-AL\\\\] done|finalized' __TRACKED_VAR__.out)", + "Bash(sacct -j 69511 --format=JobID,State,ExitCode,Elapsed,NodeList -n)", + "Bash(nvidia-smi --query-gpu=index,memory.used,memory.free,memory.total --format=csv)", + "Bash(sinfo -p compute -o \"%P %l %L %D %t\")", + "Bash(rm -rf rxnflow_env)", + "Bash(mv rxnflow_env_real rxnflow_env)", + "Bash([ -s \"$SP/synth_eval.log\" ])", + "Bash(break)", + "Bash(./bin/true)", + "Bash(awk '{print $6,$7,$8}')", + "Bash(ulimit -u)", + "Bash(echo \"ulimit -u \\(max user procs\\): $\\(ulimit -u\\)\")", + "Bash(sacct -j 69517 --format=JobID,JobName,Elapsed,State,ExitCode)", + "Bash(sacct -j 69517 --format=JobID,Elapsed,State,ExitCode)", + "Bash(command -v pre-commit)", + "Bash(pre-commit --version)", + "Bash(echo \"exit: $?\")", + "Bash(pre-commit run *)", + "Bash(sed -i 's/^### /## /' 015_fraggfn-6td3-baseline.md 018_synthesizability-metric-aizynthfinder.md)", + "Bash(grep -nE '^#{1,3} ' 015_fraggfn-6td3-baseline.md)", + "Bash(grep -nE '^#{1,3} ' 018_synthesizability-metric-aizynthfinder.md)", + "Bash(which pdftotext *)", + "Bash(python3 -c \"import fitz; print\\('pymupdf', fitz.__doc__\\)\")", + "Bash(python3 -c \"import pdfminer; print\\('pdfminer'\\)\")", + "Bash(xargs basename -a)", + "Bash(sinfo -o \"%.14P %.6D %.14F %.10l\")", + "Bash(bash -n experiments/fixed_reward/submit_aizynth_fourway.sh)", + "Bash(sacct --starttime 2026-07-01 --format=JobID,JobName%18,State,ExitCode,Elapsed)", + "Bash(sort -k3)", + "Bash(awk 'NR>=35 && NR<=106 && /\\\\\\(\"/' /home/markymoo/projects/RGFN_Fork/RGFN-Fork/external/gflownet/src/gflownet/models/bengio2021flow.py)", + "Bash(sinfo -p compute)", + "Bash(grep -c SMOKE_OK __TRACKED_VAR__/smoke.log)", + "Bash(chmod +x experiments/fixed_reward/stdlib_seh/*.sh)", + "Bash(bash -n experiments/fixed_reward/stdlib_seh/submit_fixed_rgfn_stdlib_seh.sh)", + "Bash(bash -n experiments/fixed_reward/stdlib_seh/submit_fixed_rxnflow_stdlib_seh.sh)", + "Bash(sacct -j 69606,69608 --format=JobID,JobName%22,State,Elapsed,End)", + "Bash(sacct --starttime 2026-07-01 --format=JobID,JobName%20,State,ExitCode,Elapsed)", + "Bash(ssh -o StrictHostKeyChecking=no balam004 nvidia-smi --query-gpu=index,utilization.gpu,memory.used --format=csv,noheader)", + "Bash(sinfo -p compute -o \"%.15P %.15l %.6D %.6t\")", + "Bash(sacctmgr -n show assoc user=markymoo format=account,qos,maxwall,grpwall)", + "Bash(sacctmgr -n show qos format=name,maxwall,maxtres%20)", + "Bash(scancel 69613)", + "Bash(scancel 69615)", + "Bash(sacct -j 69606,69607,69608,69609,69610,69611,69612 --format=JobID,JobName%20,State,Elapsed)", + "Bash(sacct -j 69616,69614,69608,69606 --format=JobID,JobName%22,State,Elapsed,End)", + "Bash(:)", + "Bash(grep -n \"cost\\\\|price\\\\|\\\\$\\\\|USD\\\\|per_\" docs/CHEM_LIBRARY_FORMAT.md)", + "Bash(timeout 180 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python -c \"import torch,rdkit,gin,numpy; import rgfn; import glue; print\\(\\\\\"IMPORT OK torch\\\\\",torch.__version__,\\\\\"cuda\\\\\",torch.cuda.is_available\\(\\)\\)\"')", + "Bash(timeout 900 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python scripts/analyze_gfn.py --cfg configs/glue/fixed_reward_seh_proxy_stdlib.gin --checkpoint_path '\\\\''__TRACKED_VAR__'\\\\'' --library data/libraries/glue_standard_v1 --hub_selectors highest_flow,most_modes,highest_expected_reward --mol_selectors top_k_reward,top_k_reward_diverse --m_values 4,8 --k_values 5,10 --n_trials 2 --sample_size 500 --batch_size 250 --expander observed --out '\\\\''__TRACKED_VAR__/analysis_smoke_observed'\\\\'' --write_plans')", + "Bash(timeout 120 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python -c \" *)", + "Bash([ -f \"$SP/results.csv\" ])", + "Bash(timeout 120 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python '\\\\''__TRACKED_VAR__/check_pareto.py'\\\\'' '\\\\''__TRACKED_VAR__/analysis_smoke_observed/results.csv'\\\\''')", + "Bash(timeout 900 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python scripts/analyze_gfn.py --cfg configs/glue/fixed_reward_seh_proxy_stdlib.gin --checkpoint_path '\\\\''__TRACKED_VAR__'\\\\'' --library data/libraries/glue_standard_v1 --hub_selectors highest_flow --mol_selectors top_k_reward,top_k_reward_diverse --m_values 3 --k_values 25 --n_trials 1 --sample_size 400 --batch_size 200 --expander enumerative --expander_kwargs '\\\\''{\"max_fragments_per_pattern\": 25, \"max_children\": 300}'\\\\'' --out '\\\\''__TRACKED_VAR__/analysis_smoke_enum'\\\\''')", + "Bash(timeout 60 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python - </dev/null 2>&1 && python -c \"import glue.analysis; print\\(\\\\\"reimport OK\\\\\"\\)\"')", + "Bash(timeout 900 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python scripts/analyze_gfn.py --cfg configs/glue/fixed_reward_seh_proxy_stdlib.gin --checkpoint_path '\\\\''__TRACKED_VAR__'\\\\'' --library data/libraries/glue_standard_v1 --hub_selectors highest_flow --mol_selectors top_k_reward,top_k_reward_diverse,scaffold_diverse_k --m_values 3 --k_values 20 --n_trials 1 --sample_size 400 --batch_size 200 --expander enumerative --expander_kwargs '\\\\''{\"max_fragments_per_pattern\": 25, \"max_children\": 300}'\\\\'' --out '\\\\''__TRACKED_VAR__/analysis_smoke_enum2'\\\\''')", + "Skill(dataviz)", + "Bash(timeout 90 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python -c \"import matplotlib; print\\(\\\\\"matplotlib\\\\\", matplotlib.__version__\\)\"')", + "Bash(timeout 120 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python - </dev/null 2>&1 && python scripts/analyze_gfn.py --cfg configs/glue/fixed_reward_seh_proxy_stdlib.gin --checkpoint_path '\\\\''__TRACKED_VAR__'\\\\'' --library data/libraries/glue_standard_v1 --hub_selectors highest_flow,most_modes --mol_selectors top_k_reward,top_k_reward_diverse --m_values 2,4,8 --k_values 5,15,30 --n_trials 2 --sample_size 600 --batch_size 300 --expander enumerative --expander_kwargs '\\\\''{\"max_fragments_per_pattern\": 25, \"max_children\": 300}'\\\\'' --out '\\\\''__TRACKED_VAR__/analysis_sweep_enum'\\\\''')", + "Bash(timeout 100 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python - </dev/null 2>&1 && python '\\\\''__TRACKED_VAR__/../check_pareto.py'\\\\'' '\\\\''__TRACKED_VAR__/results.csv'\\\\'' 2>/dev/null | sed -n '\\\\''1,3p;/front/,$p'\\\\''')", + "Bash(timeout 120 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python - <<'\\\\''PY'\\\\'' *)", + "Bash(sed -n 1,40p __TRACKED_VAR__/tb_text.txt)", + "Bash(sed -n '/===== PAGE 2 =====/,/===== PAGE 3 =====/p' __TRACKED_VAR__/tb_text.txt)", + "Bash(sed -n '/===== PAGE 3 =====/,/===== PAGE 4 =====/p' __TRACKED_VAR__/tb_text.txt)", + "Bash(timeout 1200 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python scripts/analyze_gfn.py --cfg configs/glue/fixed_reward_seh_proxy_stdlib.gin --checkpoint_path '\\\\''__TRACKED_VAR__'\\\\'' --library data/libraries/glue_standard_v1 --hub_selectors highest_flow,highest_tb_flow --mol_selectors top_k_reward_diverse --m_values 8 --k_values 15 --n_trials 2 --sample_size 800 --batch_size 400 --expander observed --compute_tb_flow --out '\\\\''__TRACKED_VAR__/analysis_tbflow'\\\\''')", + "Bash(timeout 600 bash -c 'source ~/bin/rgfn-smoke-env.sh >/dev/null 2>&1 && python '\\\\''__TRACKED_VAR__/tb_diag.py'\\\\'' 100 > '\\\\''__TRACKED_VAR__/tb_diag.out'\\\\'' 2>&1')", + "Bash(grep -q 'DONE\\\\|Error\\\\|Traceback' __TRACKED_VAR__/tb_diag.out)", + "Bash(sacct -j 69690 --format=JobID,State,Elapsed)", + "Bash(sacct -j 69695,69696,69697,69698,69699,69700,69701,69702,69703,69704 -n -o JobID,JobName%22,State,Elapsed,ExitCode)", + "Bash(git -C /home/markymoo/projects/RGFN_Fork/RGFN-Fork status --short)", + "Bash(git -C /home/markymoo/projects/RGFN_Fork/RGFN-Fork log --oneline -5)", + "Bash(awk '{print $NF}')" + ] + } +} diff --git a/.claude/skills/experiment-log/SKILL.md b/.claude/skills/experiment-log/SKILL.md new file mode 100644 index 0000000..dcad1a0 --- /dev/null +++ b/.claude/skills/experiment-log/SKILL.md @@ -0,0 +1,210 @@ +--- +name: experiment-log +description: Write structured experiment logs for research projects. Use at the START of any computational experiment (docking runs, MD simulations, oracle validation, pipeline tests, discrimination checks, compute benchmarks) to create a log stub capturing the goal and context. Use at the END of an experiment to write up the full results, file inventory, and connection to the publication. Also trigger when the user says "start a log", "log this experiment", "write up the results", "document what we did", "wrap this up", or "create an experiment entry". If you are at the conclusion of an experimental session and have results, files, and commands to document — write the log without waiting to be asked. +--- + +# Experiment Log Skill + +Each log has two layers that serve different readers: + +**Story layer** — Question, Context & Summary, Answer, Relevance to our Publication, Next Experiments. Written for a smart person who isn't an expert on this specific project. Prioritize being understandable and engaging over being technically precise. Tell the story of what happened and why it matters. + +**Audit layer** — everything from `# Re-creation` onwards (Relevant Files, Relevant Versions, Relevant Resources, Method, Results). Written for reproducibility. Technical accuracy matters here. Scientific terms, file paths, and exact commands belong in this layer. + +If you're unsure which layer a detail belongs in: if it helps a reader understand the experiment's purpose and outcome → Story. If it helps someone reproduce or audit the experiment → Audit. + +In the story layer: stick to facts stated explicitly in the conversation or in `RESEARCH_CONTEXT.md`. Don't infer technical explanations for *why* something works or fails — just state what happened and what it means. If you're uncertain about a technical detail, leave it out rather than guessing. + +--- + +## Step 0: Orient yourself + +Read two files before writing anything: + +1. **`Logs/RESEARCH_CONTEXT.md`** — the paper's central claim, current project status, what journals will want, and key terminology. This tells you what the experiment is *for*. If it doesn't exist, ask the user to describe the research context. + +2. **`Logs/README.md`** — the index of past experiments. Reading it tells you what's established so you can say things like "as shown in entry 002" accurately. + +--- + +## Step 1: Determine mode + +**START mode** — experiment hasn't run yet. Create a stub. +**END mode** — experiment is complete. Fill everything in, or find a stub and complete it. + +To find an existing stub: `grep -rl "\[TODO" Logs/ HistoricLogs/ 2>/dev/null` + +--- + +## Step 2: Find the next log number + +```bash +ls Logs/*.md 2>/dev/null | grep -Eo '/[0-9]+_' | grep -Eo '[0-9]+' | sort -n | tail -1 +``` + +Increment by 1, zero-pad to 3 digits: `005`, `006`, etc. If the user specifies a different output directory (e.g., "write to HistoricLogs"), use that directory but the same numbering. + +--- + +## Step 3: Write the log + +**Filename:** `NNN_kebab-case-description.md` (3-6 words, e.g., `006_brd4-vhl-generalization-test`) + +### Header + +``` +# [System] — [brief description] +**Date:** YYYY-MM-DD, ~[rough time, e.g., "10am"] +``` + +Do NOT include the log number in the title — it's already in the filename. + +--- + +### Question + +*(Story layer. Write this last if needed — it's the hardest to get right.)* + +One sentence. The test: can someone skim this and immediately understand what we were trying to find out? No acronyms, no method names. Just the question. + +Don't add implications or "could this also mean X?" — those belong in Context. Keep it clean. + +> **Good:** "Can our scoring method tell real molecular glues apart from randomly generated molecules across multiple protein systems?" +> **Good:** "Can we use the stability of the protein complex over a short simulation to score molecular glue candidates better than a static docking score can?" +> **Avoid:** "Does the DDB1 neosubstrate differential discriminate in the 7ABC system with exhaustiveness 16, and could this replace or improve on our current oracle pipeline?" + +--- + +### Context & Summary + +*(Story layer. Explain numbers in plain terms. Name specific venues. Don't pre-explain failure scenarios.)* + +Two parts: + +**Context** — Why is this experiment necessary? What earlier result or open question makes this the next logical step? Reference prior log entries by number. When citing numbers from prior entries, explain what they mean in plain terms — don't assume the reader remembers. Example: "Entry 002 showed we can tell real glues from randomly generated molecules with about 78% accuracy for the CDK12-DDB1 system." Keep to 3-5 sentences. + +**Summary** — What are we going to do? One short paragraph in plain English. If introducing a new control or comparison group, describe its PURPOSE ("randomly generated molecules that act as our negative control") before using a shorthand term for it. Don't explain what we'll do if things go wrong — focus on what we're trying to achieve. + +--- + +### Answer + +*(Story layer. Write this before writing Results — it forces clarity. Focus on meaning, not numbers.)* + +2-3 sentences on what the results *mean*. What can we now say with confidence? What did this experiment add to our understanding? Numbers belong in Results; the interpretation belongs here. + +--- + +### Relevance to our Publication + +*(Story layer. Be specific about which venue and which reviewer concern this addresses.)* + +How does this experiment help us publish? Name the specific venue (NeurIPS, Nature, etc.) and the specific thing reviewers will look for. Example: "NeurIPS reviewers will ask whether our method works beyond a single protein system — this entry answers that directly with a second validated system." One short paragraph. + +--- + +### Next Experiments + +*(Story layer. Fit everything into the publication goal. Avoid jargon — describe purpose, not method.)* + +Two subsections: + +**Refining for publication** — What will reviewers still want? Things that make the existing result more airtight: additional trials, cleaner figures, ablations showing a design choice is necessary. Ablation experiments belong here, not in Next Steps. + +**Next steps in project** — What experiments come next in the project pipeline? Frame in terms of the publication goal: "Run RGFN with each validated oracle to show the model can generate good candidates for multiple systems." + +--- + +### `# Re-creation` + +This heading is **required** — it marks where the audit layer begins. Everything below this line is for reproducibility and auditing, not for storytelling. + +--- + +### Relevant Files + +*(Audit layer. Focus on role and narrative context — WHY this file, not just WHAT it is.)* + +State the root directory once if most files share a prefix. Use these path conventions: +- `./path` — relative to the project repository root +- `/path` — absolute path outside the repository (scratch dirs, SLURM logs) +- `path` — relative to the stated root + +For **scripts**: path + one-line description of what it does. + +For **non-script files** (models, datasets, results): path + description that explains its role in the pipeline. Focus on WHY this file is used and what it represents in context — e.g., "CDK12 receptor alone (Tier 1), used to isolate E3-pocket binding from neosubstrate cooperativity." You don't need to hedge ("believed to contain") — just describe what the file is. + +Categories (only include what applies, ordered by pipeline stage): +- **Scripts** +- **Models** +- **Datasets** +- **Results** +- **Job Logs** + +--- + +### Relevant Versions + +*(Audit layer.)* + +Run: +```bash +git log --oneline -5 +git status --short +``` + +**If relevant files are committed:** paste the most recent relevant commit hash and message. + +**If relevant files are NOT yet committed:** tell the user exactly which files need to be committed, leave a `[TODO — add commit hash after pushing]` placeholder, and explicitly ask: "Can you commit the experiment files? Once you do, let me know and I'll update this section with the commit hash." Then wait — if the user confirms, update the log. + +--- + +### Relevant Resources + +*(Audit layer.)* + +Sources consulted when designing or running the experiment. Two subsections: + +**Sources** — papers, databases, PDB entries (with DOIs or URLs where available) + +**Packages** — tools and libraries used, with the specific file(s) in this project that use them + +--- + +### Method + +*(Audit layer. Big commands only — not debugging steps or package installs.)* + +Numbered steps: what was run, what it operated on, what it produced. + +--- + +### Results + +*(Audit layer. Numbers that support the Answer section. Use tables for comparisons.)* + +Label clearly: n counts, metric names, units. If pulling numbers from a prior log entry (not generated in this experiment), note which entry they came from — e.g., "(from entry 002, job 69271)" — so future readers know not to re-derive them here. + +--- + +## Step 4: Update the README index + +Add one row to `Logs/README.md` (or the equivalent in the output directory): + +``` +| [NNN](NNN_filename.md) | YYYY-MM-DD | Brief title | One-sentence verdict | +``` + +The verdict should be punchy: someone skimming the index should immediately know the key finding. + +--- + +## Quality check + +- [ ] Title has no log number (it's in the filename) +- [ ] Date includes a rough time +- [ ] Question is one clean sentence, no jargon, no implications clause +- [ ] Context explains what numbers *mean*, not just what they are +- [ ] `# Re-creation` heading present +- [ ] No `[TODO]` markers in completed END-mode sections (except intentional commit-hash placeholder) +- [ ] README row added diff --git a/.claude/skills/experiment-log/evals/evals.json b/.claude/skills/experiment-log/evals/evals.json new file mode 100644 index 0000000..f5fe1db --- /dev/null +++ b/.claude/skills/experiment-log/evals/evals.json @@ -0,0 +1,23 @@ +{ + "skill_name": "experiment-log", + "evals": [ + { + "id": 1, + "prompt": "The working directory is /Users/mark/Documents/GitHub/RGFN-Fork.\n\nWe're about to start a new experiment. We want to test whether Molecular Dynamics (MD) simulations can serve as a better oracle than docking for the CDK12-DDB1 system. The hypothesis: the *stability* of the ternary complex over a short MD trajectory (measured by interface RMSD) will correlate better with being a real glue than a single static docking score. If so, MD stability could replace or augment docking as the RGFN reward signal — potentially fixing the ceiling we hit with the CRBN system.\n\nPlease start an experiment log for this upcoming experiment. Write the log to HistoricLogs/ (not Logs/) since we're testing the skill. Reference RESEARCH_CONTEXT.md and README.md in Logs/ for project context and log numbering.", + "expected_output": "A new .md file in HistoricLogs/ with the correct header, Question section (plain-language, no jargon), and Context & Summary filled in. All post-experiment sections (Answer, Results, etc.) should have [TODO] markers. The HistoricLogs/README.md index table should be updated.", + "files": [] + }, + { + "id": 2, + "prompt": "The working directory is /Users/mark/Documents/GitHub/RGFN-Fork.\n\nWe just wrapped up a generalization experiment. We tested our docking oracle on two new molecular glue systems to see if it works beyond CDK12-DDB1.\n\nSystem 1: IKZF1-CRBN — CRBN E3 ligase + IKZF1 neosubstrate + IMiD-class glue lenalidomide (PDB 6H0F). IKZF1 is a zinc-finger transcription factor recruited to CRBN by IMiD drugs.\nSystem 2: BRD4-VHL — VHL E3 ligase + BRD4 BD1 domain + a small molecule glue (PDB 7Q2J).\n\nProtocol: same Tier 1/Tier 2 neosubstrate differential pipeline as entry 002. Decoys: 200 random molecules with the correct warhead for each system (glutarimide for IKZF1-CRBN, VHL-binder hydroxamate for BRD4-VHL). Scripts: pre-processing/dock_cluster_multi.py on Balam (jobs 70142 for IKZF1-CRBN, 70143 for BRD4-VHL, ~18 min each).\n\nResults:\n- IKZF1-CRBN: known (n=180) median neosubstrate differential = −1.8 kcal/mol, decoys (n=200) median = −1.6 kcal/mol. Frac < −1.5: 71% known vs 68% decoy. Near-zero discrimination. Same ceiling pattern as 5HXB/CRBN in entry 001.\n- BRD4-VHL: known (n=95) median differential = −2.4 kcal/mol, decoys (n=200) median = −0.5 kcal/mol. Frac < −1.5: 82% known vs 9% decoy. Strong 73-point discrimination gap.\n\nKey files:\n- Script: pre-processing/dock_cluster_multi.py\n- Datasets: pre-processing/test-data/IKZF1_Glues.csv, pre-processing/test-data/BRD4_VHL_Glues.csv\n- Results: pre-processing/multi_system_results_ikzf1.csv, pre-processing/multi_system_results_brd4.csv\n- SLURM logs: /scratch/markymoo/rgfn_runs/dock_multi_70142/, /scratch/markymoo/rgfn_runs/dock_multi_70143/\n\nPlease write a complete experiment log. Write to HistoricLogs/ (not Logs/). Reference RESEARCH_CONTEXT.md and README.md in Logs/ for context and numbering.", + "expected_output": "A complete .md file in HistoricLogs/ with all sections filled in: Question (plain language), Context & Summary, Answer (meaning not just numbers), Relevance to Publication (explicit connection to paper), Next Experiments (two subsections), Relevant Files (categorized), Relevant Versions (git output or placeholder), Method (numbered steps), Results (table with numbers). No [TODO] markers. README.md updated.", + "files": [] + }, + { + "id": 3, + "prompt": "The working directory is /Users/mark/Documents/GitHub/RGFN-Fork.\n\nThere's an in-progress experiment log at HistoricLogs/007_ablation-tier2-vs-differential.md. Please complete it with the following results.\n\nWe ran the same 160 known glues and 248 decoys from entry 002 through the existing pipeline, comparing two discrimination metrics:\n- Metric A (ablation): Tier 2 absolute Vina score only\n- Metric B (our oracle): neosubstrate differential (Tier 2 − Tier 1, same pose)\n\nResults:\n| metric | known | decoy | gap |\n|---|---|---|---|\n| Tier 2 Vina (absolute) | median −10.15 | median −7.96 | 2.19 kcal/mol |\n| frac Tier2 < −10 | 55.6% | 3.6% | 52 pts |\n| neosubstrate differential | median −2.20 | median −0.60 | 1.60 kcal/mol |\n| frac diff < −1.5 | 85.6% | 7.3% | 78 pts |\n\nBoth metrics discriminate. The differential wins on the frac < −1.5 criterion (78 pts vs 52 pts) and has a cleaner biological interpretation (it isolates glue-specific arm contact rather than total E3-pocket binding).\n\nNo new experiments were run — this analysis reused the data from entry 002 job 69271. We added a --ablation flag to pre-processing/compare_systems.py to pull both metrics from the existing results CSVs (pre-processing/docking_6td3/known_results.csv and decoy_cdk_results.csv).\n\nReference RESEARCH_CONTEXT.md and README.md in Logs/ for project context.", + "expected_output": "The existing HistoricLogs/007_ablation-tier2-vs-differential.md file is updated in place with all [TODO] sections replaced. All sections complete: Answer, Relevance to Publication, Next Experiments, Files, Versions, Method, Results. No [TODO] markers remain. README.md updated with a verdict row for entry 007.", + "files": ["HistoricLogs/007_ablation-tier2-vs-differential.md"] + } + ] +} diff --git a/.gitignore b/.gitignore index e5bfc45..ec8a6d0 100755 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,15 @@ -scripts/ configs/grid_search/ +# Reference paper PDFs — local-only (copyrighted + binary; keep repo lean). +# Cite papers via Logs/references/README.md + references.bib instead. +Logs/references/pdfs/* +!Logs/references/pdfs/.gitkeep + +# Cloned upstream baseline code — installed by external/setup_*.sh, NOT vendored +# (e.g. external/gflownet for FragGFN). Keeps the setup scripts tracked, the heavy +# upstream trees out. See docs/ARCHITECTURE.md + validation/generators/README.md. +external/*/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -160,5 +169,66 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ -experiments/ ignored/ + +# Vina-GPU-2.1 build artifacts (compiled docking oracle) +vina_workspace/ +quickvina_dir + +# Large installers / binaries (do not commit to git) +Miniconda3-latest-Linux-x86_64.sh +*.run + +# Docking / structure data artifacts (regenerable from pre-processing scripts) +*.pdb +*.pdbqt +*.cif +*.sdf +*.csv + +# superseded scaffold library (large; not part of the curated pipeline) +data/validation-molecules/Enamine_CRBN_Molecular_Glue_Library_*.smiles + +# Docking run outputs (regenerable; written by the experiments/ docking scripts) +cluster_out/ +cluster_out_crbn/ +experiments/**/cluster_out*/ + +# Model checkpoints / weights (large; document provenance in data/models/README.md) +data/models/**/*.ckpt +data/models/**/*.pt +data/models/**/*.pth + +# AiZynthFinder public dataset (templates/policy/stock; large, fetched by +# external/setup_aizynthfinder.sh via download_public_data — never commit) +data/models/aizynthfinder/ + +# RxnFlow building-block environment (fingerprints/masks/descriptors + block list; +# large, generated/fetched by external/setup_rxnflow.sh — never commit) +data/models/rxnflow_env/ + +# Weights & Biases run output (generated locally; includes broken latest-run symlink) +wandb/ + +# Generated synthetic datasets (keep the dir + docs, ignore the generated outputs) +data/synthetic/* +!data/synthetic/README.md +!data/synthetic/.gitkeep + +# Curated validation-molecule inputs (override the global *.csv ignore above) +!data/validation-molecules/*.csv + +# violin plots (regenerable) +experiments/ablations/sixway/violins_known_vs_decoy.png + +# --- experiments/: one committed dir per run/experiment (code + seeds + small +# results), grouped by type {active_learning, oracle_validation, ablations}. Run +# OUTPUTS — timestamped run dirs (YYYY-MM-DD_*) and everything under them +# (checkpoints, logs, modes, per-round CSVs) — are regenerable and ignored by +# pattern; all committed code/seeds/results outside them stay tracked. --- +experiments/**/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]_*/ +# committed seed + result CSVs sit directly in a run dir (override the global *.csv ignore) +!experiments/**/seed_*.csv +!experiments/**/*_results.csv + +oracle/drd2_current.pkl diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..d2c0a7a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,116 @@ +# CLAUDE.md — repository guide for AI agents and contributors + +This is a research fork of **RGFN** (Reaction-GFlowNet) for generating novel +**molecular glue degraders**. We extend upstream RGFN with new oracles, reward +shaping, batch-selection strategies, benchmarks, and dataset tooling. + +**Read this file and `docs/ARCHITECTURE.md` before making structural changes.** +For the science/goals, read `docs/RESEARCH_CONTEXT.md`. + +--- + +## The one rule that governs everything: old vs. new + +| Path | Owner | Rule | +|---|---|---| +| `rgfn/` | **Upstream** | Pristine RGFN. **Do not edit.** Extend it from `glue/`. | +| `configs/` (except `configs/glue/`) | **Upstream** | Treat as pristine. New configs go in `configs/glue/`. | +| `gin_config/`, `train.py`, `grid_search.py`, `tests/` (upstream parts), `data/chemistry.xlsx`, `data/targets/`, `external/setup_{gnina,gneprop,reinvent,shared}.sh` | **Upstream** | Leave as-is. | +| `glue/` (**production pipeline**) | **Ours** | All new pipeline Python: oracles, rewards, samplers, proxies, datasets, the active-learning loop. The thing we ship. | +| `configs/glue/` | **Ours** | All new gin configs (overlay upstream via `include`). | +| `scripts/` | **Ours** | **Generic** launch layer only: pipeline-wide entry points (`train.py`/`infer.py` wrappers, `active_learning.py`) + generic `submit.sh`. Experiment-specific scripts go in `experiments///`, not here. | +| `validation/` (**validation**) | **Ours** | The whole comparative-evaluation world: baseline generators (SynFlowNet, FragGFN, VAE-BO, RGFN adapter), validation-only oracles (Boltz-2), benchmark suites (PMO + our own), the harness, and committed results. | +| `data/` | **Ours** (+ upstream files) | The single **inputs** dir. Ours: `data/models/` (structures/checkpoints), `data/validation-molecules/` (curated known-glue sets), `data/synthetic/` (generated, git-ignored). Upstream (don't move): `data/chemistry.xlsx`, `data/targets/`. | +| `experiments/` | **Ours** | One self-contained dir **per run/experiment**, grouped by type: `active_learning/`, `oracle_validation/`, `ablations/`. Holds that run's code + seeds + small results + README; timestamped run outputs land alongside (git-ignored). Reusable science graduates into `glue/`. See `experiments/README.md`. | +| `Logs/`, `docs/` | **Ours** | Experiment logs + project documentation. | +| `Logs/references/` | **Ours** | Canonical bibliography for papers we build on. Cite by key (`[koziarski2024rgfn]`); annotated index in its `README.md`. PDFs in `pdfs/` are git-ignored. | + +We keep `rgfn/` and `configs/` mergeable with upstream. The only deliberate edits +to upstream files are three small operational overrides documented in +**`docs/PATCHES.md`** — read that before "fixing" anything odd in those files. + +### The second axis: production pipeline vs. validation + +Beyond upstream-vs-ours, our own code splits along a second axis — the **training +pipeline** (`glue/`) vs. the **validation/benchmarking layer** (`validation/`) — +kept apart by one rule: + +> **The dependency arrow points one way.** `validation/` may import from `glue/` +> and `rgfn/`. The production pipeline — `glue/`, `scripts/train.py`, +> `configs/glue/` — must **never** import from `validation/`. + +This is what keeps slow validation-only oracles (Boltz-2, co-folding, MD) out of +the in-loop reward by construction, and keeps the shipped pipeline understandable +without dragging in every baseline. Everything comparative lives under the single +`validation/` umbrella (generators, oracles, `suites/`, `harness/`, `results/`); +baseline generators are **thin adapters** in `validation/generators/`, their heavy +upstream code installed via `external/setup_*.sh`, not vendored. See +`validation/README.md` and `docs/ARCHITECTURE.md` for the full picture. + +--- + +## How to extend the system (don't edit `rgfn/`) + +0. **Understand before you change.** Ground the work in the science *first*: read + `docs/RESEARCH_CONTEXT.md` (goals, the validated oracle/metric, terminology), + the relevant experiment logs in `Logs/`, and the source papers in + `Logs/references/`. Base each consequential choice — architecture, metric, sign + convention — on the file that *recorded* the decision (paper section, upstream + `rgfn/` code, or our own log/analysis script), **never** on what merely seems + plausible. Confirm the change is consistent with prior results and the + project's direction, and **ask the user to clarify** anything ambiguous before + building. A design that looks reasonable but silently diverges from the + publications or our findings is the costliest mistake here — and it passes + compile/import/smoke tests, so only this step catches it. +1. **New oracle** → implement the science in `glue/oracles/`. +2. **Expose it to training** → write a `@gin.configurable` adapter in + `glue/proxies/` subclassing upstream `CachedProxyBase`/`ProxyBase`. See + `glue/proxies/example_glue_proxy.py` for the working template. +3. **New reward / sampler** → `glue/rewards/` or `glue/samplers/`, subclassing the + upstream base classes in `rgfn/api/`. +4. **Register it** → make sure the module is imported by `glue/registry.py` + (directly or via its subpackage `__init__`). Gin finds classes by name only + after they are imported. +5. **Configure it** → add a gin config in `configs/glue/` that `include`s the + upstream base and references your class as `@YourClass`. +6. **Run it** → `python scripts/train.py --cfg configs/glue/.gin`. The + wrapper imports `glue` first so gin can resolve your component; the root + `train.py` (upstream) only knows about `rgfn`. + +If gin says *"No configurable matching @X"*, the defining module wasn't imported +on the startup path — fix `glue/registry.py`, not the config. + +--- + +## Project goals (what we're building toward) + +- New **oracles** (ternary docking / neosubstrate differential; possibly MD). +- New **rewards** (differential reward isolating glue cooperativity). +- Possibly expanded **batch selection**. +- **Benchmarks** comparing RGFN vs. baselines and across protein systems. +- **Models & datasets** as input; **synthetic datasets** as output. + +--- + +## Working notes for agents + +- **Compute:** Heavy stack (torch-geometric, openbabel, meeko, gnina) + GPU + docking runs on **Balam** (SciNet). Balam may be down; when it is, work on a + Mac laptop and **validate what you can locally** (imports, `py_compile`, gin + config-include integrity, `bash -n`) — full train/dock validation happens on + Balam. State clearly in your summary what you did vs. couldn't verify. +- **Login-node smoke tests (Balam *or* Trillium):** before any interactive smoke + test that imports `glue`/`rgfn` (pulls in dgl) or runs the GPU docking oracle, + prefix the command with `source ~/bin/rgfn-smoke-env.sh &&`. That one helper + activates the `rgfn` env and sets the `LD_LIBRARY_PATH` (torch-bundled CUDA libs + for dgl + QuickVina2-GPU boost libs) + `GNINA` — and works **unchanged on both + login nodes** (Balam is SciNet-legacy; **Trillium** is an Alliance cluster where + `module load cuda/11.8.0` does not exist). Balam and Trillium **share + `/scratch` + `/home`**, so the conda envs and `$SCRATCH/vina_gpu`/`gnina` builds + are identical from either. **Jobs still submit to Balam compute only** — the + `submit_*.sh` headers are Balam-specific (SLURM account/partition/`--exclude`) + and the helper does not touch them. +- **Document as you go:** record structural changes and anything left unverified + in `docs/REFACTOR_LOG.md` so the next agent can continue or repair the work. +- **Experiment logs:** use the `experiment-log` skill for any real computational + experiment. diff --git a/Logs/001_5hxb-crbn-anchored-docking.md b/Logs/001_5hxb-crbn-anchored-docking.md new file mode 100644 index 0000000..c29b711 --- /dev/null +++ b/Logs/001_5hxb-crbn-anchored-docking.md @@ -0,0 +1,101 @@ +# 5HXB / CRBN — warhead-anchored docking oracle + decoy control +**Date:** 2026-06-11 → 2026-06-18 + +## Question + +Can docking accurately distinguish real CRBN molecular glue degraders from realistic fake molecules? + +## Context & Summary + +**Context** — This is the project's first oracle validation experiment. Before RGFN can generate useful glue candidates, we need to confirm that our scoring function actually rewards the right molecular property — that it can tell real glues apart from decoys (molecules with the correct warhead but a random arm not selected for glue activity). The CRBN/GSPT1 system (PDB: 5HXB) was the natural starting point: it's one of the best-characterized molecular glue systems, with a known glue (CC-885) and a well-studied binding mechanism. The key structural challenge is that CRBN's warhead (glutarimide) binds in a deep tri-Trp cage that blind docking rarely penetrates — off-the-shelf docking never sampled the native pose in any configuration we tried. + +**Summary** — We developed a two-pass warhead-anchored docking protocol: pin the glutarimide to its crystal cage coordinates, then sample and relax the free arm. We docked ~188 real CRBN/GSPT1 glues and 260 warhead-bearing decoys (real warhead, random drug-like arm) using this protocol. For each molecule, we scored it against the E3 pocket alone (Tier 1, CRBN only) and the full ternary complex with neosubstrate (Tier 2, CRBN+GSPT1), measuring the neosubstrate differential to see whether real glues show more cooperativity than decoys. + +## Answer + +The anchored docking works mechanically — it places 95% of known glues into the CRBN cage with near-native accuracy, recovering CC-885's native pose to −13.55 kcal/mol. But it cannot discriminate real glues from decoys: the Vina score distributions largely overlap (median gap 0.85 kcal/mol), and the neosubstrate bonus is nearly identical for real glues and decoys. The structural reason: pinning the glutarimide warhead in the cage automatically positions every arm near the GSPT1 surface, regardless of whether the arm makes a productive glue contact. GSPT1 cooperativity in this system lives in a large protein–protein interface, not in a ligand-mediated contact the docking score can resolve. + +## Relevance to our Publication + +This entry is the negative control that justifies our system selection. For NeurIPS, reviewers will ask why we didn't use CRBN — the most studied molecular glue system. Entries 001–003 together provide a direct structural answer: CRBN's geometry makes the differential a nonspecific artifact rather than a glue signal, and we show this with a side-by-side comparison on identical methodology. Documenting the failure and its structural explanation makes the system choice principled rather than arbitrary. + +## Next Experiments + +**Refining for publication** — A molecular-weight sensitivity check on the CRBN known-glue dataset (filtering to MW < ~600 Da) to confirm the discrimination ceiling isn't partly a dataset artifact: `CRBN_GSPT1_Glues.csv` includes compounds up to MW ~1009, some of which are likely bivalent/PROTAC-range and won't behave as glues under docking. + +**Next steps in project** — Pivot to a system where the neosubstrate contact is ligand-mediated and sits inside the docking box: 6TD3/CDK12-DDB1, where CR8's arm directly contacts DDB1 (entry 002). Run both systems head-to-head on identical methodology to confirm the structural explanation (entry 003). + +# Re-creation + +## Relevant Files + +Scripts: +- `./research/preprocessing/clean.py` — carves 5HXB copy 1 into Tier 1 (CRBN only) and Tier 2 (CRBN+GSPT1) receptor structures; retains Zn, removes glue; outputs to `models/` +- `./docking_gnina/anchor_dock.py` — sequential per-molecule warhead-anchored docking (Pass A and B); used in this entry, later consolidated into `dock_cluster_crbn.py` +- `./docking_gnina/dock_cluster_crbn.py` — batched multi-GPU warhead-anchored docking driver (supersedes `anchor_dock.py`) +- `./docking_gnina/make_decoys.py` — generates decoys: IMiD glutarimide scaffold + random drug-like arms via amide/urea/sulfonamide/reductive-amination coupling +- `./research/preprocessing/compare_systems.py` — prints known-vs-decoy discrimination metrics for each system and cross-system + +Models: +- `./docking_gnina/5HXB_tier2.pdbqt` — CRBN+GSPT1 ternary receptor (Tier 2); used to score neosubstrate cooperativity +- `./docking_gnina/5HXB_tier1_CRBN.pdbqt` — CRBN pocket alone (Tier 1); baseline against which the differential is computed +- `./docking_gnina/crystal_85C.pdb` — reference CC-885 native pose; used to validate the anchored approach recovers the crystal geometry + +Datasets: +- `./research/preprocessing/test-data/Enamine_CRBN_Molecular_Glue_Library_..._4560cmpds_*.smiles` — purchasable scaffold library (potential glues, not validated degraders); used as known+ positives in this entry; superseded by the curated `CRBN_GSPT1_Glues.csv` in entry 003 + +Results: +- `./docking_gnina/batch_results_passB.csv` — known-glue docking scores, Pass B (validated pipeline) +- `./docking_gnina/batch_results.csv` — known-glue docking scores, Pass A (baseline) +- `./docking_gnina/decoy_results_passB.csv` — decoy docking scores, Pass B +- `./docking_gnina/decoy_results.csv` — decoy docking scores, Pass A + +## Relevant Versions + +``` +106a4e6 Add 6TD3/CR8 glue docking oracle + 5HXB cross-system comparison +7afd7f1 Add docking pre-processing pipeline, inference, and run scripts +``` + +Relevant commit: `106a4e6` — CRBN anchored-docking scripts (`anchor_dock.py`, `dock_cluster_crbn.py`, `make_decoys.py`) and receptor files committed here as part of the same PR as 6TD3 work. + +## Relevant Resources + +**Sources** +- 5HXB crystal structure: Matyskiela et al., *Science* 2016 — CRBN·DDB1·GSPT1·CC-885 ternary complex + +**Packages** +- gnina v1.3.2 (CNN-rescored docking) — `docking_gnina/`; launched via `/scratch/markymoo/gnina/run_gnina.sh` +- RDKit — conformer generation (`make_decoys.py`) +- OpenBabel — pdbqt conversion (`clean.py`) + +## Method + +1. **Structure prep** — `research/preprocessing/clean.py` carves 5HXB copy 1: `models/5HXB_tier1_CRBN.pdb` (CRBN only), `models/5HXB_tier2_CRBN_GSPT1.pdb` (CRBN+GSPT1). Zn retained, CC-885 removed. Receptors → pdbqt via `obabel ... -xr -p 7.4`. + +2. **Sampling failure (blind docking)** — Blind docking with Vina, gnina, and gnina with exhaustiveness 64 + CNN never samples CC-885's deep tri-Trp-cage pose (top score ~−8 kcal/mol, CNN ~0.5; all displaced from native). Native pose minimizes in place to −13.8 / CNN 0.98 — sampling failure, not scoring failure. + +3. **Warhead-anchored Pass B** — `docking_gnina/anchor_dock.py`: pin glutarimide to crystal cage coordinates, sample/relax only the arm via gnina `--minimize`, select clash-aware best pose with warhead-escape filter (poses >2.5 Å from cage rejected). Pass A (hard-frozen warhead): recovers native but 29–42% of molecules yield no clash-free pose. Pass B (flexible anchor, 1.5 Å wiggle + tether filter): CC-885 → −13.55 / CNN 0.97 (≈ native gold); fit-rate jumps to 94% known / 80% decoy. + +4. **Decoy control** — `docking_gnina/make_decoys.py` builds 260 IMiD-scaffold decoys. Known positives: 10% sample of purchasable Enamine CRBN library. Both sets docked with Pass B; distributions compared via `compare_systems.py`. + +## Results + +**CC-885 native-pose recovery:** + +| config | Vina (kcal/mol) | CNNaff | +|---|---|---| +| Crystal pose (in-place min) | −13.8 | 0.98 | +| Pass B anchor dock | −13.55 | 0.97 | +| Blind docking (best of exh 64) | ~−8 | ~0.5 | + +**Known-glue vs decoy discrimination (Pass B):** + +| metric | known (library sample) | decoy (n=260) | gap | +|---|---|---|---| +| fit rate | 94% | 80% | +14 pts | +| median Vina (kcal/mol) | −6.70 | −5.86 | −0.85 | +| median CNNaff | 6.21 | 6.09 | +0.12 | +| frac Vina < −10 | 15% | 7% | +8 pts | + +Vina-physics channel sharpened ~65% Pass A→B, but medians did not close → confirmed ceiling, not sampling noise. See entry 003 for the explicit neosubstrate-differential comparison using the same methodology as 6TD3. diff --git a/Logs/002_6td3-cr8-validation-and-discrimination.md b/Logs/002_6td3-cr8-validation-and-discrimination.md new file mode 100644 index 0000000..34742df --- /dev/null +++ b/Logs/002_6td3-cr8-validation-and-discrimination.md @@ -0,0 +1,103 @@ +# 6TD3 / CDK12-DDB1 — oracle validation + discrimination run +**Date:** 2026-06-18 + +## Question + +Can the docking score tell real CDK12-DDB1 molecular glue degraders apart from realistic fake molecules? + +## Context & Summary + +**Context** — Entry 001 showed that the CRBN system hits a discrimination ceiling: the docking score is equally good at placing real glues and random warhead-bearing molecules, because GSPT1 cooperativity comes from a large protein–protein surface that docking can't resolve. The structural hypothesis for why the CRBN oracle fails is that the glue contact with the neosubstrate is not ligand-mediated — gluing GSPT1 to CRBN requires the whole recruited surface, not just the small molecule's arm. CDK12-DDB1 is structurally different: CR8's exposed arm reaches directly into the DDB1 interface, and the glue contact is made by the small molecule itself, not the protein surface. If that hypothesis is correct, the CDK12-DDB1 system should discriminate cleanly. + +**Summary** — We first validated that blind docking can recover the native CR8 crystal pose (confirming we don't need anchoring for this system). Then we docked 160 real CDK12-DDB1 glues and 248 purine-armed decoys (real warhead, random arm), scored each against CDK12 alone (Tier 1) and CDK12+DDB1 (Tier 2), and measured the DDB1 neosubstrate differential. If real glues' arms contact DDB1 and decoy arms don't, the differential should separate them decisively. + +## Answer + +Decisive discrimination: 85.6% of real glues show a strong DDB1 bonus versus only 7.3% of decoys — a 78-percentage-point gap that CRBN never produced. Blind docking also recovers the native CR8 pose at 1.23 Å ranked #1, confirming we don't need a specialized anchoring protocol. The oracle rewards a productive DDB1-contacting arm, not just warhead presence. This is the right testbed for RGFN. + +## Relevance to our Publication + +This entry provides the first validated oracle for the RGFN pipeline. For NeurIPS, reviewers will ask whether the oracle actually discriminates — this entry answers that directly with a 78-percentage-point gap on 408 molecules. Paired with entry 001, it also supports the claim that our system selection was principled: we chose CDK12-DDB1 because its glue interface is ligand-mediated, and the discrimination confirms that structural reasoning was correct. Entry 003 formalizes this into a head-to-head comparison for the methods section. + +## Next Experiments + +**Refining for publication** — Ablation: compare the DDB1 neosubstrate differential against Tier 2 absolute score alone. NeurIPS reviewers will ask whether the differential is necessary or whether Tier 2 Vina alone achieves similar discrimination. Entry 003 includes the supporting numbers; a dedicated ablation entry would isolate this as a standalone result. + +**Next steps in project** — Run RGFN with this validated oracle and evaluate the generated molecules. Also test whether the oracle generalizes to additional CDK12-DDB1 glues or other molecular glue systems (MolGlueDB has candidates). The compute pipeline needed for fast oracle evaluation is benchmarked in entry 004. + +# Re-creation + +## Relevant Files + +Scripts: +- `./research/preprocessing/clean_6td3.py` — carves 6TD3 copy 1 into Tier 1 (CDK12), Tier 2 (CDK12+DDB1), and Tier 3 (+cyclinK) receptor structures; extracts native CR8 pose +- `./research/preprocessing/docking_6td3/redock_cr8.py` — blind-redocks CR8, minimizes native pose in-place, computes Tier 1 and Tier 2 scores for native pose validation +- `./research/preprocessing/docking_6td3/make_decoys_cdk.py` — generates purine-armed decoys (CDK12 ATP-hinge warhead + random drug-like arm) +- `./research/preprocessing/docking_6td3/dock_cluster.py` — batched multi-GPU box-docking driver; docks Tier 2, selects best-CNN pose, scores that pose against Tier 1 for the differential +- `./research/preprocessing/docking_6td3/submit_dock_6td3.sh` — Slurm submission script for Balam debug_full_node (job 69271) +- `./research/preprocessing/compare_systems.py` — prints known-vs-decoy discrimination metrics for each system and cross-system + +Models: +- `./research/preprocessing/docking_6td3/6TD3_tier1.pdbqt` — CDK12 alone (Tier 1); used to score baseline pocket binding without DDB1 +- `./research/preprocessing/docking_6td3/6TD3_tier2.pdbqt` — CDK12+DDB1 (Tier 2); the docking target for the discrimination run +- `./research/preprocessing/docking_6td3/crystal_RC8.pdb` — native CR8 pose; used as the autobox ligand and as the redocking validation target +- `./models/6TD3_tier1_CDK12.pdb`, `./models/6TD3_tier2_CDK12_DDB1.pdb` — source PDB files before pdbqt conversion + +Datasets: +- `./research/preprocessing/test-data/DDB1_CDK12_Glues.csv` — 175 real CDK12-CCNK/DDB1 glues (161 unique, 123 purine-based); known+ positives for discrimination run (160 docked) + +Results: +- `./research/preprocessing/docking_6td3/known_results.csv` — per-molecule docking scores for known glues (job 69271) +- `./research/preprocessing/docking_6td3/decoy_cdk_results.csv` — per-molecule docking scores for decoys (job 69271) + +Job Logs: +- `/scratch/markymoo/rgfn_runs/dock_6td3_69271/` — full per-shard output directory (Balam scratch) +- `/scratch/markymoo/rgfn_runs/dock6td3-69271.out` — SLURM job log with timing + +## Relevant Versions + +``` +106a4e6 Add 6TD3/CR8 glue docking oracle + 5HXB cross-system comparison +7afd7f1 Add docking pre-processing pipeline, inference, and run scripts +``` + +Relevant commit: `106a4e6` — 6TD3 docking scripts (`clean_6td3.py`, `redock_cr8.py`, `dock_cluster.py`, `make_decoys_cdk.py`) and result CSVs committed here. + +## Relevant Resources + +**Sources** +- 6TD3 crystal structure: Słabicki et al., *Nature* 2020 — DDB1·CDK12–cyclinK·CR8 ternary complex (doi:10.1038/s41586-020-2133-z) + +**Packages** +- gnina v1.3.2 (CNN-rescored docking) — `docking_6td3/`; launched via `/scratch/markymoo/gnina/run_gnina.sh` +- RDKit — conformer generation and decoy arm synthesis (`make_decoys_cdk.py`) +- OpenBabel — pdbqt conversion (`clean_6td3.py`) + +## Method + +1. **Structure prep** — `research/preprocessing/clean_6td3.py` carves 6TD3 copy 1: `models/6TD3_tier1_CDK12.pdb` (CDK12 only), `models/6TD3_tier2_CDK12_DDB1.pdb` (CDK12+DDB1), optional Tier 3 (+cyclinK); native CR8 extracted to `crystal_RC8.pdb`. Receptors → pdbqt via obabel. + +2. **Validation** — `docking_6td3/redock_cr8.py`: blind-redock CR8 with autobox on crystal pose (`--autobox_ligand crystal_RC8.pdb --autobox_add 4`). Confirm native-pose recovery. Score native pose against Tier 1 and Tier 2 to verify DDB1 cooperativity is captured. + +3. **Discrimination run** — `docking_6td3/dock_cluster.py` on Balam `debug_full_node` (job 69271, 13 min): dock 160 real glues + 248 decoys to Tier 2. Take best-CNN pose per molecule. Score that pose against Tier 1 via `--score_only` → DDB1 differential = Tier2 − Tier1. Sharded across 16 workers / 4 GPUs. + +## Results + +**CR8 native-pose validation:** + +| config | RMSD to crystal | Vina (kcal/mol) | CNNaff | DDB1 bonus | +|---|---|---|---|---| +| Blind redock (rank 1) | 1.23 Å | −10.68 | 0.99 | — | +| Native in-place minimize | — | −10.56 | 0.985 | — | +| Tier 1 (CDK12 only) | — | −8.39 | — | — | +| Tier 2 (CDK12+DDB1) | — | −10.68 | — | −2.16 to −2.29 | + +**Discrimination run (job 69271, n=160 known, n=248 decoy):** + +| metric | known | decoy | gap | +|---|---|---|---| +| **frac DDB1 dVina < −1.5** | **85.6%** | **7.3%** | **+78 pts** | +| median DDB1 dVina (kcal/mol) | −2.20 | −0.60 | −1.60 | +| median Tier 2 Vina (kcal/mol) | −10.15 | −7.96 | −2.19 | +| median Tier 2 CNNaff | 7.82 | 6.70 | 1.13 | +| frac Tier 2 Vina < −10 | 55.6% | 3.6% | +52 pts | diff --git a/Logs/003_crbn-vs-6td3-cross-system.md b/Logs/003_crbn-vs-6td3-cross-system.md new file mode 100644 index 0000000..4c8a5ef --- /dev/null +++ b/Logs/003_crbn-vs-6td3-cross-system.md @@ -0,0 +1,101 @@ +# 5HXB vs 6TD3 — head-to-head neosubstrate differential comparison +**Date:** 2026-06-18 + +## Question + +Does the structural reason for CRBN's discrimination failure hold when both systems are tested on identical methodology? + +## Context & Summary + +**Context** — Entries 001 and 002 showed that CRBN-based docking can't discriminate real glues from decoys (entry 001) while CDK12-DDB1 can, with a 78-percentage-point gap (entry 002). But those two experiments used different docking protocols — CRBN used warhead-anchored Pass B while 6TD3 used straight box docking — so the comparison isn't clean. We also updated the CRBN known-glue dataset between entries (from a purchasable scaffold library to a curated set of validated degraders). This head-to-head experiment puts both systems on identical methodology — same metric, same decoy design, same evaluation pipeline — to confirm whether the performance difference reflects the systems or the methods. + +**Summary** — We ran the CRBN/GSPT1 system with a curated known-glue dataset (200 real validated degraders, 177 anchorable) against the same 260 decoys, using the same neosubstrate-differential metric used for 6TD3: score each molecule against the E3 pocket alone (Tier 1) and the full ternary complex (Tier 2), then measure the differential on the same pose. The 6TD3 result comes from entry 002 (job 69271); the CRBN result is new (job 69272, Balam debug_full_node, 5.8 min). + +## Answer + +On identical methodology, the two systems split cleanly: the 6TD3 differential separates real glues from decoys by +78 percentage points; the CRBN differential shows no separation (−3 points, with decoys marginally higher). This confirms the failure in entry 001 is structural, not methodological. In the CRBN system, anchoring the glutarimide warhead in the cage places every arm near the GSPT1 surface regardless of arm chemistry — the bonus is a geometric artifact of the crystal geometry, not a specific molecular recognition event. In 6TD3, only arms with the right geometry reach DDB1, so the bonus discriminates. CRBN is not a viable RGFN testbed; 6TD3 is. + +## Relevance to our Publication + +NeurIPS reviewers will ask whether our system selection was principled or cherry-picked. This entry provides the direct answer: we tested the most prominent molecular glue system first, found a structural ceiling with an explicit mechanistic explanation, and then validated the alternative on a head-to-head comparison. The −3 pts vs +78 pts table on the same metric is figure-ready and justifies both the oracle design and the system selection in a single result. It also validates the neosubstrate differential as the correct metric: it works exactly where it should and fails exactly where it should, for the predicted structural reason. + +## Next Experiments + +**Refining for publication** — Molecular-weight sensitivity check on the CRBN known-glue dataset: `CRBN_GSPT1_Glues.csv` includes compounds up to MW ~1009, some likely bivalent/PROTAC-range. Filter to MW < ~600 and re-run to confirm the ceiling isn't a dataset artifact. Note: CRBN does show modest absolute-binding discrimination (fit-rate 77% vs 45%, Tier 2 Vina<−10: 35% vs 12%) — document this as secondary evidence that a CRBN reward isn't entirely worthless, just not glue-specific. + +**Next steps in project** — 6TD3 is confirmed as the RGFN testbed. Run RGFN with the validated 6TD3 oracle and evaluate generated molecules. The compute cost for full oracle runs is benchmarked in entry 004. + +# Re-creation + +## Relevant Files + +Scripts: +- `./docking_gnina/dock_cluster_crbn.py` — batched warhead-anchored docking for CRBN (job 69272): builds 100 flexible-anchor confs/mol → gnina `--minimize` vs Tier 2 (CRBN+GSPT1) → best clash-aware pose → `--score_only` vs Tier 1 (CRBN) → GSPT1 differential; sharded 16 workers / 4 GPUs with per-shard CSV checkpoints +- `./docking_gnina/submit_dock_crbn.sh` — Slurm submission script for Balam debug_full_node (job 69272) +- `./research/preprocessing/compare_systems.py` — prints both systems' known-vs-decoy distributions and known−decoy gap on the differential; cross-system summary table + +Models: +- `./docking_gnina/5HXB_tier2.pdbqt` — CRBN+GSPT1 ternary receptor (Tier 2); docking target for CRBN run +- `./docking_gnina/5HXB_tier1_CRBN.pdbqt` — CRBN pocket alone (Tier 1); used for `--score_only` differential +- `./research/preprocessing/docking_6td3/6TD3_tier1.pdbqt`, `6TD3_tier2.pdbqt` — CDK12-DDB1 receptors; 6TD3 results reused from entry 002 (job 69271) + +Datasets: +- `./research/preprocessing/test-data/CRBN_GSPT1_Glues.csv` — 200 real CRBN/GSPT1 validated degraders (199 active, 188 glutarimide-bearing, 177 unique anchorable); replaces the earlier purchasable scaffold library used in entry 001 +- `./docking_gnina/decoys.smiles` — 260 glutarimide-bearing decoys (same decoy set as entry 001) + +Results: +- `./docking_gnina/known_crbn_results.csv` — per-molecule CRBN docking scores (job 69272) +- `./docking_gnina/decoy_crbn_results.csv` — per-molecule decoy docking scores (job 69272) + +Job Logs: +- `/scratch/markymoo/rgfn_runs/dock_crbn_69272/` — full per-shard output + per-shard `shard*.csv` checkpoints (Balam scratch) +- `/scratch/markymoo/rgfn_runs/dockcrbn-69272.out` — SLURM job log with timing + +Note: one gnina worker segfaulted mid-run (libc, one shard) but per-shard try/except + checkpointing contained it; job exited 0. A few molecules in that shard may be missing — not material to the distributions. + +## Relevant Versions + +``` +106a4e6 Add 6TD3/CR8 glue docking oracle + 5HXB cross-system comparison +468fcc6 Clean up pre-processing into a concise, documented pipeline +``` + +Relevant commit: `106a4e6` — `dock_cluster_crbn.py`, `compare_systems.py`, and result CSVs committed here. 6TD3 results (job 69271) from the same commit (entry 002). + +## Relevant Resources + +**Sources** +- 5HXB: Matyskiela et al., *Science* 2016 — CRBN·DDB1·GSPT1·CC-885 ternary complex +- 6TD3: Słabicki et al., *Nature* 2020 — DDB1·CDK12–cyclinK·CR8 (doi:10.1038/s41586-020-2133-z) + +**Packages** +- gnina v1.3.2 — `docking_gnina/`; launched via `/scratch/markymoo/gnina/run_gnina.sh` +- RDKit — conformer generation + +## Method + +1. **CRBN run** — `docking_gnina/dock_cluster_crbn.py` on Balam `debug_full_node` (job 69272, 5.8 min): for each of 177 anchorable known glues and 260 decoys, build 100 flexible-anchor confs/mol → gnina `--minimize` vs Tier 2 (CRBN+GSPT1) → select best on-tether pose by clash-aware Vina → `--score_only` vs Tier 1 (CRBN) → GSPT1 differential. Sharded 16 workers / 4 GPUs with per-shard CSV checkpoints. + +2. **6TD3 result** — reused from entry 002 (job 69271). No re-run. + +3. **Cross-system comparison** — `research/preprocessing/compare_systems.py`: prints both systems' known-vs-decoy distributions and known−decoy gap on the differential. + +## Results + +**Neosubstrate differential (known vs decoy gap):** + +| system | metric | known | decoy | gap | +|---|---|---|---|---| +| 5HXB / CRBN | median GSPT1 dVina (kcal/mol) | −2.95 | −2.35 | −0.60 | +| 5HXB / CRBN | frac dVina < −1.5 | 85.3% | 88.0% | **−3 pts** | +| 6TD3 / CDK12 | median DDB1 dVina (kcal/mol) | −2.20 | −0.60 | −1.60 | +| 6TD3 / CDK12 | frac dVina < −1.5 | 85.6% | 7.3% | **+78 pts** | + +**Supporting absolute-binding metrics:** + +| system | fit-rate (known / decoy) | median Tier 2 Vina (known / decoy) | frac Tier 2 Vina < −10 (known / decoy) | +|---|---|---|---| +| 5HXB / CRBN | 136/177 (77%) / 117/260 (45%) | −8.63 / −7.35 | 35.3% / 12.0% | +| 6TD3 / CDK12 | 160/160 / 248/248 | −10.15 / −7.96 | 55.6% / 3.6% | + +CRBN shows modest absolute-binding discrimination (fit-rate, Tier 2 tails) but the differential — the glue-specific cooperativity term — shows no separation. 6TD3 wins decisively on both. The 6TD3 result is from entry 002 (job 69271); the CRBN result is from job 69272. diff --git a/Logs/004_compute-benchmark.md b/Logs/004_compute-benchmark.md new file mode 100644 index 0000000..67aa847 --- /dev/null +++ b/Logs/004_compute-benchmark.md @@ -0,0 +1,94 @@ +# Compute benchmark — login node vs Balam debug_full_node (4× A100) +**Date:** 2026-06-18 + +## Question + +How much faster is the Balam cluster compared to running docking locally, and where does the speedup come from? + +## Context & Summary + +**Context** — The full discrimination runs in entries 002 and 003 (408 molecules for 6TD3, 437 for CRBN) needed to complete within Balam's one-hour `debug_full_node` limit. We had been running docking locally during development using a sequential per-molecule approach, where gnina reloads its CNN model for every molecule — a design that turns out to be extremely slow for large runs. Before committing to the cluster for all future experiments, we needed to understand where the time was going and what the actual speedup would be, so we can plan the RGFN evaluation loop around realistic oracle costs. + +**Summary** — We compared three configurations: (A) sequential per-molecule docking on the login node, one gnina call per molecule; (B) batched multi-ligand docking on the login node, one gnina call per shard so the model loads once per shard; and (C) the full Balam debug node with 4× A100 and 16 workers. Configurations A and B were timed on a fixed 24-molecule subset; configuration C timing comes from SLURM logs of the full runs (jobs 69271 and 69272). + +## Answer + +Multi-ligand batching — running a full shard of molecules in a single gnina call — is the single biggest lever. For the CRBN anchored protocol (100 confs/mol), batching amortizes the CNN model reload across ~2700 conformers per shard, accounting for roughly 27× of the speedup on its own. Adding three more A100 GPUs contributes less than the batching did. Net result: both full runs completed in under 14 minutes combined — down from a projected ~3.5 hours of sequential local docking. A surprising finding: after the dock speedup, conformer embedding is now 60% of the CRBN wall time and has become the new bottleneck. + +## Relevance to our Publication + +This entry is the compute-methods justification for the paper. NeurIPS reviewers who ask about scalability will want to know that the oracle is fast enough to use in an RGFN training loop — ~14 minutes for 400 molecules confirms that. The Amdahl analysis also flags that conformer embedding, not GPU count, is the next bottleneck if we need to scale further, which shapes future infrastructure decisions and is worth a sentence in the methods. + +## Next Experiments + +**Refining for publication** — Profile conformer embedding time separately to confirm the Amdahl floor (estimated 3.5 min of the 5.8-min CRBN wall). If we scale to larger RGFN batches, faster or parallelized ETKDG or fewer confs per molecule is the next lever. + +**Next steps in project** — The established configuration (batched, 16 workers, 4× A100) is standard for all future oracle runs. Use these timings when estimating RGFN training loop cost. + +# Re-creation + +## Relevant Files + +Scripts: +- `./research/preprocessing/docking_6td3/dock_cluster.py` — batched 6TD3 docking driver; used for config B (local, `N_GPU=1 PROCS_PER_GPU=4`) and config C (cluster defaults) +- `./docking_gnina/dock_cluster_crbn.py` — batched CRBN docking driver; used for config C + +Note: config A sequential runners (`dock_6td3_batch.py` / `batch_anchor_dock.py`) have been removed and consolidated into the batching drivers above. The docking method is unchanged; only the sharding differs. + +Job Logs: +- `/scratch/markymoo/rgfn_runs/dock6td3-69271.out` — SLURM log for 6TD3 config C (job 69271); measured wall time source +- `/scratch/markymoo/rgfn_runs/dockcrbn-69272.out` — SLURM log for CRBN config C (job 69272); measured wall time source + +Note: config A/B benchmark subset runs (~24 molecules) were transient on the login node; outputs were not saved. + +## Relevant Versions + +``` +468fcc6 Clean up pre-processing into a concise, documented pipeline +106a4e6 Add 6TD3/CR8 glue docking oracle + 5HXB cross-system comparison +2fbdfb4 Remove Vina-GPU build cruft and Miniconda installer +``` + +Relevant commits: `468fcc6` — consolidation of sequential runners into batched `dock_cluster*.py` drivers. `2fbdfb4` — removed Vina-GPU build that was an earlier failed speedup attempt. + +## Relevant Resources + +**Sources** +- No external citations; timing analysis from first principles + SLURM logs + +**Packages** +- gnina v1.3.2 — docking engine; CNN model reload is the dominant per-call cost for the `--minimize` workload +- RDKit ETKDG — conformer embedding; becomes the Amdahl floor after the dock speedup + +## Method + +Three configurations benchmarked: + +- **A — sequential (login node):** 1 GPU, one gnina invocation per molecule. CNN model reloads every call. Original `dock_6td3_batch.py` / `batch_anchor_dock.py` design. +- **B — batched + 4-proc (login node):** 1 GPU, 4 worker processes, each docking a multi-ligand shard in one gnina call (model loads once per shard). `dock_cluster.py` with `N_GPU=1 PROCS_PER_GPU=4`. +- **C — full node (cluster):** Balam `debug_full_node`, 4× A100, 16 workers (4/GPU), batched. + +Timing: A and B measured on a fixed 24-molecule subset on the login node (s/mol), then extrapolated to full-run wall time. C taken directly from SLURM logs of jobs 69271 (6TD3) and 69272 (CRBN). CRBN config A rate extrapolated from an earlier completed 368-mol Pass B local run (368 mol in 101 min = 16.1 s/mol); the local 6TD3 sequential run was paused so 6TD3 config A rate is from the login-node benchmark subset. + +## Results + +**6TD3 — global box docking (exhaustiveness 16), 408 molecules:** + +| config | dock rate (s/mol) | wall time (408 mol) | +|---|---|---| +| A — sequential, 1 GPU | 12.5 | ~88 min (extrapolated) | +| B — 1 GPU / 4 proc / batched | 2.5 | ~17 min | +| **C — 4 GPU / 16 proc (job 69271)** | **1.57** | **13.4 min (measured)** | + +Total speedup A→C: ~6.6×. Decomposition: batching + 4 proc on one GPU gives 5.0× (A→B); adding 3 more GPUs gives 1.6× (B→C). 6TD3 global docking is CPU-bound (Vina Monte-Carlo search), so extra GPUs help little. + +**CRBN — warhead-anchored `--minimize`, 100 confs/mol, 437 molecules:** + +| config | dock rate (s/mol) | wall time (437 mol) | +|---|---|---| +| A — sequential, 1 GPU (368-mol Pass B run) | 16.1 | ~120 min (extrapolated) | +| **C — 4 GPU / 16 proc (job 69272)** | **0.32** | **5.8 min (measured)** | + +Total speedup A→C: ~21×; dock phase alone ~51×. Batching amortizes CNN model reload across ~2700 confs/shard (~27× of the win), on top of 16-way parallelism. After the dock speedup, conformer embedding (3.5 min) is 60% of the 5.8-min wall — new Amdahl bottleneck. + +**Combined cluster wall time (both full runs):** 13.4 min + 5.8 min = ~19 min. diff --git a/Logs/005_tier2-vina-roc-pr-curves.md b/Logs/005_tier2-vina-roc-pr-curves.md new file mode 100644 index 0000000..92fe71a --- /dev/null +++ b/Logs/005_tier2-vina-roc-pr-curves.md @@ -0,0 +1,81 @@ +# Tier 2 Vina discrimination curves — ROC and Precision-Recall for 6TD3 and 5HXB +**Date:** 2026-06-23 + +## Question + +How well does raw Tier 2 docking score alone separate real molecular glues from decoys across both our tested protein systems? + +## Context & Summary + +**Context** — Entries 001–003 characterized discrimination using threshold-based metrics (percentage of molecules above a score cutoff, and the neosubstrate differential gap). Entry 003 showed a +78-percentage-point separation for 6TD3 vs. −3 points for CRBN on the neosubstrate differential. Those summary statistics collapse the full score distribution into a single threshold. ROC and PR curves show the full picture across every possible threshold and are the standard figures reviewers will expect when evaluating a scoring oracle. This entry produces those curves for both systems using their Tier 2 Vina score — the absolute ternary-complex docking score before any differential is applied. It also answers a specific ablation question flagged in docs/RESEARCH_CONTEXT.md: does Tier 2 absolute score alone work, or is the neosubstrate differential truly necessary? + +**Summary** — We compute ROC-AUC and average precision (area under the PR curve) for Tier 2 Vina score as a binary classifier (real glue vs. decoy) for 6TD3 and 5HXB, using the same result CSVs produced in entries 002 and 003. No new docking is run. The output is a two-panel figure (ROC left, PR right) with both systems overlaid. + +## Answer + +Tier 2 Vina alone is a strong discriminator for 6TD3 (AUC = 0.890, AP = 0.872) and only a weak one for CRBN (AUC = 0.627, AP = 0.726). This confirms that 6TD3 is a high-quality oracle even on absolute binding score, and makes the ablation case that Tier 2 score alone is useful — but the neosubstrate differential (entry 003) further sharpens discrimination by removing the warhead-binding component. For CRBN, the modest absolute-binding signal (AUC above chance) is consistent with the fit-rate numbers from entry 003 (77% vs. 45%), but the differential adds nothing, confirming the system ceiling is structural. + +## Relevance to our Publication + +NeurIPS reviewers will ask for the ablation: does the neosubstrate differential add anything over just using the raw docking score? This entry provides half of that answer — Tier 2 Vina alone is already strong for 6TD3, which means any further gain from the differential is a bonus, not a necessity. It also provides publication-ready discrimination figures for both systems in a format that directly supports the oracle validation argument. The PR curve is especially important at NeurIPS because the known/decoy class ratio differs between systems (39% known for 6TD3 vs. 54% for CRBN), and PR curves are interpretable at different baselines. + +## Next Experiments + +**Refining for publication** — Add the neosubstrate differential (`ddb1_dvina` / `gspt1_dvina`) as a second pair of curves on the same figure, so the ROC/PR plot directly shows differential vs. absolute-score discrimination for both systems in one panel. This is the ablation figure. + +**Next steps in project** — Run RGFN with the validated 6TD3 oracle and evaluate generated molecules against these baseline curves to show the generative model produces candidates with better oracle scores than random warhead-bearing molecules. + +# Re-creation + +## Relevant Files + +Root: `./research/preprocessing/docking_gnina/analysis/` + +**Scripts** +- `plot_discrimination_curves.py` — loads Tier 2 Vina scores for both systems, builds ROC and PR curves via sklearn, outputs `discrimination_curves.png`; paths are resolved relative to the script via `pathlib` so it runs from any directory + +**Datasets** (inputs; no new docking run) +- `./research/preprocessing/docking_6td3/known_results.csv` — per-molecule 6TD3 docking results for 160 known CDK12-DDB1 glues (from entry 002, job 69271) +- `./research/preprocessing/docking_6td3/decoy_cdk_results.csv` — per-molecule 6TD3 docking results for 248 decoys (from entry 002, job 69271) +- `./research/preprocessing/docking_gnina/known_crbn_results.csv` — per-molecule CRBN docking results for 136 anchorable known glues (from entry 003, job 69272) +- `./research/preprocessing/docking_gnina/decoy_crbn_results.csv` — per-molecule CRBN docking results for 117 decoys (from entry 003, job 69272) + +**Results** +- `./research/preprocessing/docking_gnina/analysis/discrimination_curves.png` — two-panel ROC + PR figure; both systems overlaid; dashed baseline lines on PR panel + +## Relevant Versions + +`0c3f154` — Add Tier 2 Vina ROC/PR discrimination curves (entry 005) + +## Relevant Resources + +**Sources** +- Entry 002 (`002_6td3-cr8-validation-and-discrimination.md`) — source of 6TD3 result CSVs +- Entry 003 (`003_crbn-vs-6td3-cross-system.md`) — source of CRBN result CSVs + +**Packages** +- scikit-learn — `roc_curve`, `auc`, `precision_recall_curve`, `average_precision_score`; used in `plot_discrimination_curves.py` +- matplotlib — figure rendering; used in `plot_discrimination_curves.py` +- pandas / numpy — data loading and label array construction + +## Method + +1. Load `known_results.csv` and `decoy_cdk_results.csv` for 6TD3; `known_crbn_results.csv` and `decoy_crbn_results.csv` for CRBN. Filter to `status == 'ok'` rows only. +2. Concatenate Tier 2 Vina scores; assign binary labels (1 = known glue, 0 = decoy). Negate scores before passing to sklearn (more negative Vina = stronger predicted binding = more positive class). +3. Compute `roc_curve` + `auc` and `precision_recall_curve` + `average_precision_score` for each system. +4. Render two-panel figure: ROC left, PR right; dashed diagonal reference on ROC; dashed class-baseline references on PR. Save at 200 dpi. + +``` +~/anaconda3/bin/python research/preprocessing/docking_gnina/analysis/plot_discrimination_curves.py +``` + +## Results + +| System | ROC-AUC | Avg Precision | n known | n decoy | PR baseline | +|---|---|---|---|---|---| +| 6TD3 (CDK12–DDB1) | 0.890 | 0.872 | 160 | 248 | 0.392 | +| 5HXB (CRBN–GSPT1) | 0.627 | 0.726 | 136 | 117 | 0.538 | + +Note average precision is also PR-AUC + +Only molecules with `status='ok'` included; CRBN n=136 known reflects the 177-anchorable set with 41 no-valid-pose failures (see entry 003). 6TD3 n=248 decoys reflects same decoy set scored in entry 002; all 248 returned `status='ok'`. Input score CSVs are from entries 002 and 003 — no new docking run. diff --git a/Logs/006_6td3-violin-distributions.md b/Logs/006_6td3-violin-distributions.md new file mode 100644 index 0000000..4098f29 --- /dev/null +++ b/Logs/006_6td3-violin-distributions.md @@ -0,0 +1,118 @@ +# 6TD3 Metric Comparison (Tier 1, Tier 2, CNN, Vina, Difference) for known glues vs decoys. + +**Date:** 2026-06-25, ~3pm + +## Question + +For the 6TD3 system, which metric is the best at discriminating between glues and decoys: Tier 1 score, Tier 2 score, or T2 - T1? What about when comparing Vina versus CNN? + + +## Context & Summary + +**Context** — Entry 002 established that the CDK12-DDB1 oracle separates real molecular glues from realistic fakes (decoys with the right warhead but a random arm), and entry 005 backed this with ROC/PR curves on the absolute Tier 2 score. But the oracle gives us *six* candidate signals to reward RGFN with — two scoring functions (Vina and the gnina CNN affinity) crossed with three tiers (Tier 1 = kinase pocket only, Tier 2 = kinase + DDB1, and the Tier 2 − Tier 1 differential) — and we have never ranked them against each other on the same footing. Picking the wrong signal would train the generator toward the wrong thing, so before wiring an oracle into the active-learning loop we need to know which of the six actually discriminates best. + +**Summary** — We reused the existing docking results from entry 002 (no new docking) and put all six metrics through the same head-to-head test against our negative control — decoys built with the correct CDK12 warhead but an arbitrary drug-like arm, so any metric that separates binders from decoys is reading something beyond mere warhead binding. For each metric we drew a known-vs-decoy violin plot, then scored discrimination on a unit-free footing (Cohen's d and AUROC, so Vina kcal/mol and CNN pK are comparable) and confirmed it with a confusion matrix at the best single threshold. + +## Answer + +We found that Vina ΔT2−T1 is the best discriminator, with CNN Tier 2 being a close second. Vina ΔT2−T1 has a Cohen's D of 2.38 and AUROC of **0.946**, which means that Vina ΔT2−T1 is a VERY STRONG discriminator between real and fake glues. This gives precedence to our choice of Vina ΔT2−T1 as our oracle signal. + +## Relevance to our Publication + +This is the ablation Digital Discovery / J. Cheminformatics reviewers will ask for: *is the Tier 2 − Tier 1 differential actually necessary, or does an absolute score do just as well?* By ranking all six candidate signals on the same data, we answer it directly — the Vina differential wins, and the kinase-only Tier 1 scores trail badly, showing the discriminating information lives in what DDB1 adds, not in the warhead pocket. That justifies our oracle-signal choice with evidence rather than assertion, and the violins double as a publication-ready supplementary figure that makes the tier-by-tier, score-by-score logic legible at a glance. + +## Next Experiments + +**Refining for publication** — The per-panel effect-size / AUROC / Mann–Whitney table is now in place (see Results); the remaining airtightness step is to match molecular-weight distributions between known and decoy sets so a reviewer can't attribute the Tier 2 shift to ligand size alone, and ideally report held-out (rather than in-sample) confusion matrices via cross-validation so the Youden cut isn't fit on the data it scores. A combined figure pairing these violins with the entry-005 ROC/PR curves would make a compact, self-contained discrimination story. + +**Next steps in project** — Wire the winning signal — the Vina Tier 2 − Tier 1 differential — into the active-learning loop as the 6TD3 oracle reward, then produce the analogous distribution plots for *generated* molecules, showing the generator's output populates the high-scoring region these known glues occupy. + +# Re-creation + +## Relevant Files + +Root: `research/preprocessing/` + +Scripts: +- `./full_comparison/plot_violins.py` — loads the entry-002 result CSVs, filters to `status=='ok'`, recomputes the Tier 2 − Tier 1 differentials, and renders the six-panel violin figure plus a medians/gap summary CSV. +- `./full_comparison/discrimination_stats.py` — computes the scale-free discrimination stats per panel (Cohen's d, AUROC, Mann-Whitney U p-value) and writes `discrimination_stats.csv` ranked by AUROC. +- `./full_comparison/confusion_matrices.py` — picks the Youden-J-optimal cut per panel and renders a 2×3 grid of confusion matrices (binders = positive class) plus `confusion_matrices.csv`. + +Datasets (inputs, reused from entry 002 — no new docking): +- `./docking_6td3/known_results.csv` — per-molecule docking scores for 160 real CDK12-DDB1 glues (job 69271); the KNOWN+ positives. +- `./docking_6td3/decoy_cdk_results.csv` — per-molecule docking scores for 248 purine-armed decoys (job 69271); the negative control. + +Results: +- `./full_comparison/violins_known_vs_decoy.png` — the six-panel violin figure (2×3 grid). +- `./full_comparison/violin_medians.csv` — per-panel decoy median, known median, and known-minus-decoy gap. +- `./full_comparison/discrimination_stats.csv` — per-panel gap, Cohen's d, AUROC, MWU p-value. +- `./full_comparison/confusion_matrices.png` — 2×3 grid of confusion matrices at the Youden-optimal cut. +- `./full_comparison/confusion_matrices.csv` — per-panel threshold, TP/FP/FN/TN, accuracy, sensitivity, specificity, precision. + +## Relevant Versions + +``` +f418de8 [CODE] Mock Active Learning Loop (no balam) +a3d21c1 [DOCS] Remove 000 Template +1c54a01 [DOCS] Remove template +``` + +The three scripts and their outputs (`research/preprocessing/full_comparison/`) were committed in `c723fcb`. The input CSVs were committed previously with entry 002 (`106a4e6`). + +## Relevant Resources + +**Sources** +- Score data originate from entry 002 (6TD3 discrimination run, Balam job 69271); CR8/6TD3 structure: Słabicki et al., *Nature* 2020 (doi:10.1038/s41586-020-2133-z). + +**Packages** +- matplotlib 3.10.6 (violin + confusion-matrix rendering), pandas 2.3.3, numpy 2.3.5 — `plot_violins.py`, `discrimination_stats.py`, `confusion_matrices.py`. +- scipy (Mann-Whitney U) — `discrimination_stats.py`; scikit-learn (`roc_curve`, `confusion_matrix`) — `confusion_matrices.py`. + +## Method + +1. **Load + filter** — read `known_results.csv` and `decoy_cdk_results.csv`, keep only `status=='ok'` rows (160 known, 248 decoy). +2. **Derive differentials** — recompute `dvina = vina_t2 − vina_t1` and `dcnnaff = cnnaff_t2 − cnnaff_t1` per molecule (matches the stored `ddb1_dvina` / `ddb1_dcnnaff` columns) so the plotted quantity is explicit. +3. **Render violins** — `python research/preprocessing/full_comparison/plot_violins.py` draws six violin panels (decoy vs. binder per panel) with median lines, jittered raw points, and a median annotation; writes the PNG and `violin_medians.csv`. +4. **Discrimination stats** — `python research/preprocessing/full_comparison/discrimination_stats.py` computes, per panel, Cohen's d (pooled SD, oriented so positive = binders better), AUROC (= MWU U / (n_known·n_decoy)), and a one-sided Mann-Whitney U p-value; writes `discrimination_stats.csv` ranked by AUROC. +5. **Confusion matrices** — `python research/preprocessing/full_comparison/confusion_matrices.py` orients each metric (negate lower-is-better Vina), picks the Youden-J-optimal ROC cut, classifies all samples (binders = positive), and renders the 2×3 confusion-matrix grid + `confusion_matrices.csv`. + +## Results + +n = 160 known glues, 248 decoys (status=ok), all six metrics computed on the same entry-002 poses (job 69271). + +**The head-to-head ranking (`discrimination_stats.csv`).** Ranked by AUROC — P(a random binder scores better than a random decoy) — which is unit-free and so the fair way to put Vina (kcal/mol) and CNN (pK) on the same scale: + +| metric | Cohen's d | AUROC | MWU p | +|---|---|---|---| +| **Vina ΔT2−T1** | **2.38** | **0.946** | 1.3e-52 | +| CNN Tier 2 | 1.95 | 0.907 | 3.9e-44 | +| Vina Tier 2 | 1.21 | 0.890 | 8.4e-41 | +| CNN Tier 1 | 1.52 | 0.863 | 1.9e-35 | +| CNN ΔT2−T1 | 1.18 | 0.850 | 3.1e-33 | +| Vina Tier 1 | 0.41 | 0.691 | 4.0e-11 | + +**Vina ΔT2−T1 wins** (AUROC 0.946 — it ranks a real glue above a decoy 95% of the time), with CNN Tier 2 a close second (0.907). The pattern across the table is the point: within each scoring function, adding DDB1 (Tier 1 → Tier 2) and isolating its contribution (→ Δ) lifts discrimination, and **Vina Tier 1 is the worst signal of all** (0.691) — kinase-pocket binding alone barely tells glues from decoys. That is exactly why the differential is the right oracle signal: the discriminating information lives in what DDB1 adds, not in the warhead pocket. (Caveats: AUROC is the fair ranker; Cohen's d assumes ~normal equal-variance populations that docking-score tails violate, which is why the d and AUROC orderings disagree slightly. The six metrics are not independent — Tier 2 = Tier 1 + differential by construction.) + +**Supporting — median shift and raw gap (`violin_medians.csv`; consistent with entry 002).** Useful for reading the violins, but the gap is in native units and so is *not* comparable across rows (the CNN differential's tiny +0.19 gap still gives AUROC 0.850 because ΔCNN has a very tight spread). Ordered as the figure panels: + +| metric | decoy median | known median | gap (known − decoy) | better | +|---|---|---|---|---| +| Vina Tier 1 (CDK12 only) | −7.35 | −8.02 | −0.67 | lower | +| Vina Tier 2 (CDK12+DDB1) | −7.96 | −10.15 | −2.19 | lower | +| Vina Tier 2 − Tier 1 (DDB1 ΔVina) | −0.60 | −2.20 | −1.60 | lower | +| CNN affinity Tier 1 (CDK12 only) | 6.68 | 7.59 | +0.92 | higher | +| CNN affinity Tier 2 (CDK12+DDB1) | 6.70 | 7.82 | +1.13 | higher | +| CNN affinity Tier 2 − Tier 1 (DDB1 ΔCNN) | 0.04 | 0.23 | +0.19 | higher | + +**Confirmation — confusion matrices at the Youden-optimal cut (`confusion_matrices.csv`, in-sample; n=160 binders / 248 decoys).** Threshold fit on the same data it scores — descriptive separability, not held-out performance. + +| panel | cut (native) | TP | FP | FN | TN | acc | sens | spec | +|---|---|---|---|---|---|---|---|---| +| Vina ΔT2−T1 | ≤ −1.58 | 136 | 12 | 24 | 236 | 0.91 | 0.85 | 0.95 | +| CNN Tier 2 | ≥ 7.36 | 128 | 27 | 32 | 221 | 0.86 | 0.80 | 0.89 | +| CNN Tier 1 | ≥ 7.20 | 129 | 39 | 31 | 209 | 0.83 | 0.81 | 0.84 | +| Vina Tier 2 | ≤ −8.98 | 127 | 41 | 33 | 207 | 0.82 | 0.79 | 0.83 | +| CNN ΔT2−T1 | ≥ 0.12 | 127 | 39 | 33 | 209 | 0.82 | 0.79 | 0.84 | +| Vina Tier 1 | ≤ −7.86 | 96 | 57 | 64 | 191 | 0.70 | 0.60 | 0.77 | + +The accuracy ranking agrees with the AUROC ranking at both ends — **Vina ΔT2−T1 best (acc 0.91), Vina Tier 1 worst (acc 0.70)** — with only the middle pair swapping, so the head-to-head verdict is robust to how it is measured. As a bonus check, the Vina differential's Youden-optimal cut (−1.58) lands essentially on the −1.5 threshold entry 002 chose by hand, independently confirming that cutoff. diff --git a/Logs/007_6td3-molecular-weight-control.md b/Logs/007_6td3-molecular-weight-control.md new file mode 100644 index 0000000..d7c0842 --- /dev/null +++ b/Logs/007_6td3-molecular-weight-control.md @@ -0,0 +1,118 @@ +# 6TD3 — controlling the glue-vs-decoy comparison for molecular weight + +**Date:** 2026-06-25, ~5pm + +## Question + +When our oracle scores a real glue higher than a fake one, is it reading genuine glue cooperativity — or just the fact that real glues happen to be bigger molecules? + +## Context & Summary + +**Context** — Entry 006 ranked the six candidate oracle signals on the 6TD3 system and crowned the Vina Tier 2 − Tier 1 differential the best discriminator between real glues and decoys (it ranks a real glue above a fake one ~95% of the time). But it flagged one loose end that a reviewer would seize on: real glues and our decoys might differ in *size*, and bigger molecules generally dock better simply by making more contacts. If that were the whole story, our "glue signal" would be a molecular-weight artifact in disguise. This entry closes that gap. + +**Summary** — We computed the molecular weight of every known glue and every decoy, confirmed the two groups really do differ in size, then built a size-balanced comparison: each real glue was paired with a decoy of nearly identical molecular weight (1:1 nearest-neighbor matching), and we re-scored discrimination on those matched pairs. To make sure the result wasn't an accident of which pairs got matched, we repeated the matching 500 times and report the average with a confidence interval. + +## Answer + +The size confound is real and large — known glues are ~93 Da heavier than decoys on average — so the concern was justified. But the winning signal survives it: once glues and decoys are matched on molecular weight, the Vina Tier 2 − Tier 1 differential stays a strong discriminator (it drops only from 0.95 to 0.87). In sharp contrast, the *absolute* scores were mostly reading size — Vina Tier 2 falls from 0.89 to 0.65, and kinase-pocket-only Vina Tier 1 collapses from 0.69 to **below chance (0.38)**. The differential wins precisely because it cancels out ligand size and leaves the DDB1-recruitment signal we actually care about. + +## Relevance to our Publication + +This is the exact robustness check a Digital Discovery or J. Cheminformatics reviewer demands of any "our metric separates actives from decoys" claim: *prove it isn't a trivial property like size*. We now answer it head-on. The result is also stronger than a mere defense — it turns molecular weight from a liability into supporting evidence for our central design choice. The differential isn't just the best discriminator (entry 006); it's the best *because* it is the one signal that doesn't reduce to ligand size, which is exactly the argument for using a Tier 2 − Tier 1 differential rather than an absolute docking score. The before/after AUROC figure is a clean supplementary panel that makes the ablation case visually. + +## Next Experiments + +**Refining for publication** — The matched comparison necessarily restricts to the molecular-weight range the two sets share; 51 of the heaviest known glues sit above the decoy ceiling and can't be matched at all. Reporting a held-out (cross-validated) version of the entry-006 confusion matrices, and generating decoys that span the full glue weight range, would make the control airtight. Pairing this figure with the entry-005 ROC/PR curves and the entry-006 violins would give a single self-contained discrimination story. + +**Next steps in project** — Wire the molecular-weight-robust signal (Vina Tier 2 − Tier 1) into the active-learning loop as the 6TD3 oracle reward, and confirm RGFN's *generated* molecules earn their high differential scores through arm-recruitment rather than by simply drifting to high molecular weight — i.e. watch generated-molecule size alongside the reward so the generator can't game the oracle the way the absolute scores could be gamed. + +# Re-creation + +## Relevant Files + +Root: `research/preprocessing/full_comparison_mw/` + +Scripts: +- `./mw_common.py` — shared spine: loads the entry-002 docking CSVs (status='ok'), adds an RDKit molecular-weight column and the two Tier 2 − Tier 1 differentials, and provides the AUROC helper and the reproducible greedy MW-matching routine. The other three scripts import it. +- `./mw_distributions.py` — draws the known-vs-decoy molecular-weight violin and reports the size gap (Cohen's d, AUROC, Mann-Whitney U, Kolmogorov-Smirnov). Establishes that the confound exists. +- `./mw_score_correlation.py` — regresses each of the six oracle signals on molecular weight (Spearman/Pearson, pooled and within-group), showing which signals are size-driven. +- `./mw_matched_discrimination.py` — the decisive test: 1:1 nearest-neighbor MW matching (15 Da caliper), recomputes AUROC on the matched subset, bootstraps 500 randomized matchings for a 95% CI, and compares to the full-set AUROC. + +Datasets (inputs, reused from entry 002 — no new docking): +- `../docking_6td3/known_results.csv` — per-molecule docking scores for 160 real CDK12-DDB1 glues (job 69271); the KNOWN+ positives. SMILES column is the molecular-weight source. +- `../docking_6td3/decoy_cdk_results.csv` — per-molecule docking scores for 248 purine-armed decoys (job 69271); the negative control. + +Results: +- `./mw_distribution.png` / `./mw_distribution.csv` — molecular-weight violin and the size-gap summary stats. +- `./mw_score_correlation.png` / `./mw_score_correlation.csv` — 2×3 scatter grid (MW vs each metric) and the per-metric correlation table. +- `./mw_matched_auroc.png` / `./mw_matched_auroc.csv` — full vs MW-matched AUROC (bar pairs with bootstrap CI) and the per-metric numeric table. +- `./mw_match_balance.csv` — molecular-weight balance before vs after matching (medians, gap, MWU p). + +## Relevant Versions + +``` +5633939 [docs] update commit in logs +c723fcb [CODE] Little determination of CNN vs Vina scoring on Tier 1 vs Tier 2 vs Tier 2 - Tier 1. +f418de8 [CODE] Mock Active Learning Loop (no balam) +``` + +The four scripts and their outputs (now under `experiments/ablations/mw/` after the repo restructure) were committed in `13cf8b6` ("[CODE] MW Matched Decoys and corrected active learning oracle"). + +## Relevant Resources + +**Sources** +- Score data originate from entry 002 (6TD3 discrimination run, Balam job 69271); CR8/6TD3 structure: Słabicki et al., *Nature* 2020 (doi:10.1038/s41586-020-2133-z), `[slabicki2020cr8]`. +- Builds directly on entry 006 (which metric discriminates best) and entry 005 (Tier 2 ROC/PR curves). + +**Packages** +- RDKit 2023.09.5 (`rdkit.Chem.Descriptors.MolWt`, average molecular weight from SMILES) — `mw_common.py`. +- pandas 2.x, numpy — all scripts. +- scipy (`mannwhitneyu`, `ks_2samp`, `pearsonr`, `spearmanr`) — distribution tests and correlations. +- matplotlib 3.10.x — all figures. +- Run inside the `rgfn` conda env (only env with RDKit locally): `conda run -n rgfn python