0.7.0

.gitignore

@@ -165,6 +165,9 @@ configs/local/default.yaml
 # Forks
 /workdir/
 /forks/alphafold3/*prediction_outputs/
+/forks/boltz/boltz/
+/forks/boltz/prediction_inputs/
+/forks/boltz/prediction_outputs/
 /forks/chai-lab-orig/
 /forks/chai-lab*/chai-lab/
 /forks/chai-lab*/prediction_inputs/
@@ -199,3 +202,4 @@ configs/local/default.yaml
 /forks/TULIP/outputs/
 /forks/Vina/ADFR/
 scripts/*inference*/
+scoring/

.pre-commit-config.yaml

@@ -128,7 +128,7 @@ repos:
     hooks:
       - id: codespell
         args:
-          - --skip=logs/**,data/**,*.ipynb,posebench/utils/data_utils.py,posebench/utils/residue_utils.py,posebench/data/components/fasta_preparation.py,posebench/models/minimize_energy.py,posebench/data/components/create_casp15_ensemble_input_csv.py,posebench/analysis/casp15_ligand_scoring/casp_parser.py,*Components-smiles-stereo-oe.smi,notebooks/pdb_reports/transferase/*
+          - --skip=logs/**,data/**,notebooks/**,*.ipynb,posebench/utils/data_utils.py,posebench/utils/residue_utils.py,posebench/data/components/fasta_preparation.py,posebench/models/minimize_energy.py,posebench/data/components/create_casp15_ensemble_input_csv.py,posebench/analysis/casp15_ligand_scoring/casp_parser.py,*Components-smiles-stereo-oe.smi,notebooks/pdb_reports/transferase/*
           # - --ignore-words-list=abc,def
 
   # jupyter notebook cell output clearing

CHANGELOG.md

@@ -1,3 +1,24 @@
+### 0.7.0 - 08/11/2025
+
+**Additions**:
+
+- Added new baseline methods (Boltz-1/2 w/ and w/o with multiple sequence alignments (MSAs)).
+- Added a new failure mode analysis notebook incorporating PLINDER for robust method generalization analysis(n.b., see new arXiv preprint for more details).
+- Added a script for performing custom method generalization analyses (e.g., when new methods are released which were trained on more recent splits of the PDB compared to AlphaFold 3's cutoff date of September 30, 2021).
+
+**Changes**:
+
+- Updated the `posebusters` package version to (at the time of writing) the latest version `0.4.5` to integrate new (stricter) pose checks (e.g., regarding flat aliphatic rings). Overall, each method's PoseBusters validity scores have decreased slightly on average (as expected).
+
+**Bug Fixes**:
+
+- Fixed a bug related to the number of DockGen-E protein chains provided to Chai-1 for inference on this dataset. Chai-1's DockGen-E results accordingly have been updated.
+
+**Results**:
+
+- With all the above changes in place, re-analyzed all baseline methods for each benchmark dataset, and updated the baseline predictions hosted on Zenodo
+- **NOTE**: The updated arXiv preprint should be publicly available by 08/15/2025
+
 ### 0.6.0 - 02/09/2025
 
 **Additions**:
@@ -21,7 +42,7 @@
 
 **Results**:
 
-- With all the above changed in place, simplified, re-ran, and re-analyzed all baseline methods for each benchmark dataset, and updated the baseline predictions and datasets (now containing standardized MSAs) hosted on Zenodo
+- With all the above changes in place, simplified, re-ran, and re-analyzed all baseline methods for each benchmark dataset, and updated the baseline predictions and datasets (now containing standardized MSAs) hosted on Zenodo
 - **NOTE**: The updated arXiv preprint should be publicly available by 02/12/2025
 
 ### 0.5.0 - 09/30/2024

configs/analysis/complex_alignment.yaml

@@ -1,5 +1,5 @@
-method: neuralplexer # the method for which to align predictions - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `alphafold3`)
-vina_binding_site_method: p2rank # the method to use for Vina binding site prediction - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `alphafold3`, `p2rank`)
+method: neuralplexer # the method for which to align predictions - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `alphafold3`)
+vina_binding_site_method: p2rank # the method to use for Vina binding site prediction - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `alphafold3`, `p2rank`)
 dataset: posebusters_benchmark # the dataset to use - NOTE: must be one of (`posebusters_benchmark`, `astex_diverse`, `dockgen`, `casp15`)
 ensemble_ranking_method: consensus # the method with which to rank-order and select the top ensemble prediction for each target - NOTE: must be one of (`consensus`, `ff`)
 input_data_dir: ${oc.env:PROJECT_ROOT}/data/${dataset}_set # the input protein-ligand complex directory to recursively parse

configs/analysis/inference_analysis.yaml

@@ -1,6 +1,6 @@
 full_report: true # whether to generate a full PoseBusters report (i.e. with all metrics) or a summary report (i.e. with only the most important metrics)
-method: diffdock # the method for which to score predictions - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `alphafold3`, `vina`, `ensemble`)
-vina_binding_site_method: p2rank # the method to use for Vina binding site prediction - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `alphafold3`, `p2rank`)
+method: diffdock # the method for which to score predictions - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `alphafold3`, `vina`, `ensemble`)
+vina_binding_site_method: p2rank # the method to use for Vina binding site prediction - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `alphafold3`, `p2rank`)
 dataset: posebusters_benchmark # the dataset to use - NOTE: must be one of (`posebusters_benchmark`, `astex_diverse`, `dockgen`, `casp15`)
 ensemble_ranking_method: consensus # the method with which to rank-order and select the top ensemble prediction for each target - NOTE: must be one of (`consensus`, `ff`)
 input_csv_path: ${resolve_method_input_csv_path:${method},${dataset},${pocket_only_baseline}} # the input CSV filepath with which to run inference

configs/analysis/inference_analysis_casp.yaml

@@ -1,8 +1,8 @@
 full_report: true # whether to generate a full PoseBusters report (i.e. with all metrics) or a summary report (i.e. with only the most important metrics)
 python_exec_path: ${oc.env:HOME}/mambaforge/envs/casp15_ligand_scoring/bin/python3 # the Python executable to use
 scoring_script_path: ${oc.env:PROJECT_ROOT}/posebench/analysis/casp15_ligand_scoring/score_predictions.py # the path to the script to use for scoring CASP predictions
-method: diffdock # the method for which to score predictions - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `alphafold3`, `vina`, `ensemble`, `tulip`)
-vina_binding_site_method: p2rank # the method to use for Vina binding site prediction - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `alphafold3`, `p2rank`)
+method: diffdock # the method for which to score predictions - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `alphafold3`, `vina`, `ensemble`, `tulip`)
+vina_binding_site_method: p2rank # the method to use for Vina binding site prediction - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `alphafold3`, `p2rank`)
 dataset: casp15 # the dataset to use - NOTE: must be one of (`casp15`)
 ensemble_ranking_method: consensus # the method with which to rank-order and select the top ensemble prediction for each target - NOTE: must be one of (`consensus`, `ff`)
 predictions_dir: ${oc.env:PROJECT_ROOT}/data/test_cases/${dataset}/top_${method}_ensemble_predictions_${repeat_index} # the directory containing the predictions to analyze
@@ -15,4 +15,5 @@ no_ilcl: false # whether to score a model trained without an inter-ligand clash
 relax_protein: false # whether to relax the protein - NOTE: currently periodically yields unpredictable protein-ligand separation
 v1_baseline: false # whether to score the v1 baseline predictions
 allow_missing_predictions: true # whether to allow missing predictions for a target
-force_rescore: false # whether to force CASP15 and PoseBusters rescoring of the predictions
+force_casp15_rescore: false # whether to force CASP15 rescoring of the predictions
+force_pb_rescore: false # whether to force PoseBusters rescoring of the predictions

configs/data/boltz_input_preparation.yaml

@@ -0,0 +1,10 @@
+dataset: posebusters_benchmark # the dataset to use - NOTE: must be one of (`posebusters_benchmark`, `astex_diverse`, `dockgen`, `casp15`)
+input_data_dir: ${oc.env:PROJECT_ROOT}/data/${dataset}_set # the input protein-ligand complex directory to recursively parse
+msa_dir: ${oc.env:PROJECT_ROOT}/data/${dataset}_set/${dataset}_boltz_msas # the directory containing the `.csv` MSA files prepared for Boltz via `posebench/data/components/prepare_boltz_msas.py`; if not provided, Boltz will be run in single-sequence mode
+output_scripts_path: ${oc.env:PROJECT_ROOT}/forks/boltz/prediction_inputs/${dataset} # the output directory in which to save the input files
+posebusters_ccd_ids_filepath: ${oc.env:PROJECT_ROOT}/data/posebusters_pdb_ccd_ids.txt # the path to the PoseBusters PDB CCD IDs file that lists the targets that do not contain any crystal contacts
+dockgen_test_ids_filepath: ${oc.env:PROJECT_ROOT}/data/dockgen_set/split_test.txt # the path to the DockGen test set IDs file
+protein_filepath: null # the path to the protein structure file to use
+ligand_smiles: null # the ligand SMILES string for which to predict the binding pose
+input_id: null # the input ID to use for inference
+pocket_only_baseline: false # whether to prepare the pocket-only baseline

configs/data/boltz_output_extraction.yaml

@@ -0,0 +1,13 @@
+dataset: posebusters_benchmark # the dataset to use - NOTE: must be one of (`posebusters_benchmark`, `astex_diverse`, `dockgen`, `casp15`)
+prediction_inputs_dir: ${oc.env:PROJECT_ROOT}/forks/boltz/prediction_inputs/${dataset}
+prediction_outputs_dir: ${oc.env:PROJECT_ROOT}/forks/boltz/prediction_outputs/${dataset}_${repeat_index}
+inference_outputs_dir: ${oc.env:PROJECT_ROOT}/forks/boltz/inference/boltz_${dataset}_outputs_${repeat_index}
+input_data_dir: ${oc.env:PROJECT_ROOT}/data/${dataset}_set # the input protein-ligand complex directory to recursively parse
+posebusters_ccd_ids_filepath: ${oc.env:PROJECT_ROOT}/data/posebusters_pdb_ccd_ids.txt # the path to the PoseBusters PDB CCD IDs file that lists the targets that do not contain any crystal contacts
+dockgen_test_ids_filepath: ${oc.env:PROJECT_ROOT}/data/dockgen_set/split_test.txt # the path to the DockGen test set IDs file
+complex_filepath: null # if not `null`, this should be the path to the complex PDB file for which to extract outputs
+complex_id: null # if not `null`, this should be the complex ID of the single complex for which to extract outputs
+ligand_smiles: null # if not `null`, this should be the (i.e., `.` fragment-separated) complex ligand SMILES string of the single complex for which to extract outputs
+output_dir: null # if not `null`, this should be the path to the output file to which to write the extracted outputs
+repeat_index: 1 # the repeat index with which inference was run
+pocket_only_baseline: false # whether to prepare the pocket-only baseline

configs/data/components/prepare_boltz_msas.yaml

@@ -0,0 +1,6 @@
+dataset: posebusters_benchmark # the dataset to use - NOTE: must be one of (`posebusters_benchmark`, `astex_diverse`, `dockgen`, `casp15`)
+input_data_dir: ${oc.env:PROJECT_ROOT}/data/${dataset}_set # the input protein-ligand complex directory to recursively parse
+input_msa_dir: ${oc.env:PROJECT_ROOT}/data/${dataset}_set/${dataset}_msas # where the original MSA files are placed
+output_msa_dir: ${oc.env:PROJECT_ROOT}/data/${dataset}_set/${dataset}_boltz_msas # where the processed MSA files should be stored
+skip_existing: true # whether to skip processing if the output file already exists
+pocket_only_baseline: false # whether to prepare the pocket-only baseline

configs/data/components/prepare_chai_msas.yaml

@@ -1,4 +1,4 @@
 dataset: posebusters_benchmark # the dataset to use - NOTE: must be one of (`posebusters_benchmark`, `astex_diverse`, `dockgen`, `casp15`)
 input_msa_dir: ${oc.env:PROJECT_ROOT}/data/${dataset}_set/${dataset}_msas # where the original MSA files are placed
 output_msa_dir: ${oc.env:PROJECT_ROOT}/data/${dataset}_set/${dataset}_chai_msas # where the processed MSA files should be stored
-skip_existing: True # whether to skip processing if the output file already exists
+skip_existing: true # whether to skip processing if the output file already exists

configs/model/boltz_inference.yaml

@@ -0,0 +1,10 @@
+model: boltz1 # the model to use for inference - NOTE: must be one of (`boltz1`, `boltz2`)
+dataset: posebusters_benchmark # the dataset to use - NOTE: must be one of (`posebusters_benchmark`, `astex_diverse`, `dockgen`, `casp15`)
+input_dir: ${oc.env:PROJECT_ROOT}/forks/boltz/prediction_inputs/${dataset} # the input directory with which to run inference
+output_dir: ${oc.env:PROJECT_ROOT}/forks/boltz/prediction_outputs/${dataset}_${repeat_index} # the output directory to which to save the inference results
+cuda_device_index: 0 # the CUDA device to use for inference, or `null` to use CPU
+repeat_index: 1 # the repeat index to use for inference
+use_potentials: true # whether to use inference-time potentials to potentially improve pose plausibility
+skip_existing: true # whether to skip running inference if the prediction for a target already exists
+pocket_only_baseline: false # whether to run the pocket-only baseline
+max_num_inputs: null # if provided, the number of (dataset subset) inputs over which to run inference

configs/model/ensemble_generation.yaml

@@ -1,5 +1,5 @@
 # General inference arguments:
-ensemble_methods: [diffdock, dynamicbind, neuralplexer, rfaa] # the methods from which to gather predictions for ensembling - NOTE: must be one of (`diffdock`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `alphafold3`, `vina`, `tulip`)
+ensemble_methods: [diffdock, dynamicbind, neuralplexer, rfaa] # the methods from which to gather predictions for ensembling - NOTE: must be one of (`diffdock`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `alphafold3`, `vina`, `tulip`)
 generate_vina_scripts: false # whether to generate Vina scripts using other methods' binding site predictions - NOTE: `resume` must also be `true` when this is `true`, meaning other methods' predictions must have already been generated locally
 rank_single_method_intrinsically: true # whether to rank single-method predictions using either `consensus` or `vina` ranking (false) or instead using their intrinsic (explicit) rank assignment (true)
 output_bash_file_dir: ensemble_generation_scripts # the directory in which to save the generated Bash scripts
@@ -146,6 +146,9 @@ rfaa_inference_dir_name: null # the name of the inference output directory to us
 # Chai-1 inference arguments:
 chai_out_path: ${oc.env:PROJECT_ROOT}/forks/chai-lab/inference/chai-lab_ensemble_outputs # the output directory to which to write the predictions
 chai_skip_existing: true # whether to skip running inference if the prediction for a target already exists
+# Boltz inference arguments:
+boltz_out_path: ${oc.env:PROJECT_ROOT}/forks/boltz/inference/boltz_ensemble_outputs # the output directory to which to write the predictions
+boltz_skip_existing: true # whether to skip running inference if the prediction for a target already exists
 # AlphaFold 3 inference arguments:
 alphafold3_out_path: ${oc.env:PROJECT_ROOT}/forks/alphafold3/inference/alphafold3_ensemble_outputs # the output directory to which to write the predictions
 # Vina inference arguments:

configs/model/inference_relaxation.yaml

@@ -1,5 +1,5 @@
-method: diffdock # the method for which to relax predictions - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `alphafold3`, `vina`, `tulip`)
-vina_binding_site_method: p2rank # the method to use for Vina binding site prediction - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `alphafold3`, `p2rank`)
+method: diffdock # the method for which to relax predictions - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `alphafold3`, `vina`, `tulip`)
+vina_binding_site_method: p2rank # the method to use for Vina binding site prediction - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `alphafold3`, `p2rank`)
 dataset: posebusters_benchmark # the dataset for which to relax predictions - NOTE: must be one of (`posebusters_benchmark`, `astex_diverse`, `dockgen`, `casp15`)
 ensemble_ranking_method: consensus # the method with which to rank-order and select the top ensemble prediction for each target - NOTE: must be one of (`consensus`, `ff`)
 num_processes: 1 # the number of parallel processes to use for relaxation

configs/scripts/benchmark_baseline_compute_resources.yaml

@@ -1,5 +1,5 @@
-method: diffdock # the method for which to score predictions - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `vina`, `ensemble`)
-vina_binding_site_method: p2rank # the method to use for Vina binding site prediction - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `p2rank`)
+method: diffdock # the method for which to score predictions - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `vina`, `ensemble`)
+vina_binding_site_method: p2rank # the method to use for Vina binding site prediction - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `p2rank`)
 dataset: astex_diverse # the dataset to use - NOTE: must be one of (`posebusters_benchmark`, `astex_diverse`, `dockgen`, `casp15`)
 repeat_index: 1 # the repeat index which was used for inference
 max_num_inputs: 21 # the number of (dataset subset) inputs over which to benchmark each baseline method's compute resource usage

configs/scripts/build_inference_script.yaml

@@ -1,6 +1,6 @@
 # run arguments:
-method: diffdock # the method for which to score predictions - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `vina`, `ensemble`)
-vina_binding_site_method: p2rank # the method to use for Vina binding site prediction - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `p2rank`)
+method: diffdock # the method for which to score predictions - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `vina`, `ensemble`)
+vina_binding_site_method: p2rank # the method to use for Vina binding site prediction - NOTE: must be one of (`diffdock`, `fabind`, `dynamicbind`, `neuralplexer`, `flowdock`, `rfaa`, `chai-lab`, `boltz`, `p2rank`)
 ensemble_ranking_method: consensus # the method to use for ensemble ranking - NOTE: must be one of (`consensus`, `ff`)
 dataset: astex_diverse # the dataset to use - NOTE: must be one of (`posebusters_benchmark`, `astex_diverse`, `dockgen`, `casp15`)
 repeat_index: 1 # the repeat index which was used for inference
@@ -22,6 +22,7 @@ methods_to_sweep: [
     "flowdock",
     "rfaa",
     "chai-lab",
+    "boltz",
     "vina",
     "ensemble",
   ] # the methods to sweep