Pretraining chapter [WIP] (#128)

* What is pretraining?

* Added more info about different applications, please paraphrase

* Add a starting point

* Rough draft with some working code - Thanks Sam!

* add figs and refs

* removed old ref

* repeated paragraph

* Transfer learning code example

* Regression example for finetuning

* forgot to save the notebook

* Update pretraining.ipynb

* added future section and fixed installs

* quick text edits

Co-authored-by: Samantha Cox <[email protected]>
Co-authored-by: Sam Cox <[email protected]>

Author: hgandhi2411

_static/images/TL_FT.gif

@@ -37,6 +37,7 @@
         "rdkit>=2022",
         "sympy",
         "e3nn",
+        "simpletransformers",
     ],
     test_suite="tests",
     long_description="""

_static/images/ssl_graphs.png

@@ -1,3 +1,15 @@
+@article{martins2012bbb,
+author = {Martins, Ines Filipa and Teixeira, Ana L and Pinheiro, Luis and Falcao, Andre O},
+doi = {10.1021/ci300124c},
+journal = {Journal of Chemical Information and Modeling},
+number = {6},
+pages = {1686--1697},
+title = {{A Bayesian Approach to in Silico Blood-Brain Barrier Penetration Modeling}},
+url = {https://doi.org/10.1021/ci300124c},
+volume = {52},
+year = {2012}
+}
+
 @article{ramakrishnan2014quantum,
   title     = {Quantum chemistry structures and properties of 134 kilo molecules},
   author    = {Ramakrishnan, Raghunathan and Dral, Pavlo O and Rupp, Matthias and Von Lilienfeld, O Anatole},
@@ -1642,6 +1654,7 @@ @article{zhang2021motif
   pages={15870--15882},
   year={2021}
 }
+
 @article{you2020graph,
   title={Graph contrastive learning with augmentations},
   author={You, Yuning and Chen, Tianlong and Sui, Yongduo and Chen, Ting and Wang, Zhangyang and Shen, Yang},
@@ -1650,19 +1663,22 @@ @article{you2020graph
   pages={5812--5823},
   year={2020}
 }
+
 @inproceedings{sun2021mocl,
   title={MoCL: data-driven molecular fingerprint via knowledge-aware contrastive learning from molecular graph},
   author={Sun, Mengying and Xing, Jing and Wang, Huijun and Chen, Bin and Zhou, Jiayu},
   booktitle={Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery \& Data Mining},
   pages={3585--3594},
   year={2021}
 }
+
 @article{liu2021pre,
   title={Pre-training molecular graph representation with 3d geometry},
   author={Liu, Shengchao and Wang, Hanchen and Liu, Weiyang and Lasenby, Joan and Guo, Hongyu and Tang, Jian},
   journal={arXiv preprint arXiv:2110.07728},
   year={2021}
 }
+
 @inproceedings{erhan2010does,
   title={Why does unsupervised pre-training help deep learning?},
   author={Erhan, Dumitru and Courville, Aaron and Bengio, Yoshua and Vincent, Pascal},
@@ -1671,19 +1687,22 @@ @inproceedings{erhan2010does
   year={2010},
   organization={JMLR Workshop and Conference Proceedings}
 }
+
 @article{xie2022self,
   title={Self-supervised learning of graph neural networks: A unified review},
   author={Xie, Yaochen and Xu, Zhao and Zhang, Jingtun and Wang, Zhengyang and Ji, Shuiwang},
   journal={IEEE Transactions on Pattern Analysis and Machine Intelligence},
   year={2022},
   publisher={IEEE}
 }
+
 @article{mao2020survey,
   title={A survey on self-supervised pre-training for sequential transfer learning in neural networks},
   author={Mao, Huanru Henry},
   journal={arXiv preprint arXiv:2007.00800},
   year={2020}
 }
+
 @inproceedings{finn2017model,
   title={Model-agnostic meta-learning for fast adaptation of deep networks},
   author={Finn, Chelsea and Abbeel, Pieter and Levine, Sergey},
@@ -1692,6 +1711,7 @@ @inproceedings{finn2017model
   year={2017},
   organization={PMLR}
 }
+
 @inproceedings{wang2019smiles,
   title={SMILES-BERT: large scale unsupervised pre-training for molecular property prediction},
   author={Wang, Sheng and Guo, Yuzhi and Wang, Yuhong and Sun, Hongmao and Huang, Junzhou},