diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml new file mode 100644 index 0000000..196d07a --- /dev/null +++ b/.github/workflows/draft-pdf.yml @@ -0,0 +1,24 @@ +name: Draft PDF +on: [push] + +jobs: + paper: + runs-on: ubuntu-latest + name: Paper Draft + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build draft PDF + uses: openjournals/openjournals-draft-action@master + with: + journal: joss + # This should be the path to the paper within your repo. + paper-path: paper/paper.md + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: paper + # This is the output path where Pandoc will write the compiled + # PDF. Note, this should be the same directory as the input + # paper.md + path: paper.pdf diff --git a/README.md b/README.md index 48ee8f3..e7d69ca 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ ![Docs](https://github.com/Novartis/torchsurv/actions/workflows/docs.yml/badge.svg?branch=main) [![PyPI - Version](https://img.shields.io/pypi/v/torchsurv)](https://pypi.org/project/torchsurv/) [![arXiv](https://img.shields.io/badge/arXiv-2404.10761-f9f107.svg)](https://arxiv.org/abs/2404.10761) +[![status](https://camo.githubusercontent.com/22fa65b2a659780cddfac609463c5fe719e3ea82a28eb7a61e24b7c4e40eb56d/68747470733a2f2f6a6f73732e7468656f6a2e6f72672f7061706572732f30326437343936646132623963633334663961366530346361626632323938642f7374617475732e737667)](https://joss.theoj.org/papers/02d7496da2b9cc34f9a6e04cabf2298d) [![Documentation](https://img.shields.io/badge/GithubPage-Sphinx-blue)](https://opensource.nibr.com/torchsurv/) [![Downloads](https://static.pepy.tech/badge/torchsurv)](https://pepy.tech/project/torchsurv) diff --git a/paper/paper.bib b/paper/paper.bib new file mode 100644 index 0000000..ca86c9d --- /dev/null +++ b/paper/paper.bib @@ -0,0 +1,384 @@ +@article{davidson2019lifelines, + title={lifelines: survival analysis in Python}, + author={Davidson-Pilon, Cameron}, + journal={Journal of Open Source Software}, + volume={4}, + doi={10.21105/joss.01317}, + number={40}, + pages={1317}, + year={2019} +} + +@article{therneau2015survival, + title={Package ‘survival’}, + author={Therneau, Terry M and Lumley, Thomas}, + journal={R Top Doc}, + volume={128}, + doi={10.32614/CRAN.package.survival}, + number={10}, + pages={28--33}, + year={2015} +} + +@misc{paszke2019pytorch, + title={PyTorch: An Imperative Style, High-Performance Deep Learning Library}, + author={Adam Paszke and Sam Gross and Francisco Massa and Adam Lerer and James Bradbury and Gregory Chanan and Trevor Killeen and Zeming Lin and Natalia Gimelshein and Luca Antiga and Alban Desmaison and Andreas Köpf and Edward Yang and Zach DeVito and Martin Raison and Alykhan Tejani and Sasank Chilamkurthy and Benoit Steiner and Lu Fang and Junjie Bai and Soumith Chintala}, + year={2019}, + eprint={1912.01703}, + archivePrefix={arXiv}, + primaryClass={cs.LG}, + doi={10.48550/arXiv.1912.01703}, + url={https://arxiv.org/abs/1912.01703}, +} + +@article{polsterl2020scikit, + title={scikit-survival: A Library for Time-to-Event Analysis Built on Top of scikit-learn}, + author={P{\"o}lsterl, Sebastian}, + journal={The Journal of Machine Learning Research}, + volume={21}, + number={1}, + pages={8747--8752}, + year={2020}, + publisher={JMLRORG}, + doi={10.5281/zenodo.3352342} +} + +@article{potapov2023packagesurvAUC, + title={Package ‘survAUC’}, + author={Potapov, Sergej and Adler, Werner and Schmid, Matthias and Potapov, Maintainer Sergej}, + journal={Statistics in Medicine}, + volume={25}, + doi={10.32614/CRAN.package.survAUC}, + pages={3474--3486}, + year={2023} +} + +@inproceedings{he2020momentum, + title={Momentum contrast for unsupervised visual representation learning}, + author={He, Kaiming and Fan, Haoqi and Wu, Yuxin and Xie, Saining and Girshick, Ross}, + booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, + doi={10.1109/cvpr42600.2020.00975}, + pages={9729--9738}, + year={2020} +} + +@article{katzman2018deepsurv, + title={DeepSurv: personalized treatment recommender system using a Cox proportional hazards deep neural network}, + author={Katzman, Jared L and Shaham, Uri and Cloninger, Alexander and Bates, Jonathan and Jiang, Tingting and Kluger, Yuval}, + journal={BMC medical research methodology}, + volume={18}, + number={1}, + doi={10.1186/s12874-018-0482-1}, + pages={1--12}, + year={2018}, + publisher={BioMed Central} +} + +@article{Kvamme2019pycox, + author = {H{{\aa}}vard Kvamme and {{\O}}rnulf Borgan and Ida Scheel}, + title = {Time-to-Event Prediction with Neural Networks and Cox Regression}, + journal = {Journal of Machine Learning Research}, + year = {2019}, + volume = {20}, + number = {129}, + pages = {1--30}, + url = {http://jmlr.org/papers/v20/18-424.html} +} + +@inproceedings{nagpal2022auton, + title={auton-survival: An open-source package for regression, counterfactual estimation, evaluation and phenotyping with censored time-to-event data}, + author={Nagpal, Chirag and Potosnak, Willa and Dubrawski, Artur}, + booktitle={Machine Learning for Healthcare Conference}, + pages={585--608}, + year={2022}, + doi={10.48550/arXiv.2204.07276}, + organization={PMLR} +} + +@article{Cox1972, + title = {Regression Models and Life‐Tables}, + volume = {34}, + ISSN = {2517-6161}, + number = {2}, + journal = {Journal of the Royal Statistical Society: Series B (Methodological)}, + publisher = {Wiley}, + author = {Cox, D. R.}, + year = {1972}, + doi={10.1007/978-1-4612-4380-9_37}, + month = jan, + pages = {187–202} +} + +@article{Kzlers2018, + title = {The Weibull Distribution}, + volume = {15}, + ISSN = {1740-9713}, + number = {2}, + journal = {Significance}, + publisher = {Oxford University Press (OUP)}, + author = {Kızılers\"{u}, Ayşe and Kreer, Markus and Thomas, Anthony W.}, + year = {2018}, + month = apr, + pages = {10–11} +} + +@article{Carroll2003, + title = {On the use and utility of the Weibull model in the analysis of survival data}, + volume = {24}, + ISSN = {0197-2456}, + url = {http://dx.doi.org/10.1016/S0197-2456(03)00072-2}, + DOI = {10.1016/s0197-2456(03)00072-2}, + number = {6}, + journal = {Controlled Clinical Trials}, + publisher = {Elsevier BV}, + author = {Carroll, Kevin J.}, + year = {2003}, + month = dec, + pages = {682–701} +} + +@Manual{survivalpackage, + title = {A Package for Survival Analysis in R}, + author = {Terry M Therneau}, + year = {2024}, + note = {R package version 3.5-8}, + url = {https://CRAN.R-project.org/package=survival}, + doi={10.32614/CRAN.package.survival}, + } + +@Manual{torchlifeAbeywardana, + title = {torchlife: Survival Analysis using pytorch}, + author = {Sachinthaka Abeywardana}, + year = {2021}, + url = {https://sachinruk.github.io/torchlife//index.html}, + doi={10.32614/CRAN.package.survival}, + } + +@Manual{survAUCpackage, + title = {Estimators of prediction accuracy for time-to-event data}, + author = {Sergej Potapov and Werner Adler and Matthias Schmid}, + year = {2023}, + note = {R package version 1.2-0}, + doi={10.32614/CRAN.package.survAUC}, + url = {https://CRAN.R-project.org/package=survAUC}, + } + +@Manual{timeROCpackage, + title = {Time-Dependent ROC Curve and AUC for Censored Survival Data}, + author = {Paul Blanche}, + year = {2019}, + note = {R package version 0.4}, + url = {https://CRAN.R-project.org/package=timeROC}, + doi={10.32614/CRAN.package.timeROC} + } + +@Manual{risksetROCpackage, + title = {Riskset ROC Curve Estimation from Censored Survival Data}, + author = {Patrick J. Heagerty}, + year = {2022}, + note = {R package version 1.0.4.1}, + url = {https://CRAN.R-project.org/package=risksetROC}, + doi={10.32614/CRAN.package.risksetROC} + } + +@Manual{survivalROCpackage, + title = {Time-Dependent ROC Curve Estimation from Censored Survival Data}, + author = {Patrick J. Heagerty}, + year = {2022}, + note = {R package version 1.0.3.1}, + url = {https://CRAN.R-project.org/package=survivalROC}, + doi={10.32614/CRAN.package.survivalROC} + } + +@article{survcomppackage, + title = {survcomp: an R/Bioconductor package for performance assessment and comparison of survival models}, + volume = {27}, + ISSN = {1367-4803}, + url = {http://dx.doi.org/10.1093/bioinformatics/btr511}, + DOI = {10.1093/bioinformatics/btr511}, + number = {22}, + journal = {Bioinformatics}, + publisher = {Oxford University Press (OUP)}, + author = {Schr\"{o}der, Markus S. and Culhane, Aedín C. and Quackenbush, John and Haibe-Kains, Benjamin}, + year = {2011}, + month = sep, + pages = {3206–3208} +} + +@Manual{riskRegressionpackage, + title = {Risk Regression Models and Prediction Scores for Survival Analysis with Competing Risks}, + author = {Thomas Alexander Gerds and Johan Sebastian Ohlendorff and Paul Blanche and Rikke Mortensen and Marvin Wright and Nikolaj Tollenaar and John Muschelli and Ulla Brasch Mogensen and Brice Ozenne}, + year = {2023}, + note = {R package version 2023.12.21}, + url = {https://CRAN.R-project.org/package=riskRegression}, + doi={10.32614/CRAN.package.riskRegression} + } + +@Manual{SurvMetricspackage, + title = {Predictive Evaluation Metrics in Survival Analysis}, + author = {Hanpu Zhou and Xuewei Cheng and Sizheng Wang and Yi Zou and Hong Wang}, + year = {2022}, + note = {R package version 0.5.0}, + url = {https://CRAN.R-project.org/package=SurvMetrics}, + doi={10.32614/CRAN.package.SurvMetrics} + } + +@Manual{pecpackage, + title = {Prediction Error Curves for Risk Prediction Models in Survival Analysis}, + author = {Thomas A. Gerds}, + year = {2023}, + note = {R package version 2023.04.12}, + url = {https://CRAN.R-project.org/package=pec}, + doi={10.32614/CRAN.package.pec} + } + +@article{Heagerty2000, + title = {Time‐Dependent ROC Curves for Censored Survival Data and a Diagnostic Marker}, + volume = {56}, + ISSN = {1541-0420}, + url = {http://dx.doi.org/10.1111/j.0006-341x.2000.00337.x}, + DOI = {10.1111/j.0006-341x.2000.00337.x}, + number = {2}, + journal = {Biometrics}, + publisher = {Oxford University Press (OUP)}, + author = {Heagerty, Patrick J. and Lumley, Thomas and Pepe, Margaret S.}, + year = {2000}, + month = jun, + pages = {337–344} +} + +@article{Heagerty2005, + author = {Heagerty, Patrick J. and Zheng, Yingye}, + doi = {10.1111/j.0006-341x.2005.030814.x}, + issn = {1541-0420}, + journal = {Biometrics}, + month = {February}, + number = {1}, + pages = {92–105}, + publisher = {Oxford University Press (OUP)}, + title = {Survival Model Predictive Accuracy and ROC Curves}, + url = {http://dx.doi.org/10.1111/j.0006-341x.2005.030814.x}, + volume = {61}, + year = {2005} +} + + +@article{Blanche2013, + title = {Review and comparison of ROC curve estimators for a time‐dependent outcome with marker‐dependent censoring}, + volume = {55}, + ISSN = {1521-4036}, + url = {http://dx.doi.org/10.1002/bimj.201200045}, + DOI = {10.1002/bimj.201200045}, + number = {5}, + journal = {Biometrical Journal}, + publisher = {Wiley}, + author = {Blanche, Paul and Dartigues, Jean‐Fran\c{c}ois and Jacqmin‐Gadda, Hélène}, + year = {2013}, + month = jun, + pages = {687–704} +} + +@article{Uno2007, + title = {Evaluating Prediction Rules fort-Year Survivors With Censored Regression Models}, + volume = {102}, + ISSN = {1537-274X}, + url = {http://dx.doi.org/10.1198/016214507000000149}, + DOI = {10.1198/016214507000000149}, + number = {478}, + journal = {Journal of the American Statistical Association}, + publisher = {Informa UK Limited}, + author = {Uno, Hajime and Cai, Tianxi and Tian, Lu and Wei, L. J}, + year = {2007}, + month = jun, + pages = {527–537} +} + +@article{Blanche2018, + author = {Blanche, Paul and Kattan, Michael W and Gerds, Thomas A}, + doi = {10.1093/biostatistics/kxy006}, + issn = {1468-4357}, + journal = {Biostatistics}, + month = {February}, + number = {2}, + pages = {347–357}, + publisher = {Oxford University Press (OUP)}, + title = {The c-index is not proper for the evaluation of $t$-year predicted risks}, + url = {http://dx.doi.org/10.1093/biostatistics/kxy006}, + volume = {20}, + year = {2018} +} + +@article{Uno_2011, + author = {Uno, Hajime and Cai, Tianxi and Pencina, Michael J. and D’Agostino, Ralph B. and Wei, L. J.}, + doi = {10.1002/sim.4154}, + issn = {1097-0258}, + journal = {Statistics in Medicine}, + month = {January}, + number = {10}, + pages = {1105–1117}, + publisher = {Wiley}, + title = {On the C‐statistics for evaluating overall adequacy of risk prediction procedures with censored survival data}, + url = {http://dx.doi.org/10.1002/sim.4154}, + volume = {30}, + year = {2011} +} + +@article{Harrell1996, + author = {Harrell, Frank E. and Lee, Kerry L. and Mark, Daniel B.}, + doi = {10.1002/(sici)1097-0258(19960229)15:4<361::aid-sim168>3.0.co;2-4}, + issn = {1097-0258}, + journal = {Statistics in Medicine}, + month = {February}, + number = {4}, + pages = {361–387}, + publisher = {Wiley}, + title = {Multivariate Prognostic Models: Issues in Developing models, Evaluating Assumptions and Adequacy, and Measuring and Reducing Errors}, + url = {http://dx.doi.org/10.1002/(SICI)1097-0258(19960229)15:4<361::AID-SIM168>3.0.CO;2-4}, + volume = {15}, + year = {1996} +} + +@article{Graf_1999, + author = {Graf, Erika and Schmoor, Claudia and Sauerbrei, Willi and Schumacher, Martin}, + doi = {10.1002/(sici)1097-0258(19990915/30)18:17/18<2529::aid-sim274>3.0.co;2-5}, + issn = {1097-0258}, + journal = {Statistics in Medicine}, + month = {September}, + number = {17–18}, + pages = {2529–2545}, + publisher = {Wiley}, + title = {Assessment and comparison of prognostic classification schemes for survival data}, + url = {http://dx.doi.org/10.1002/(SICI)1097-0258(19990915/30)18:17/18<2529::AID-SIM274>3.0.CO;2-5}, + volume = {18}, + year = {1999} +} + +@article{Efron1977, + title = {The Efficiency of Cox’s Likelihood Function for Censored Data}, + volume = {72}, + ISSN = {1537-274X}, + url = {http://dx.doi.org/10.1080/01621459.1977.10480613}, + DOI = {10.1080/01621459.1977.10480613}, + number = {359}, + journal = {Journal of the American Statistical Association}, + publisher = {Informa UK Limited}, + author = {Efron, Bradley}, + year = {1977}, + month = sep, + pages = {557–565} +} + +@article{Breslow1975, + title = {Analysis of Survival Data under the Proportional Hazards Model}, + volume = {43}, + ISSN = {0306-7734}, + url = {http://dx.doi.org/10.2307/1402659}, + DOI = {10.2307/1402659}, + number = {1}, + journal = {International Statistical Review / Revue Internationale de Statistique}, + publisher = {JSTOR}, + author = {Breslow, N. E.}, + year = {1975}, + month = apr, + pages = {45} +} \ No newline at end of file diff --git a/paper/paper.md b/paper/paper.md new file mode 100644 index 0000000..cc21d58 --- /dev/null +++ b/paper/paper.md @@ -0,0 +1,154 @@ +--- +title: 'TorchSurv: A Lightweight Package for Deep Survival Analysis' +tags: + - Python + - Deep Learning + - Survival Analysis + - PyTorch +authors: + - name: Mélodie Monod + orcid: 0000-0001-6448-2051 + affiliation: 1 + - name: Peter Krusche + affiliation: 1 + - name: Qian Cao + affiliation: 2 + - name: Berkman Sahiner + affiliation: 2 + - name: Nicholas Petrick + affiliation: 2 + - name: David Ohlssen + affiliation: 3 + - name: Thibaud Coroller + orcid: 0000-0001-7662-8724 + corresponding: true + affiliation: 3 +affiliations: + - name: Novartis Pharma AG, Switzerland + index: 1 + - name: Center for Devices and Radiological Health, Food and Drug Administration, MD, USA + index: 2 + - name: Novartis Pharmaceuticals Corporation, NJ, USA + index: 3 +date: 29 July 2024 +bibliography: paper.bib +--- + +# Summary + +`TorchSurv` is a Python [package](https://pypi.org/project/torchsurv/) that serves as a companion tool to perform deep survival modeling within the `PyTorch` environment [@paszke2019pytorch]. With its lightweight design, minimal input requirements, full `PyTorch` backend, and freedom from restrictive parameterizations, `TorchSurv` facilitates efficient deep survival model implementation and is particularly beneficial for high-dimensional and complex data scenarios. At its core, `TorchSurv` features calculations of log-likelihoods for prominent survival models (Cox proportional hazards model [@Cox1972], Weibull Accelerated Time Failure (AFT) model [@Carroll2003]) and offers evaluation metrics, including the time-dependent Area Under under the Receiver operating characteristic (ROC) curve (AUC), the Concordance index (C-index) and the Brier Score. +`TorchSurv` has been rigorously [tested](https://opensource.nibr.com/torchsurv/benchmarks.html) using both open-source and synthetically generated survival data, against R and python packages. The package is thoroughly documented and includes illustrative examples. The latest documentation for TorchSurv can be found on our [website](https://opensource.nibr.com/torchsurv/). + +# Statement of need + +Survival analysis plays a crucial role in various domains, such as medicine, economics or engineering. Thus, sophisticated survival models sugin deep learning opens new opportunities to leverage complex dataset and relationships. However, no existing library provides the flexibility to define the survival model's parameters using a custom `PyTorch`-based neural network. + +\autoref{tab:bibliography} compares the functionalities of `TorchSurv` with those of +`auton-survival` [@nagpal2022auton], +`pycox` [@Kvamme2019pycox], +`torchlife` [@torchlifeAbeywardana], +`scikit-survival` [@polsterl2020scikit], +`lifelines` [@davidson2019lifelines], and +`deepsurv` [@katzman2018deepsurv]. +Existing libraries constrain users to predefined forms for defining the parameters (e.g., linear function of covariates). While there exist log-likelihood functions in the libraries, they cannot be leveraged. +The limitations on the log-likelihood functions include protected functions (locked model architecture), specialized input requirements (format or class type), and reliance on external libraries like `NumPy` or `Pandas`. Dependence on external libraries hinders automatic gradient calculation within `PyTorch`. Additionally, the implementation of likelihood functions instead of log-likelihood functions, as done by some packages, introduces numerical instability. + +![**Survival analysis libraries in Python.** $^1$[@nagpal2022auton], $^{2}$[@Kvamme2019pycox], $^{3}$[@torchlifeAbeywardana], $^{4}$[@polsterl2020scikit], $^{5}$[@davidson2019lifelines], $^{6}$[@katzman2018deepsurv]. A green tick indicates a fully supported feature, a red cross indicates an unsupported feature, a blue crossed tick indicates a partially supported feature. For computing the concordance index, `pycox` requires the use of the estimated survival function as the risk score and does not support other types of time-dependent risk scores. `scikit-survival` does not support time-dependent risk scores in both the concordance index and AUC computation. Additionally, both `pycox` and `scikit-survival` impose the use of inverse probability of censoring weighting (IPCW) for subject-specific weights. `scikit-survival` only offers the Breslow approximation of the Cox partial log-likelihood in case of ties in the event time.\label{tab:bibliography}](table_1.png) + +![**Survival analysis libraries in R.** $^1$[@survivalpackage], $^{2}$[@survAUCpackage], $^{3}$[@timeROCpackage], $^{4}$[@risksetROCpackage], $^{5}$[@survcomppackage], $^{6}$[@survivalROCpackage], $^{7}$[@riskRegressionpackage], $^{8}$[@SurvMetricspackage], $^{9}$[@pecpackage]. A green tick indicates a fully supported feature, a red cross indicates an unsupported feature, a blue crossed tick indicates a partially supported feature. For obtaining the evaluation metrics, packages `survival`, `riskRegression`, `SurvMetrics` and `pec` require the fitted model object as input (a specific object format) and `RisksetROC` imposes to use a smoothing method. Packages `timeROC`, `riskRegression` and `pec` force the user to choose a form for subject-specific weights (e.g., inverse probability of censoring weighting (IPCW)). Packages `survcomp` and `SurvivalROC` do not implement the general AUC but the censoring-adjusted AUC estimator proposed by @Heagerty2000.\label{tab:bibliography_R}](table_2.png) + + +# Functionality + +## Loss functions + +**Cox loss function** is defined as the negative of the Cox proportional hazards model's partial log-likelihood [@Cox1972]. The function requires the subject-specific log relative hazards and the survival response (i.e., event indicator and time-to-event or censoring). In case of ties in the event times, the user can choose between the Breslow method [@Breslow1975] and the Efron method [@Efron1977] to approximate the Cox partial log-likelihood. +```python +from torchsurv.loss import cox + +# PyTorch model outputs ONE log hazard per observation +my_model = MyPyTorchCoxModel() + +for data in dataloader: + x, event, time = data # covariate, event indicator, time + log_hzs = my_model(x) # torch.Size([64, 1]), if batch size is 64 + loss = cox.neg_partial_log_likelihood(log_hzs, event, time) + loss.backward() # native torch backend +``` + +**Weibull loss function** is defined as the negative of the Weibull AFT's log-likelihood [@Carroll2003]. The function requires the subject-specific log parameters of the Weibull distribution (i.e., the log scale and the log shape) and the survival response. The log parameters of the Weibull distribution should be obtained from a `PyTorch`-based model pre-specified by the user. + +```python +from torchsurv.loss import weibull + +# PyTorch model outputs TWO Weibull parameters per observation +my_model = MyPyTorcWeibullhModel() + +for data in dataloader: + x, event, time = data + log_params = my_model(x) # torch.Size([64, 2]), if batch size is 64 + loss = weibull.neg_log_likelihood(log_params, event, time) + loss.backward() +``` + +**Momentum** helps training model when the batch size is greatly limited by computational resources (i.e. large files). This impacts the stability of model optimization, especially when rank-based loss is used. Inspired from MoCO [@he2020momentum], we implemented a momentum loss that decouples batch size from survival loss, increasing the effective batch size and allowing robust train of a model, even when using a very limited batch size (e.g., $\leq 16$). + +```python +from torchsurv.loss import Momentum +my_model = MyPyTorchXCoxModel() +my_loss = cox.neg_partial_log_likelihood # Works with any TorchSurv loss +momentum = Momentum(backbone=my_model, loss=my_loss) + +for data in dataloader: + x, event, time = data + loss = model_momentum(x, event, time) # torch.Size([16, 1]) + loss.backward() + +# Inference is computed with target network (k) +log_hzs = model_momentum.infer(x) # torch.Size([16, 1]) +``` + +## Evaluation Metrics Functions + +The `TorchSurv` package offers a comprehensive set of metrics to evaluate the predictive performance of survival models, including the `AUC`, `C-index`, and `Brier score`. The inputs of the evaluation metrics functions are the subject-specific risk score estimated on the test set and the survival response on the test set. The risk score measures the risk (or a proxy thereof) that a subject has an event. We provide definitions for each metric and demonstrate their use through illustrative code snippets. + +**AUC.** The AUC measures the discriminatory capacity of the survival model at a given time $t$, i.e., the ability to provide a reliable ranking of times-to-event based on estimated subject-specific risk scores [@Heagerty2005;@Uno2007;@Blanche2013]. + +```python +from torchsurv.metrics import Auc +auc = Auc() +auc(log_hzs, event, time) # AUC at each time +auc(log_hzs, event, time, new_time=torch.tensor(10.)) # AUC at time 10 +``` + +**C-index.** The C-index is a generalization of the AUC that represents the assessment of the discriminatory capacity of the survival model across the entire time period [@Harrell1996;@Uno_2011]. + +```python +from torchsurv.metrics import ConcordanceIndex +cindex = ConcordanceIndex() +cindex(log_hzs, event, time) +``` + +**Brier Score.** The Brier score evaluates the accuracy of a model at a given time $t$ [@Graf_1999]. It represents the average squared distance between the observed survival status and the predicted survival probability. The Brier score cannot be obtained for the Cox proportional hazards model because the survival function is not available, but it can be obtained for the Weibull ATF model. + +```python +from torchsurv.metrics import Brier +surv = survival_function(log_params, time) +brier = Brier() +brier(surv, event, time) # Brier score at each time +brier.integral() # Integrated Brier score over time +``` + +**Additional features.** In `TorchSurv`, the evaluation metrics can be obtained for time-dependent and time-independent risk scores (e.g., for proportional and non-proportional hazards). Additionally, subjects can be optionally weighted (e.g., by the inverse probability of censoring weighting (IPCW)). Lastly, functionalities including the confidence interval, one-sample hypothesis test to determine whether the metric is better than that of a random predictor, and two-sample hypothesis test to compare two evaluation metrics between different models are implemented. The following code snippet exemplifies the aforementioned functionalities for the C-index. + +```python +cindex.confidence_interval() # CI, default alpha=.05 +cindex.p_value(alternative='greater') # pvalue, H0:c=0.5, HA:c>0.5 +cindex.compare(cindex_other) # pvalue, H0:c1=c2, HA:c1>c2 +``` + +# Conflicts of interest + +MM, PK, DO and TC are employees and stockholders of Novartis, a global pharmaceutical company. + +# References diff --git a/paper/table_1.png b/paper/table_1.png new file mode 100644 index 0000000..efac5e0 Binary files /dev/null and b/paper/table_1.png differ diff --git a/paper/table_2.png b/paper/table_2.png new file mode 100644 index 0000000..b76bcd6 Binary files /dev/null and b/paper/table_2.png differ