diff --git a/reports/paper.bib b/reports/paper.bib index a1e023b..280d71b 100644 --- a/reports/paper.bib +++ b/reports/paper.bib @@ -329,6 +329,27 @@ @misc{zhao_physics_2022 file = {C\:\\Users\\sterg\\Zotero\\storage\\FFITBMRW\\Zhao et al. - 2022 - Physics Guided Generative Adversarial Networks for.pdf} } +@article{choudhary_jarvis-leaderboard_2024, + title = {{{JARVIS-Leaderboard}}: A Large Scale Benchmark of Materials Design Methods}, + shorttitle = {{{JARVIS-Leaderboard}}}, + author = {Choudhary, Kamal and Wines, Daniel and Li, Kangming and Garrity, Kevin F. and Gupta, Vishu and Romero, Aldo H. and Krogel, Jaron T. and Saritas, Kayahan and Fuhr, Addis and Ganesh, Panchapakesan and Kent, Paul R. C. and Yan, Keqiang and Lin, Yuchao and Ji, Shuiwang and Blaiszik, Ben and Reiser, Patrick and Friederich, Pascal and Agrawal, Ankit and Tiwary, Pratyush and Beyerle, Eric and Minch, Peter and Rhone, Trevor David and Takeuchi, Ichiro and Wexler, Robert B. and {Mannodi-Kanakkithodi}, Arun and Ertekin, Elif and Mishra, Avanish and Mathew, Nithin and Wood, Mitchell and Rohskopf, Andrew Dale and {Hattrick-Simpers}, Jason and Wang, Shih-Han and Achenie, Luke E. K. and Xin, Hongliang and Williams, Maureen and Biacchi, Adam J. and Tavazza, Francesca}, + year = {2024}, + month = may, + journal = {npj Comput Mater}, + volume = {10}, + number = {1}, + pages = {1--17}, + publisher = {{Nature Publishing Group}}, + issn = {2057-3960}, + doi = {10.1038/s41524-024-01259-w}, + urldate = {2024-05-25}, + abstract = {Lack of rigorous reproducibility and validation are significant hurdles for scientific development across many fields. Materials science, in particular, encompasses a variety of experimental and theoretical approaches that require careful benchmarking. Leaderboard efforts have been developed previously to mitigate these issues. However, a comprehensive comparison and benchmarking on an integrated platform with multiple data modalities with perfect and defect materials data is still lacking. This work introduces JARVIS-Leaderboard, an open-source and community-driven platform that facilitates benchmarking and enhances reproducibility. The platform allows users to set up benchmarks with custom tasks and enables contributions in the form of dataset, code, and meta-data submissions. We cover the following materials design categories: Artificial Intelligence (AI), Electronic Structure (ES), Force-fields (FF), Quantum Computation (QC), and Experiments (EXP). For AI, we cover several types of input data, including atomic structures, atomistic images, spectra, and text. For ES, we consider multiple ES approaches, software packages, pseudopotentials, materials, and properties, comparing results to experiment. For FF, we compare multiple approaches for material property predictions. For QC, we benchmark Hamiltonian simulations using various quantum algorithms and circuits. Finally, for experiments, we use the inter-laboratory approach to establish benchmarks. There are 1281 contributions to 274 benchmarks using 152 methods with more than 8 million data points, and the leaderboard is continuously expanding. The JARVIS-Leaderboard is available at the website: https://pages.nist.gov/jarvis\_leaderboard/}, + copyright = {2024 This is a U.S. Government work and not under copyright protection in the US; foreign copyright protection may apply}, + langid = {english}, + keywords = {Computational methods,Electronic properties and materials}, + file = {C:\Users\sterg\Zotero\storage\WU9JWMUL\Choudhary et al_2024_JARVIS-Leaderboard.pdf} +} + @misc{choudhary_large_2023, title = {Large {{Scale Benchmark}} of {{Materials Design Methods}}}, author = {Choudhary, Kamal and Wines, Daniel and Li, Kangming and Garrity, Kevin F. and Gupta, Vishu and Romero, Aldo H. and Krogel, Jaron T. and Saritas, Kayahan and Fuhr, Addis and Ganesh, Panchapakesan and Kent, Paul R. C. and Yan, Keqiang and Lin, Yuchao and Ji, Shuiwang and Blaiszik, Ben and Reiser, Patrick and Friederich, Pascal and Agrawal, Ankit and Tiwary, Pratyush and Beyerle, Eric and Minch, Peter and Rhone, Trevor David and Takeuchi, Ichiro and Wexler, Robert B. and {Mannodi-Kanakkithodi}, Arun and Ertekin, Elif and Mishra, Avanish and Mathew, Nithin and Baird, Sterling G. and Wood, Mitchell and Rohskopf, Andrew Dale and {Hattrick-Simpers}, Jason and Wang, Shih-Han and Achenie, Luke E. K. and Xin, Hongliang and Williams, Maureen and Biacchi, Adam J. and Tavazza, Francesca},