@inproceedings{dayan1990reinforcement,
  author    = {Dayan, Peter},
  title     = {Reinforcement Comparison},
  editor    = {Touretzky, David S. and Elman, Jeffrey L. and Sejnowski, Terrence J. and Hinton, Geoffrey E.},
  booktitle = {Proceedings of the 1990 Connectionist Models Summer School},
  year      = {1990},
  pages     = {45--51},
  address   = {San Mateo, CA},
  publisher = {Morgan Kaufmann}
}


@phdthesis{sutton1984temporal,
  author   = {Sutton, Richard S.},
  title    = {Temporal Credit Assignment in Reinforcement Learning},
  school   = {University of Massachusetts, Amherst},
  year     = {1984},
  address  = {Amherst, MA},
  type     = {Ph.D. dissertation}
}

@inproceedings{ouyang,
 author = {Ouyang, Long and Wu, Jeffrey and Jiang, Xu and Almeida, Diogo and Wainwright, Carroll and Mishkin, Pamela and Zhang, Chong and Agarwal, Sandhini and Slama, Katarina and Ray, Alex and Schulman, John and Hilton, Jacob and Kelton, Fraser and Miller, Luke and Simens, Maddie and Askell, Amanda and Welinder, Peter and Christiano, Paul F and Leike, Jan and Lowe, Ryan},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {S. Koyejo and S. Mohamed and A. Agarwal and D. Belgrave and K. Cho and A. Oh},
 pages = {27730--27744},
 publisher = {Curran Associates, Inc.},
 title = {Training language models to follow instructions with human feedback},
 url = {https://proceedings.neurips.cc/paper_files/paper/2022/file/b1efde53be364a73914f58805a001731-Paper-Conference.pdf},
 volume = {35},
 year = {2022}
}

@inproceedings{rafailov,
 author = {Rafailov, Rafael and Sharma, Archit and Mitchell, Eric and Manning, Christopher D and Ermon, Stefano and Finn, Chelsea},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {A. Oh and T. Naumann and A. Globerson and K. Saenko and M. Hardt and S. Levine},
 pages = {53728--53741},
 publisher = {Curran Associates, Inc.},
 title = {Direct Preference Optimization: Your Language Model is Secretly a Reward Model},
 url = {https://proceedings.neurips.cc/paper_files/paper/2023/file/a85b405ed65c6477a4fe8302b5e06ce7-Paper-Conference.pdf},
 volume = {36},
 year = {2023}
}


@misc{bai2022constitutionalaiharmlessnessai,
      title={Constitutional AI: Harmlessness from AI Feedback}, 
      author={Yuntao Bai and Saurav Kadavath and Sandipan Kundu and Amanda Askell and Jackson Kernion and Andy Jones and Anna Chen and Anna Goldie and Azalia Mirhoseini and Cameron McKinnon and Carol Chen and Catherine Olsson and Christopher Olah and Danny Hernandez and Dawn Drain and Deep Ganguli and Dustin Li and Eli Tran-Johnson and Ethan Perez and Jamie Kerr and Jared Mueller and Jeffrey Ladish and Joshua Landau and Kamal Ndousse and Kamile Lukosuite and Liane Lovitt and Michael Sellitto and Nelson Elhage and Nicholas Schiefer and Noemi Mercado and Nova DasSarma and Robert Lasenby and Robin Larson and Sam Ringer and Scott Johnston and Shauna Kravec and Sheer El Showk and Stanislav Fort and Tamera Lanham and Timothy Telleen-Lawton and Tom Conerly and Tom Henighan and Tristan Hume and Samuel R. Bowman and Zac Hatfield-Dodds and Ben Mann and Dario Amodei and Nicholas Joseph and Sam McCandlish and Tom Brown and Jared Kaplan},
      year={2022},
      eprint={2212.08073},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2212.08073}, 
}

@misc{guan2025deliberativealignmentreasoningenables,
      title={Deliberative Alignment: Reasoning Enables Safer Language Models}, 
      author={Melody Y. Guan and Manas Joglekar and Eric Wallace and Saachi Jain and Boaz Barak and Alec Helyar and Rachel Dias and Andrea Vallone and Hongyu Ren and Jason Wei and Hyung Won Chung and Sam Toyer and Johannes Heidecke and Alex Beutel and Amelia Glaese},
      year={2025},
      eprint={2412.16339},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2412.16339}, 
}

@misc{wei2023chainofthoughtpromptingelicitsreasoning,
      title={Chain-of-Thought Prompting Elicits Reasoning in Large Language Models}, 
      author={Jason Wei and Xuezhi Wang and Dale Schuurmans and Maarten Bosma and Brian Ichter and Fei Xia and Ed Chi and Quoc Le and Denny Zhou},
      year={2023},
      eprint={2201.11903},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2201.11903}, 
}

@inproceedings{
findeis2025inverse,
title={Inverse Constitutional {AI}: Compressing Preferences into Principles},
author={Arduin Findeis and Timo Kaufmann and Eyke H{\"u}llermeier and Samuel Albanie and Robert D. Mullins},
booktitle={The Thirteenth International Conference on Learning Representations},
year={2025},
url={https://openreview.net/forum?id=9FRwkPw3Cn}
}

@misc{zhang2024incontextprinciplelearningmistakes,
      title={In-Context Principle Learning from Mistakes}, 
      author={Tianjun Zhang and Aman Madaan and Luyu Gao and Steven Zheng and Swaroop Mishra and Yiming Yang and Niket Tandon and Uri Alon},
      year={2024},
      eprint={2402.05403},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2402.05403}, 
}

@misc{zhan2025sprialigninglargelanguage,
      title={SPRI: Aligning Large Language Models with Context-Situated Principles}, 
      author={Hongli Zhan and Muneeza Azmat and Raya Horesh and Junyi Jessy Li and Mikhail Yurochkin},
      year={2025},
      eprint={2502.03397},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2502.03397}, 
}

@inproceedings{franken,
 author = {Fr\"{a}nken, Jan-Philipp and Zelikman, Eric and Rafailov, Rafael and Gandhi, Kanishk and Gerstenberg, Tobias and Goodman, Noah D.},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {A. Globerson and L. Mackey and D. Belgrave and A. Fan and U. Paquet and J. Tomczak and C. Zhang},
 pages = {61328--61371},
 publisher = {Curran Associates, Inc.},
 title = {Self-Supervised Alignment with Mutual Information: Learning to Follow Principles without Preference Labels},
 url = {https://proceedings.neurips.cc/paper_files/paper/2024/file/70d638f3177d2f0bbdd9f400b43f0683-Paper-Conference.pdf},
 volume = {37},
 year = {2024}
}

@inproceedings{madaan,
 author = {Madaan, Aman and Tandon, Niket and Gupta, Prakhar and Hallinan, Skyler and Gao, Luyu and Wiegreffe, Sarah and Alon, Uri and Dziri, Nouha and Prabhumoye, Shrimai and Yang, Yiming and Gupta, Shashank and Majumder, Bodhisattwa Prasad and Hermann, Katherine and Welleck, Sean and Yazdanbakhsh, Amir and Clark, Peter},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {A. Oh and T. Naumann and A. Globerson and K. Saenko and M. Hardt and S. Levine},
 pages = {46534--46594},
 publisher = {Curran Associates, Inc.},
 title = {Self-Refine: Iterative Refinement with Self-Feedback},
 url = {https://proceedings.neurips.cc/paper_files/paper/2023/file/91edff07232fb1b55a505a9e9f6c0ff3-Paper-Conference.pdf},
 volume = {36},
 year = {2023}
}

@inproceedings{zelikman,
 author = {Zelikman, Eric and Wu, Yuhuai and Mu, Jesse and Goodman, Noah},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {S. Koyejo and S. Mohamed and A. Agarwal and D. Belgrave and K. Cho and A. Oh},
 pages = {15476--15488},
 publisher = {Curran Associates, Inc.},
 title = {STaR: Bootstrapping Reasoning With Reasoning},
 url = {https://proceedings.neurips.cc/paper_files/paper/2022/file/639a9a172c044fbb64175b5fad42e9a5-Paper-Conference.pdf},
 volume = {35},
 year = {2022}
}

@inproceedings{mt-bench,
 author = {Zheng, Lianmin and Chiang, Wei-Lin and Sheng, Ying and Zhuang, Siyuan and Wu, Zhanghao and Zhuang, Yonghao and Lin, Zi and Li, Zhuohan and Li, Dacheng and Xing, Eric and Zhang, Hao and Gonzalez, Joseph E and Stoica, Ion},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {A. Oh and T. Naumann and A. Globerson and K. Saenko and M. Hardt and S. Levine},
 pages = {46595--46623},
 publisher = {Curran Associates, Inc.},
 title = {Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena},
 url = {https://proceedings.neurips.cc/paper_files/paper/2023/file/91f18a1287b398d378ef22505bf41832-Paper-Datasets_and_Benchmarks.pdf},
 volume = {36},
 year = {2023}
}

@misc{alpaca_eval,
  author = {Xuechen Li and Tianyi Zhang and Yann Dubois and Rohan Taori and Ishaan Gulrajani and Carlos Guestrin and Percy Liang and Tatsunori B. Hashimoto },
  title = {AlpacaEval: An Automatic Evaluator of Instruction-following Models},
  year = {2023},
  month = {5},
  publisher = {GitHub},
  journal = {GitHub repository},
  howpublished = {\url{https://github.com/tatsu-lab/alpaca_eval}}
}

@misc{prometheus,
      title={Prometheus 2: An Open Source Language Model Specialized in Evaluating Other Language Models}, 
      author={Seungone Kim and Juyoung Suk and Shayne Longpre and Bill Yuchen Lin and Jamin Shin and Sean Welleck and Graham Neubig and Moontae Lee and Kyungjae Lee and Minjoon Seo},
      year={2024},
      eprint={2405.01535},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2405.01535}, 
}

@inproceedings{
sun2024salmon,
title={{SALMON}: Self-Alignment with Instructable Reward Models},
author={Zhiqing Sun and Yikang Shen and Hongxin Zhang and Qinhong Zhou and Zhenfang Chen and David Daniel Cox and Yiming Yang and Chuang Gan},
booktitle={The Twelfth International Conference on Learning Representations},
year={2024},
url={https://openreview.net/forum?id=xJbsmB8UMx}
}

@inproceedings{dromedary,
 author = {Sun, Zhiqing and Shen, Yikang and Zhou, Qinhong and Zhang, Hongxin and Chen, Zhenfang and Cox, David and Yang, Yiming and Gan, Chuang},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {A. Oh and T. Naumann and A. Globerson and K. Saenko and M. Hardt and S. Levine},
 pages = {2511--2565},
 publisher = {Curran Associates, Inc.},
 title = {Principle-Driven Self-Alignment of Language Models from Scratch with Minimal Human Supervision},
 url = {https://proceedings.neurips.cc/paper_files/paper/2023/file/0764db1151b936aca59249e2c1386101-Paper-Conference.pdf},
 volume = {36},
 year = {2023}
}

@inproceedings{petridis-etal-2024-constitutionalexperts,
    title = "{C}onstitutional{E}xperts: Training a Mixture of Principle-based Prompts",
    author = "Petridis, Savvas  and
      Wedin, Ben  and
      Yuan, Ann  and
      Wexler, James  and
      Thain, Nithum",
    editor = "Ku, Lun-Wei  and
      Martins, Andre  and
      Srikumar, Vivek",
    booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)",
    month = aug,
    year = "2024",
    address = "Bangkok, Thailand",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2024.acl-short.52/",
    doi = "10.18653/v1/2024.acl-short.52",
    pages = "574--582",
    abstract = "Large language models (LLMs) are highly capable at a variety of tasks given the right prompt, but writing one is still a difficult and tedious process. In this work, we introduce ConstitutionalExperts, a method for learning a prompt consisting of constitutional principles (i.e. rules), given a training dataset. Unlike prior methods that optimize the prompt as a single entity, our method incrementally improves the prompt by surgically editing individual principles. We also show that we can improve overall performance by learning unique prompts for different semantic regions of the training data and using a mixture-of-experts (MoE) architecture to route inputs at inference time. We compare our method to other state of the art prompt-optimization techniques across six benchmark datasets. We also investigate whether MoE improves these other techniques. Our results suggest that ConstitutionalExperts outperforms other prompt optimization techniques by 10.9{\%} (F1) and that mixture-of-experts improves all techniques, suggesting its broad applicability."
}

@inproceedings{chen-etal-2024-iteralign,
    title = "{I}ter{A}lign: Iterative Constitutional Alignment of Large Language Models",
    author = "Chen, Xiusi  and
      Wen, Hongzhi  and
      Nag, Sreyashi  and
      Luo, Chen  and
      Yin, Qingyu  and
      Li, Ruirui  and
      Li, Zheng  and
      Wang, Wei",
    editor = "Duh, Kevin  and
      Gomez, Helena  and
      Bethard, Steven",
    booktitle = "Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers)",
    month = jun,
    year = "2024",
    address = "Mexico City, Mexico",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2024.naacl-long.78/",
    doi = "10.18653/v1/2024.naacl-long.78",
    pages = "1423--1433",
    abstract = "With the rapid development of large language models (LLMs), aligning LLMs with human values and societal norms to ensure their reliability and safety has become crucial. Reinforcement learning with human feedback (RLHF) and Constitutional AI (CAI) have been proposed for LLM alignment. However, these methods require either heavy human annotations or explicitly pre-defined constitutions, which are labor-intensive and resource-consuming. To overcome these drawbacks, we study constitution-based LLM alignment and propose a data-driven constitution discovery and self-alignment framework called IterAlign. IterAlign leverages red teaming to unveil the weaknesses of an LLM and automatically discovers new constitutions using a stronger LLM. These constitutions are then used to guide self-correction of the base LLM. Such a constitution discovery pipeline can be run iteratively and automatically to discover new constitutions that specifically target the alignment gaps in the current LLM. Empirical results on several safety benchmark datasets and multiple base LLMs show that IterAlign successfully improves truthfulness, helpfulness, harmlessness and honesty, improving the LLM alignment by up to 13.5{\%} in harmlessness."
}

@misc{liu2025inferencetimescalinggeneralistreward,
      title={Inference-Time Scaling for Generalist Reward Modeling}, 
      author={Zijun Liu and Peiyi Wang and Runxin Xu and Shirong Ma and Chong Ruan and Peng Li and Yang Liu and Yu Wu},
      year={2025},
      eprint={2504.02495},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2504.02495}, 
}

@misc{openai2024gpt4ocard,
      title={GPT-4o System Card}, 
      author={OpenAI},
      year={2024},
      eprint={2410.21276},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2410.21276}, 
}

@misc{ramji2024selfrefinementlanguagemodelsexternal,
      title={Self-Refinement of Language Models from External Proxy Metrics Feedback}, 
      author={Keshav Ramji and Young-Suk Lee and Ramón Fernandez Astudillo and Md Arafat Sultan and Tahira Naseem and Asim Munawar and Radu Florian and Salim Roukos},
      year={2024},
      eprint={2403.00827},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2403.00827}, 
}

@misc{doosterlinck2024anchoredpreferenceoptimizationcontrastive,
      title={Anchored Preference Optimization and Contrastive Revisions: Addressing Underspecification in Alignment}, 
      author={Karel D'Oosterlinck and Winnie Xu and Chris Develder and Thomas Demeester and Amanpreet Singh and Christopher Potts and Douwe Kiela and Shikib Mehri},
      year={2024},
      eprint={2408.06266},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2408.06266}, 
}

@inproceedings{RISE,
 author = {Qu, Yuxiao and Zhang, Tianjun and Garg, Naman and Kumar, Aviral},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {A. Globerson and L. Mackey and D. Belgrave and A. Fan and U. Paquet and J. Tomczak and C. Zhang},
 pages = {55249--55285},
 publisher = {Curran Associates, Inc.},
 title = {Recursive Introspection: Teaching Language Model Agents How to Self-Improve},
 url = {https://proceedings.neurips.cc/paper_files/paper/2024/file/639d992f819c2b40387d4d5170b8ffd7-Paper-Conference.pdf},
 volume = {37},
 year = {2024}
}


@inproceedings{
kumar2025training,
title={Training Language Models to Self-Correct via Reinforcement Learning},
author={Aviral Kumar and Vincent Zhuang and Rishabh Agarwal and Yi Su and John D Co-Reyes and Avi Singh and Kate Baumli and Shariq Iqbal and Colton Bishop and Rebecca Roelofs and Lei M Zhang and Kay McKinney and Disha Shrivastava and Cosmin Paduraru and George Tucker and Doina Precup and Feryal Behbahani and Aleksandra Faust},
booktitle={The Thirteenth International Conference on Learning Representations},
year={2025},
url={https://openreview.net/forum?id=CjwERcAU7w}
}

@techreport{openai2025o3_o4mini_systemcard,
  title       = {{OpenAI o3 and o4-mini System Card}},
  author      = {{OpenAI}},
  institution = {OpenAI},
  year        = {2025},
  month       = apr,
  type        = {System Card},
  url         = {https://cdn.openai.com/pdf/2221c875-02dc-4789-800b-e7758f3722c1/o3-and-o4-mini-system-card.pdf}
}

@misc{deepseekai2025deepseekr1incentivizingreasoningcapability,
      title={DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning}, 
      author={DeepSeek-AI},
      year={2025},
      eprint={2501.12948},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2501.12948}, 
}

@inproceedings{huang-etal-2023-large,
    title = "Large Language Models Can Self-Improve",
    author = "Huang, Jiaxin  and
      Gu, Shixiang  and
      Hou, Le  and
      Wu, Yuexin  and
      Wang, Xuezhi  and
      Yu, Hongkun  and
      Han, Jiawei",
    editor = "Bouamor, Houda  and
      Pino, Juan  and
      Bali, Kalika",
    booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing",
    month = dec,
    year = "2023",
    address = "Singapore",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.emnlp-main.67/",
    doi = "10.18653/v1/2023.emnlp-main.67",
    pages = "1051--1068",
    abstract = "Large Language Models (LLMs) have achieved excellent performances in various tasks. However, fine-tuning an LLM requires extensive supervision. Human, on the other hand, may improve their reasoning abilities by self-thinking without external inputs. In this work, we demonstrate that an LLM is also capable of self-improving with only unlabeled datasets. We use a pre-trained LLM to generate {\textquotedblleft}high-confidence{\textquotedblright} rationale-augmented answers for unlabeled questions using Chain-of-Though (CoT) prompting and self-consistency, and fine-tune the LLM using those self-generated solutions as target outputs. We show that without any ground truth label, our approach improves the general reasoning ability of a 540B-parameter LLM (74.4{\%}$\rightarrow$82.1{\%} on GSM8K, 90.0{\%}$\rightarrow$94.4{\%} on OpenBookQA, and 63.4{\%}$\rightarrow$67.9{\%} on ANLI-A3) and can also be adapted to extreme low-resource cases where even training questions and CoT prompts are limited. We conduct ablation studies and show that fine-tuning on diverse reasoning paths is critical for self-improvement."
}

@misc{patel2024largelanguagemodelsselfimprove,
      title={Large Language Models Can Self-Improve At Web Agent Tasks}, 
      author={Ajay Patel and Markus Hofmarcher and Claudiu Leoveanu-Condrei and Marius-Constantin Dinu and Chris Callison-Burch and Sepp Hochreiter},
      year={2024},
      eprint={2405.20309},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2405.20309}, 
}

@inproceedings{
huang2025selfimprovement,
title={Self-Improvement in Language Models: The Sharpening Mechanism},
author={Audrey Huang and Adam Block and Dylan J Foster and Dhruv Rohatgi and Cyril Zhang and Max Simchowitz and Jordan T. Ash and Akshay Krishnamurthy},
booktitle={The Thirteenth International Conference on Learning Representations},
year={2025},
url={https://openreview.net/forum?id=WJaUkwci9o}
}

@inproceedings{SPIN,
author = {Chen, Zixiang and Deng, Yihe and Yuan, Huizhuo and Ji, Kaixuan and Gu, Quanquan},
title = {Self-play fine-tuning convertsweak language models to strong language models},
year = {2024},
publisher = {JMLR.org},
abstract = {Harnessing the power of human-annotated data through Supervised Fine-Tuning (SFT) is pivotal for advancing Large Language Models (LLMs). In this paper, we delve into the prospect of growing a strong LLM out of a weak one without the need for acquiring additional human-annotated data. We propose a new fine-tuning method called Self-Play fIne-tuNing (SPIN), which starts from a supervised fine-tuned model. At the heart of SPIN lies a self-play mechanism, where the LLM refines its capability by playing against instances of itself. More specifically, the LLM generates its own training data from its previous iterations, refining its policy by discerning these self-generated responses from those obtained from human-annotated data. Our method progressively elevates the LLM from a nascent model to a formidable one, unlocking the full potential of human-annotated demonstration data for SFT. Theoretically, we prove that the global optimum to the training objective function of our method is achieved only when the LLM policy aligns with the target data distribution. Empirically, we evaluate our method on several benchmark datasets including the HuggingFace Open LLM Leaderboard, MT-Bench, and datasets from Big-Bench. Our results show that SPIN can significantly improve the LLM's performance across a variety of benchmarks and even outperform models trained through direct preference optimization (DPO) supplemented with extra GPT-4 preference data. This sheds light on the promise of self-play, enabling the achievement of humanlevel performance in LLMs without the need for expert opponents. Codes are available at https://github.com/uclaml/SPIN.},
booktitle = {Proceedings of the 41st International Conference on Machine Learning},
articleno = {256},
numpages = {22},
location = {Vienna, Austria},
series = {ICML'24}
}

@inproceedings{
song2025mind,
title={Mind the Gap: Examining the Self-Improvement Capabilities of Large Language Models},
author={Yuda Song and Hanlin Zhang and Carson Eisenach and Sham M. Kakade and Dean Foster and Udaya Ghai},
booktitle={The Thirteenth International Conference on Learning Representations},
year={2025},
url={https://openreview.net/forum?id=mtJSMcF3ek}
}

@inproceedings{kojima,
 author = {Kojima, Takeshi and Gu, Shixiang (Shane) and Reid, Machel and Matsuo, Yutaka and Iwasawa, Yusuke},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {S. Koyejo and S. Mohamed and A. Agarwal and D. Belgrave and K. Cho and A. Oh},
 pages = {22199--22213},
 publisher = {Curran Associates, Inc.},
 title = {Large Language Models are Zero-Shot Reasoners},
 url = {https://proceedings.neurips.cc/paper_files/paper/2022/file/8bb0d291acd4acf06ef112099c16f326-Paper-Conference.pdf},
 volume = {35},
 year = {2022}
}


@inproceedings{ELV,
author = {Zhou, Wangchunshu and Hu, Jinyi and Zhang, Hanlin and Liang, Xiaodan and Sun, Maosong and Xiong, Chenyan and Tang, Jian},
title = {Towards interpretable natural language understanding with explanations as latent variables},
year = {2020},
isbn = {9781713829546},
publisher = {Curran Associates Inc.},
address = {Red Hook, NY, USA},
abstract = {Recently generating natural language explanations has shown very promising results in not only offering interpretable explanations but also providing additional information and supervision for prediction. However, existing approaches usually require a large set of human annotated explanations for training while collecting a large set of explanations is not only time consuming but also expensive. In this paper, we develop a general framework for interpretable natural language understanding that requires only a small set of human annotated explanations for training. Our framework treats natural language explanations as latent variables that model the underlying reasoning process of a neural model. We develop a variational EM framework for optimization where an explanation generation module and an explanation-augmented prediction module are alternatively optimized and mutually enhance each other. Moreover, we further propose an explanation-based self-training method under this framework for semi-supervised learning. It alternates between assigning pseudo-labels to unlabeled data and generating new explanations to iteratively improve each other. Experiments on two natural language understanding tasks demonstrate that our framework can not only make effective predictions in both supervised and semi-supervised settings, but also generate good natural language explanations.},
booktitle = {Proceedings of the 34th International Conference on Neural Information Processing Systems},
articleno = {571},
numpages = {12},
location = {Vancouver, BC, Canada},
series = {NIPS '20}
}

@inproceedings{TRICE,
author = {Phan, Du and Hoffman, Matthew D. and Dohan, David and Douglas, Sholto and Le, Tuan Anh and Parisi, Aaron and Sountsov, Pavel and Sutton, Charles and Vikram, Sharad and Saurous, Rif A.},
title = {Training chain-of-thought via latent-variable inference},
year = {2023},
publisher = {Curran Associates Inc.},
address = {Red Hook, NY, USA},
abstract = {Large language models (LLMs) solve problems more accurately and interpretably when instructed to work out the answer step by step using a "chain-of-thought" (CoT) prompt. One can also improve LLMs' performance on a specific task by supervised fine-tuning, i.e., by using gradient ascent on some tunable parameters to maximize the average log-likelihood of correct answers from a labeled training set. Naively combining CoT with supervised tuning requires supervision not just of the correct answers, but also of detailed rationales that lead to those answers; these rationales are expensive to produce by hand. Instead, we propose a fine-tuning strategy that tries to maximize the marginal log-likelihood of generating a correct answer using CoT prompting, approximately averaging over all possible rationales. The core challenge is sampling from the posterior over rationales conditioned on the correct answer; we address it using a simple Markov-chain Monte Carlo (MCMC) expectation-maximization (EM) algorithm inspired by the self-taught reasoner (STaR), memoized wake-sleep, Markovian score climbing, and persistent contrastive divergence. This algorithm also admits a novel control-variate technique that drives the variance of our gradient estimates to zero as the model improves. Applying our technique to GSM8K and the tasks in BIG-Bench Hard, we find that this MCMC-EM fine-tuning technique typically improves the model's accuracy on held-out examples more than STaR or prompt-tuning with or without CoT.},
booktitle = {Proceedings of the 37th International Conference on Neural Information Processing Systems},
articleno = {3184},
numpages = {23},
location = {New Orleans, LA, USA},
series = {NIPS '23}
}

@misc{chen2024languagemodelshiddenreasoners,
      title={Language Models are Hidden Reasoners: Unlocking Latent Reasoning Capabilities via Self-Rewarding}, 
      author={Haolin Chen and Yihao Feng and Zuxin Liu and Weiran Yao and Akshara Prabhakar and Shelby Heinecke and Ricky Ho and Phil Mui and Silvio Savarese and Caiming Xiong and Huan Wang},
      year={2024},
      eprint={2411.04282},
      archivePrefix={arXiv},
      primaryClass={cs.AI},
      url={https://arxiv.org/abs/2411.04282}, 
}

@misc{ruan2025reasoninglearnlatentthoughts,
      title={Reasoning to Learn from Latent Thoughts}, 
      author={Yangjun Ruan and Neil Band and Chris J. Maddison and Tatsunori Hashimoto},
      year={2025},
      eprint={2503.18866},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2503.18866}, 
}

@misc{phi4,
      title={Phi-4 Technical Report}, 
      author={Marah Abdin and Jyoti Aneja and Harkirat Behl and Sébastien Bubeck and Ronen Eldan and Suriya Gunasekar and Michael Harrison and Russell J. Hewett and Mojan Javaheripi and Piero Kauffmann and James R. Lee and Yin Tat Lee and Yuanzhi Li and Weishung Liu and Caio C. T. Mendes and Anh Nguyen and Eric Price and Gustavo de Rosa and Olli Saarikivi and Adil Salim and Shital Shah and Xin Wang and Rachel Ward and Yue Wu and Dingli Yu and Cyril Zhang and Yi Zhang},
      year={2024},
      eprint={2412.08905},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2412.08905}, 
}

@misc{granite,
  title        = {Granite 3.0 Language Models},
  author       = {Granite Team, IBM},
  year         = {2024},
  month        = oct,
  howpublished = {\url{https://www.rivista.ai/wp-content/uploads/2024/10/paper-1.pdf}},
}

@misc{ibm-granite-3.1-8b-instruct,
  title        = {Granite-3.1-8B-Instruct},
  author       = {{Granite Team and IBM}},
  year         = {2024},
  month        = dec,
  publisher    = {Hugging Face},
  howpublished = {\url{https://huggingface.co/ibm-granite/granite-3.1-8b-instruct}},
  note         = {Release Date: December 18, 2024.}
}


@misc{grattafiori2024llama3herdmodels,
  title         = {The Llama 3 Herd of Models},
  author        = {Grattafiori, Aaron and others},
  year          = {2024},
  eprint        = {2407.21783},
  archivePrefix = {arXiv},
  primaryClass  = {cs.AI},
  url           = {https://arxiv.org/abs/2407.21783},
}

@misc{meta-llama-3.1-8b-instruct,
  title        = {Llama-3.1-8B-Instruct},
  author       = {{Meta}},
  year         = {2024},
  month        = jul,
  publisher    = {Hugging Face},
  howpublished = {\url{https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct}},
  note         = {Release Date: July 23, 2024.}
}

@misc{qwen2025qwen25technicalreport,
      title={Qwen2.5 Technical Report}, 
      author={Qwen},
      year={2025},
      eprint={2412.15115},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2412.15115}, 
}

@misc{sentence-transformers-all-MiniLM-L6-v2,
  title        = {{all-MiniLM-L6-v2}},
  author       = {{Sentence Transformers}},
  year         = {2021},
  publisher    = {Hugging Face},
  howpublished = {\url{https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2}},
}

@inproceedings{
loshchilov2018decoupled,
title={Decoupled Weight Decay Regularization},
author={Ilya Loshchilov and Frank Hutter},
booktitle={International Conference on Learning Representations},
year={2019},
url={https://openreview.net/forum?id=Bkg6RiCqY7},
}

@inproceedings{yang-etal-2018-hotpotqa,
    title = "{H}otpot{QA}: A Dataset for Diverse, Explainable Multi-hop Question Answering",
    author = "Yang, Zhilin  and
      Qi, Peng  and
      Zhang, Saizheng  and
      Bengio, Yoshua  and
      Cohen, William  and
      Salakhutdinov, Ruslan  and
      Manning, Christopher D.",
    editor = "Riloff, Ellen  and
      Chiang, David  and
      Hockenmaier, Julia  and
      Tsujii, Jun{'}ichi",
    booktitle = "Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing",
    month = oct # "-" # nov,
    year = "2018",
    address = "Brussels, Belgium",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/D18-1259/",
    doi = "10.18653/v1/D18-1259",
    pages = "2369--2380",
    abstract = "Existing question answering (QA) datasets fail to train QA systems to perform complex reasoning and provide explanations for answers. We introduce HotpotQA, a new dataset with 113k Wikipedia-based question-answer pairs with four key features: (1) the questions require finding and reasoning over multiple supporting documents to answer; (2) the questions are diverse and not constrained to any pre-existing knowledge bases or knowledge schemas; (3) we provide sentence-level supporting facts required for reasoning, allowing QA systems to reason with strong supervision and explain the predictions; (4) we offer a new type of factoid comparison questions to test QA systems' ability to extract relevant facts and perform necessary comparison. We show that HotpotQA is challenging for the latest QA systems, and the supporting facts enable models to improve performance and make explainable predictions."
}

@misc{bai2022traininghelpfulharmlessassistant,
      title={Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback}, 
      author={Yuntao Bai and Andy Jones and Kamal Ndousse and Amanda Askell and Anna Chen and Nova DasSarma and Dawn Drain and Stanislav Fort and Deep Ganguli and Tom Henighan and Nicholas Joseph and Saurav Kadavath and Jackson Kernion and Tom Conerly and Sheer El-Showk and Nelson Elhage and Zac Hatfield-Dodds and Danny Hernandez and Tristan Hume and Scott Johnston and Shauna Kravec and Liane Lovitt and Neel Nanda and Catherine Olsson and Dario Amodei and Tom Brown and Jack Clark and Sam McCandlish and Chris Olah and Ben Mann and Jared Kaplan},
      year={2022},
      eprint={2204.05862},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2204.05862}, 
}

@misc{cui2024ultrafeedbackboostinglanguagemodels,
      title={UltraFeedback: Boosting Language Models with Scaled AI Feedback}, 
      author={Ganqu Cui and Lifan Yuan and Ning Ding and Guanming Yao and Bingxiang He and Wei Zhu and Yuan Ni and Guotong Xie and Ruobing Xie and Yankai Lin and Zhiyuan Liu and Maosong Sun},
      year={2024},
      eprint={2310.01377},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2310.01377}, 
}

@inproceedings{tldr,
 author = {Stiennon, Nisan and Ouyang, Long and Wu, Jeffrey and Ziegler, Daniel and Lowe, Ryan and Voss, Chelsea and Radford, Alec and Amodei, Dario and Christiano, Paul F},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M.F. Balcan and H. Lin},
 pages = {3008--3021},
 publisher = {Curran Associates, Inc.},
 title = {Learning to summarize with human feedback},
 url = {https://proceedings.neurips.cc/paper_files/paper/2020/file/1f89885d556929e98d3ef9b86448f951-Paper.pdf},
 volume = {33},
 year = {2020}
}

@misc{zhou2023instructionfollowingevaluationlargelanguage,
      title={Instruction-Following Evaluation for Large Language Models}, 
      author={Jeffrey Zhou and Tianjian Lu and Swaroop Mishra and Siddhartha Brahma and Sujoy Basu and Yi Luan and Denny Zhou and Le Hou},
      year={2023},
      eprint={2311.07911},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2311.07911}, 
}

@article{pr-latent-var-models,
  author  = {Kuzman Ganchev and Jo&#227;o Gra{\c{c}}a and Jennifer Gillenwater and Ben Taskar},
  title   = {Posterior Regularization for Structured Latent Variable Models},
  journal = {Journal of Machine Learning Research},
  year    = {2010},
  volume  = {11},
  number  = {67},
  pages   = {2001--2049},
  url     = {http://jmlr.org/papers/v11/ganchev10a.html}
}

@article{dubois2024length,
  title={Length-Controlled AlpacaEval: A Simple Way to Debias Automatic Evaluators},
  author={Dubois, Yann and Galambosi, Bal{\'a}zs and Liang, Percy and Hashimoto, Tatsunori B},
  journal={arXiv preprint arXiv:2404.04475},
  year={2024}
}

@misc{singhal2024longwaygoinvestigating,
      title={A Long Way to Go: Investigating Length Correlations in RLHF}, 
      author={Prasann Singhal and Tanya Goyal and Jiacheng Xu and Greg Durrett},
      year={2024},
      eprint={2310.03716},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2310.03716}, 
}

@misc{head2020scikitoptimize,
  author       = {Tim Head and Manoj Kumar and Holger Nahrstaedt and Gilles Louppe and Iaroslav Shcherbatyi},
  title        = {{scikit-optimize}: Sequential model-based optimization in Python},
  year         = {2020},
  month        = sep # "~4",
  publisher    = {Zenodo},
  doi          = {10.5281/zenodo.4014775},
  url          = {https://doi.org/10.5281/zenodo.4014775},
}


@misc{katsis2025mtragmultiturnconversationalbenchmark,
      title={MTRAG: A Multi-Turn Conversational Benchmark for Evaluating Retrieval-Augmented Generation Systems}, 
      author={Yannis Katsis and Sara Rosenthal and Kshitij Fadnis and Chulaka Gunasekara and Young-Suk Lee and Lucian Popa and Vraj Shah and Huaiyu Zhu and Danish Contractor and Marina Danilevsky},
      year={2025},
      eprint={2501.03468},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2501.03468}, 
}

@misc{deepseekai2025deepseekv3technicalreport,
      title={DeepSeek-V3 Technical Report}, 
      author={DeepSeek-AI and Aixin Liu and Bei Feng and Bing Xue and Bingxuan Wang and Bochao Wu and Chengda Lu and Chenggang Zhao and Chengqi Deng and Chenyu Zhang and Chong Ruan and Damai Dai and Daya Guo and Dejian Yang and Deli Chen and Dongjie Ji and Erhang Li and Fangyun Lin and Fucong Dai and Fuli Luo and Guangbo Hao and Guanting Chen and Guowei Li and H. Zhang and Han Bao and Hanwei Xu and Haocheng Wang and Haowei Zhang and Honghui Ding and Huajian Xin and Huazuo Gao and Hui Li and Hui Qu and J. L. Cai and Jian Liang and Jianzhong Guo and Jiaqi Ni and Jiashi Li and Jiawei Wang and Jin Chen and Jingchang Chen and Jingyang Yuan and Junjie Qiu and Junlong Li and Junxiao Song and Kai Dong and Kai Hu and Kaige Gao and Kang Guan and Kexin Huang and Kuai Yu and Lean Wang and Lecong Zhang and Lei Xu and Leyi Xia and Liang Zhao and Litong Wang and Liyue Zhang and Meng Li and Miaojun Wang and Mingchuan Zhang and Minghua Zhang and Minghui Tang and Mingming Li and Ning Tian and Panpan Huang and Peiyi Wang and Peng Zhang and Qiancheng Wang and Qihao Zhu and Qinyu Chen and Qiushi Du and R. J. Chen and R. L. Jin and Ruiqi Ge and Ruisong Zhang and Ruizhe Pan and Runji Wang and Runxin Xu and Ruoyu Zhang and Ruyi Chen and S. S. Li and Shanghao Lu and Shangyan Zhou and Shanhuang Chen and Shaoqing Wu and Shengfeng Ye and Shengfeng Ye and Shirong Ma and Shiyu Wang and Shuang Zhou and Shuiping Yu and Shunfeng Zhou and Shuting Pan and T. Wang and Tao Yun and Tian Pei and Tianyu Sun and W. L. Xiao and Wangding Zeng and Wanjia Zhao and Wei An and Wen Liu and Wenfeng Liang and Wenjun Gao and Wenqin Yu and Wentao Zhang and X. Q. Li and Xiangyue Jin and Xianzu Wang and Xiao Bi and Xiaodong Liu and Xiaohan Wang and Xiaojin Shen and Xiaokang Chen and Xiaokang Zhang and Xiaosha Chen and Xiaotao Nie and Xiaowen Sun and Xiaoxiang Wang and Xin Cheng and Xin Liu and Xin Xie and Xingchao Liu and Xingkai Yu and Xinnan Song and Xinxia Shan and Xinyi Zhou and Xinyu Yang and Xinyuan Li and Xuecheng Su and Xuheng Lin and Y. K. Li and Y. Q. Wang and Y. X. Wei and Y. X. Zhu and Yang Zhang and Yanhong Xu and Yanhong Xu and Yanping Huang and Yao Li and Yao Zhao and Yaofeng Sun and Yaohui Li and Yaohui Wang and Yi Yu and Yi Zheng and Yichao Zhang and Yifan Shi and Yiliang Xiong and Ying He and Ying Tang and Yishi Piao and Yisong Wang and Yixuan Tan and Yiyang Ma and Yiyuan Liu and Yongqiang Guo and Yu Wu and Yuan Ou and Yuchen Zhu and Yuduan Wang and Yue Gong and Yuheng Zou and Yujia He and Yukun Zha and Yunfan Xiong and Yunxian Ma and Yuting Yan and Yuxiang Luo and Yuxiang You and Yuxuan Liu and Yuyang Zhou and Z. F. Wu and Z. Z. Ren and Zehui Ren and Zhangli Sha and Zhe Fu and Zhean Xu and Zhen Huang and Zhen Zhang and Zhenda Xie and Zhengyan Zhang and Zhewen Hao and Zhibin Gou and Zhicheng Ma and Zhigang Yan and Zhihong Shao and Zhipeng Xu and Zhiyu Wu and Zhongyu Zhang and Zhuoshu Li and Zihui Gu and Zijia Zhu and Zijun Liu and Zilin Li and Ziwei Xie and Ziyang Song and Ziyi Gao and Zizheng Pan},
      year={2025},
      eprint={2412.19437},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2412.19437}, 
}

@inproceedings{kwon2023efficient,
  title={Efficient Memory Management for Large Language Model Serving with PagedAttention},
  author={Woosuk Kwon and Zhuohan Li and Siyuan Zhuang and Ying Sheng and Lianmin Zheng and Cody Hao Yu and Joseph E. Gonzalez and Hao Zhang and Ion Stoica},
  booktitle={Proceedings of the ACM SIGOPS 29th Symposium on Operating Systems Principles},
  year={2023}
}

@article{em,
author = {Dempster, A. P. and Laird, N. M. and Rubin, D. B.},
title = {Maximum Likelihood from Incomplete Data Via the EM Algorithm},
journal = {Journal of the Royal Statistical Society: Series B (Methodological)},
volume = {39},
number = {1},
pages = {1-22},
keywords = {maximum likelihood, incomplete data, em algorithm, posterior mode},
doi = {https://doi.org/10.1111/j.2517-6161.1977.tb01600.x},
url = {https://rss.onlinelibrary.wiley.com/doi/abs/10.1111/j.2517-6161.1977.tb01600.x},
eprint = {https://rss.onlinelibrary.wiley.com/doi/pdf/10.1111/j.2517-6161.1977.tb01600.x},
abstract = {Summary A broadly applicable algorithm for computing maximum likelihood estimates from incomplete data is presented at various levels of generality. Theory showing the monotone behaviour of the likelihood and convergence of the algorithm is derived. Many examples are sketched, including missing value situations, applications to grouped, censored or truncated data, finite mixture models, variance component estimation, hyperparameter estimation, iteratively reweighted least squares and factor analysis.},
year = {1977}
}

@article{mc-em,
 ISSN = {01621459, 1537274X},
 URL = {http://www.jstor.org/stable/2290005},
 abstract = {The first part of this article presents the Monte Carlo implementation of the E step of the EM algorithm. Given the current guess to the maximizer of the posterior distribution, latent data patterns are generated from the conditional predictive distribution. The expected value of the augmented log-posterior is then updated as a mixture of augmented log-posteriors, mixed over the generated latent data patterns (multiple imputations). In the M step of the algorithm, this mixture is maximized to obtain the update to the maximizer of the observed posterior. The gradient and Hessian of the observed log posterior are also expressed as mixtures, mixed over the multiple imputations. The relation between the Monte Carlo EM (MCEM) algorithm and the data augmentation algorithm is noted. Two modifications to the MCEM algorithm (the poor man's data augmentation algorithms), which allow for the calculation of the entire posterior, are then presented. These approximations serve as diagnostics for the validity of the normal approximation to the posterior, as well as starting points for the full data augmentation analysis. The methodology is illustrated with two examples.},
 author = {Greg C. G. Wei and Martin A. Tanner},
 journal = {Journal of the American Statistical Association},
 number = {411},
 pages = {699--704},
 publisher = {[American Statistical Association, Taylor & Francis, Ltd.]},
 title = {A Monte Carlo Implementation of the EM Algorithm and the Poor Man's Data Augmentation Algorithms},
 urldate = {2025-05-15},
 volume = {85},
 year = {1990}
}

@article{vonNeumann1951RandomDigits,
  author       = {von Neumann, John},
  title        = {Various Techniques Used in Connection with Random Digits},
  year         = {1951},
  note         = {J.\ Res.\ Natl.\ Bur.\ Stand.\ Appl.\ Math.\ Series, vol.\ 3, pp.\ 36--38 (1955).},
  url          = {https://mcnp.lanl.gov/pdf_files/InBook_Computing_1961_Neumann_JohnVonNeumannCollectedWorks_VariousTechniquesUsedinConnectionwithRandomDigits.pdf}
}

@misc{sutton2019bitter,
  author       = {Sutton, Richard S.},
  title        = {The Bitter Lesson},
  howpublished = {\url{https://www.cs.utexas.edu/~eunsol/courses/data/bitter_lesson.pdf}},
  year         = {2019},
}


@inproceedings{lin-2004-rouge,
    title = "{ROUGE}: A Package for Automatic Evaluation of Summaries",
    author = "Lin, Chin-Yew",
    booktitle = "Text Summarization Branches Out",
    month = jul,
    year = "2004",
    address = "Barcelona, Spain",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/W04-1013/",
    pages = "74--81"
}