{
  "@context": "https://agentflare.org/schema",
  "type": "ScholarlyArticle",
  "tier": "L2-full",
  "title": "Code Generation with LLMs: Research Digest",
  "description": "Foundational work covers surveys, planning, fine-tuning and test-based evaluation. Recent preprints move toward verifier-grounded reinforcement learning, algorithmic diversity benchmarks, and training and decoding for diffusion language models.",
  "canonical": "https://agentflare.org/scholar/code-generation-with-llms-research-digest.html",
  "category": "scholar",
  "updated": "2026-10-16",
  "generated_at": "2026-10-16T00:37:01.652Z",
  "facts": [
    {
      "label": "Field",
      "value": "code generation large language models"
    },
    {
      "label": "Foundational",
      "value": "10"
    },
    {
      "label": "Recent (arXiv)",
      "value": "8"
    },
    {
      "label": "Updated",
      "value": "2026-10-16"
    }
  ],
  "data": {
    "topic": "code generation large language models",
    "arxiv_query": "abs:\"code generation\" AND abs:\"language model\"",
    "generated_at": "2026-10-16T00:37:01.652Z",
    "papers": [
      {
        "title": "A survey on large language models for code generation",
        "link": "https://dl.acm.org/doi/abs/10.1145/3747588",
        "authors": "J Jiang, F Wang, J Shen, S Kim, S Kim",
        "citations": 1589,
        "year": "",
        "snippet": "… Large language models have become transformative tools for automated code generation, … of LLMs for code generation, as illustrated in Figure 1. The landscape of LLMs for …",
        "takeaway": "Surveys LLMs for code generation, describing them as transformative tools for automated code generation and mapping the landscape of models."
      },
      {
        "title": "Planning with large language models for code generation",
        "link": "https://arxiv.org/abs/2303.05510",
        "authors": "S Zhang, Z Chen, Y Shen, M Ding et al.",
        "citations": 361,
        "year": "2023",
        "snippet": "… Existing large language model-based code generation pipelines typically use beam search … generate achieve high token-matching-based scores, they often fail to compile or generate …",
        "takeaway": null
      },
      {
        "title": "Self-planning code generation with large language models",
        "link": "https://dl.acm.org/doi/abs/10.1145/3672456",
        "authors": "X Jiang, Y Dong, L Wang, Z Fang, Q Shang et al.",
        "citations": 556,
        "year": "",
        "snippet": "… (LLMs) have demonstrated impressive ability in code generation, … planning into code generation to help the model understand … code generation approach with large language models, …",
        "takeaway": "Proposes a self-planning approach that builds planning into code generation to help the model understand the problem."
      },
      {
        "title": "Large language models for code generation: A comprehensive survey of challenges, techniques, evaluation, and applications",
        "link": "https://arxiv.org/abs/2503.01245",
        "authors": "N Huynh, B Lin",
        "citations": 110,
        "year": "2025",
        "snippet": "… Large Language Models (LLMs) have … human languages to automatically generate executable code. We begin with understanding LLMs’ limitations and challenges in automated code …",
        "takeaway": null
      },
      {
        "title": "Usage of large language model for code generation tasks: A review",
        "link": "https://link.springer.com/article/10.1007/s42979-025-04241-5",
        "authors": "S Bistarelli, M Fiore, I Mercanti, M Mongiello",
        "citations": 44,
        "year": "2021",
        "snippet": "… They can be used for lots of applications, including assistance in code generation tasks. … of using a Large Language Model for the generation of simple and complex code. This paper …",
        "takeaway": "A PRISMA literature review of 66 papers from 2021 to 2023, finding Python dominant and a significant research gap on ethical constraints."
      },
      {
        "title": "Codet5+: Open code large language models for code understanding and generation",
        "link": "https://aclanthology.org/2023.emnlp-main.68/",
        "authors": "Y Wang, H Le, A Gotmare, N Bui, J Li et al.",
        "citations": 1166,
        "year": "2023",
        "snippet": "… SoTA results of 35.0% pass@1 and 54.5% pass@10 on the HumanEval code generation task against other open code LLMs, even surpassing the OpenAI code-cushman-001 model. …",
        "takeaway": null
      },
      {
        "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
        "link": "https://proceedings.neurips.cc/paper_files/paper/2023/hash/43e9d647ccd3e4b7b5baab53f0368686-Abstract-Conference.html",
        "authors": "J Liu, CS Xia, Y Wang, L Zhang",
        "citations": 2721,
        "year": "2023",
        "snippet": "… Program synthesis has been long studied with recent approaches focused on directly using the power of Large Language Models (LLMs) to generate code. Programming benchmarks, …",
        "takeaway": "Introduces EvalPlus, which augments benchmarks with many automatically generated test cases to rigorously assess the functional correctness of LLM-synthesized code."
      },
      {
        "title": "On the effectiveness of large language models in domain-specific code generation",
        "link": "https://dl.acm.org/doi/abs/10.1145/3697012",
        "authors": "X Gu, M Chen, Y Lin, Y Hu, H Zhang, C Wan et al.",
        "citations": 179,
        "year": "",
        "snippet": "… and challenge existing benchmarks, our work emphasizes a crucial aspect of code language models: their proficiency in generating domain-specific code. Specifically, we concentrate …",
        "takeaway": "Examines LLM proficiency in generating domain-specific code, a aspect that existing benchmarks do not emphasize."
      },
      {
        "title": "Exploring parameter-efficient fine-tuning techniques for code generation with large language models",
        "link": "https://dl.acm.org/doi/abs/10.1145/3714461",
        "authors": "M Weyssow, X Zhou, K Kim, D Lo et al.",
        "citations": 180,
        "year": "",
        "snippet": "… a large spectrum of SLMs and LLMs for code generation on two datasets covering codes of … Throughout our study, we compare all the studied models on a Python code generation task…",
        "takeaway": "Studies parameter-efficient fine-tuning across a large spectrum of small and large models on a Python code generation task."
      },
      {
        "title": "Large language model-aware in-context learning for code generation",
        "link": "https://dl.acm.org/doi/abs/10.1145/3715908",
        "authors": "J Li, C Tao, J Li♂, G Li, Z Jin, H Zhang et al.",
        "citations": 109,
        "year": "",
        "snippet": "… Existing studies have found that the performance of ICL-based code generation … a Large language model-Aware selection approach for In-context-Learning-based code generation …",
        "takeaway": "Proposes a large language model-aware selection approach for in-context-learning-based code generation."
      }
    ],
    "recent": [
      {
        "id": "2610.21427v1",
        "title": "Learning from Interaction Experience via Localized Context Distillation",
        "abstract": "Learning from interaction experience is essential for enabling large language models to continually improve during deployment. Context distillation offers a promising path: privileged information extracted from experience is provided to a teacher, whose behavior is then distilled into a student model that has no access to it. However, both existing forms of context distillation have limitations. Off-policy context distillation trains the student on teacher-generated trajectories and thus suffers from exposure bias; whereas on-policy context distillation trains on the student's own rollouts, whose states are often uninformative and fail to convey behaviors the student cannot yet produce. To resolve this dilemma, we introduce Localized Context Distillation (LoCD). LoCD first localizes the critical spans of a student trajectory that require revision. The teacher then regenerates only these spans from the student's own prefix, and the student is trained on the regenerated segments. LoCD thereby concentrates learning on the most informative states while keeping exposure bias low. Extensive experiments across math reasoning, code generation, and agent tasks demonstrate that LoCD consistently outperforms strong baselines and achieves more stable training with smaller and sparser parameter updates.",
        "published": "2026-10-15",
        "authors": [
          "Zhezheng Hao",
          "Yongkang Yang",
          "Xilin Xia",
          "Jiarui Yu",
          "Yi Liu",
          "Fanjunduo Wei",
          "Peijie Dong",
          "Tianfu Wang",
          "Qiang Lin",
          "Can Wang",
          "Hande Dong",
          "Jiawei Chen"
        ],
        "category": "cs.AI",
        "link": "https://arxiv.org/abs/2610.21427v1"
      },
      {
        "id": "2610.20497v2",
        "title": "Hindsight-Privileged RLVR: Single-Rollout Token Credit Without Group Sampling",
        "abstract": "Reinforcement learning with verifiable rewards (RLVR) trains language models from checked task outcomes, but assigning a terminal reward to individual reasoning tokens or actions remains difficult. Group-relative methods such as Group Relative Policy Optimization (GRPO) sample several attempts per prompt, but lose their reward-based learning signal when all attempts receive the same outcome. We propose Hindsight-Privileged RLVR (HP-RLVR), a single-rollout method that turns a completed trajectory's own future into token-level update weights. The generating policy snapshot scores each eligible token under its causal history and under a hindsight context containing the realized future and verifier feedback. A normalized Jensen--Shannon divergence between the two views weights the reward relative to a historical baseline, yielding bounded token credit for both successful and failed attempts, with no reference solution or learned critic. In paired Qwen3-8B runs over a shared 100-step schedule, HP-RLVR trains 1.21×, 1.46×, and 2.30× faster than GRPO on mathematics, code generation, and ALFWorld, using one eighth the rollouts per prompt. Accuracy is comparable overall, with HP-RLVR ahead on mathematics at all three sizes and GRPO ahead on code and ALFWorld. In ALFWorld, 43.2\\% (4B) and 49.0\\% (8B) of GRPO's gradient-norm records are zero, versus none for HP-RLVR.",
        "published": "2026-10-14",
        "authors": [
          "Wei Chen",
          "Yubing Wu",
          "Zhou Xu",
          "Haoyue Zhang",
          "Delu Zeng",
          "Jiahao Wang",
          "Junmei Yang",
          "Pan tiewen",
          "Jingfeng Liu",
          "Zhiheng Zhou",
          "Bo Li",
          "Min Chen"
        ],
        "category": "cs.LG",
        "link": "https://arxiv.org/abs/2610.20497v2"
      },
      {
        "id": "2610.19605v1",
        "title": "PN-Mix: Pair-Norm Guided Mixed-Precision Key Cache Quantization for Efficient Long-Output LLM Inference",
        "abstract": "Long-output inference in large language models suffers from severe KV cache memory and bandwidth bottlenecks. While KV cache quantization alleviates this issue, low-bit quantization (e.g., 2-bit) severely degrades model accuracy. Recent mixed-precision key-cache quantization methods mitigate this degradation by allocating higher precision to important key channels, but estimate channel saliency using only key statistics. Through attention logit perturbation analysis at the RoPE channel-pair level, we show that quantization errors in channel pairs with large query--key pair-norm products contribute significantly to logit variation. Based on this insight, we propose PN-Mix, a mixed-precision key-cache quantization method for RoPE-based decoder-only LLMs, which allocates higher precision to salient channel pairs identified once during the prefill stage and reused throughout decoding. Across standard reasoning, code-generation, and AIME benchmarks, PN-Mix with 2.26-bit average precision achieves comparable or better average accuracy than prior mixed-precision methods using 2.32--2.39 bits. Furthermore, in our Pareto sweep, PN-Mix degrades gracefully down to 1.5 bits, whereas the compared baselines exhibit sharper performance drops below 2.25 bits.",
        "published": "2026-10-14",
        "authors": [
          "Gil Jae Song",
          "Sungjoo Yoo"
        ],
        "category": "cs.LG",
        "link": "https://arxiv.org/abs/2610.19605v1"
      },
      {
        "id": "2610.19301v1",
        "title": "Beyond ICD Codes: Semantic IDs for Generative Medical Coding",
        "abstract": "Generative approaches to automatic International Classification of Diseases (ICD) coding use large language models (LLMs) to predict ICD codes from clinical documents. However, original ICD code strings are not ideal generation targets because the ICD hierarchy does not always reflect clinical relationships, and LLMs struggle to interpret and generate these codes. We introduce SemICD, a framework that trains LLMs to generate Semantic IDs (SIDs) instead. SemICD constructs SIDs by hierarchically clustering embeddings of textual code profiles, allowing semantically related codes to share prefixes while each complete SID uniquely identifies one ICD code. To align the SID tokens with clinical semantics, we employ two auxiliary tasks for bidirectional generation between textual code profiles and SIDs. Experiments on MIMIC-III and MIMIC-IV with four LLMs show that SemICD outperforms direct code generation, improving Micro-F1 by 1.34-8.00 points. Our code is available at https://github.com/youwillbethebest/SemICD.",
        "published": "2026-10-13",
        "authors": [
          "Yihang Cheng",
          "Qian Zhang",
          "Yonghui",
          "Huang",
          "Baoshuo Kan",
          "Junyi Shen"
        ],
        "category": "cs.AI",
        "link": "https://arxiv.org/abs/2610.19301v1"
      },
      {
        "id": "2610.19298v1",
        "title": "Verifier-Grounded Agentic RL for Provably Correct Code",
        "abstract": "Reinforcement learning for code usually rewards passing tests, which a model can do while remaining wrong on the inputs the tests do not cover. A program verifier instead decides whether a program satisfies a formal specification for all inputs. Language models are almost always paired with verifiers at inference time only, and turning a verifier into a reward raises two difficulties. It reports only acceptance or rejection, which gives no gradient when every attempt fails, and a policy can cheat it, most simply by telling it to assume what the proof was supposed to establish. We build an agentic environment and reward that address both. The reward reads the verifier's diagnostics rather than its verdict, scoring a candidate by the weighted fraction of proof obligations it discharges, extended with hidden probes from a reference proof's annotations. An integrity audit outranks the verifier, rejecting any candidate that alters the specification or assumes what it was meant to prove. The model works as an agent against a live verifier over many turns, with a separately trained proof specialist as a frozen collaborator. Across Qwen3 models from 1.7B to 14B and a 35B Qwen3.6 mixture-of-experts model, held-out verification on code generation rises from 66.1% to 93.5% on DafnyBench and from 56.3% to 100% on SWE-Proof. This reward leads every other training signal we compare, including a binary pass-or-fail reward trained under otherwise identical settings, with the widest margin on code generation. When the agent must also deliver the repository patch, graded by each project's withheld tests, the fraction of held-out issues it resolves rises from 59.6% to 72.5%. Evaluated unchanged on three verification suites withheld from training, the checkpoints improve on all of them, including one whose language and verifier are absent from training.",
        "published": "2026-10-13",
        "authors": [
          "George Ma",
          "Yuhao Zhang",
          "Bing He",
          "Zeren Shui",
          "Ferhat Erata",
          "Murali Krishna Ramanathan",
          "Somayeh Sojoudi",
          "Hao Zhou"
        ],
        "category": "cs.LG",
        "link": "https://arxiv.org/abs/2610.19298v1"
      },
      {
        "id": "2610.19157v1",
        "title": "DivAlgo: A systematic evaluation of algorithmic diversity for code generation models",
        "abstract": "Code generation models are actively being used in professional and educational settings. However, the competence of large language models (LLMs) to generate algorithmically diverse code remains largely unknown. We propose the DivAlgo benchmark, a large-scale dataset constituting algorithmically diverse solutions to programming problems. We also introduce a lightweight similarity metric to enable large-scale evaluation. Our experiments across 24 modern code generation models found that uncontrolled model-generated code is significantly less diverse than human solutions (capturing 31% of human diversity), and even the best diverse code generation strategy could achieve only 70% of human diversity. Moreover, while it was possible to steer the models to generate solutions based on human strategies, they were unable to reproduce the diversity of the human solutions. These findings indicate the limitations of code generation models in capturing the algorithmic diversity humans exhibit, and that overexposure to model-generated code can narrow programmers' algorithmic vocabulary.",
        "published": "2026-10-13",
        "authors": [
          "Anubhav Jangra",
          "Smaranda Muresan"
        ],
        "category": "cs.SE",
        "link": "https://arxiv.org/abs/2610.19157v1"
      },
      {
        "id": "2610.18001v1",
        "title": "Match the Canvas, Not Just the Answer: Predictability-Aware SFT for Diffusion Language Models",
        "abstract": "Diffusion language models (DLMs) enable flexible parallel generation by resolving tokens according to the model's evolving predictability rather than a fixed left-to-right order. This flexibility, however, introduces training choices absent from standard autoregressive teacher forcing: which reference tokens are visible when a target is supervised and how much generation space is allocated. Standard supervised fine-tuning (SFT) typically makes these choices through random masking, independently of inference. Our analysis shows how this training-inference mismatch can misdirect supervision. Random masking can conceal prediction errors or penalize uncertainty that decoding would otherwise resolve, while the attention-visible input length can change which response formulations the model favors, even when the underlying solution is unchanged. Motivated by these findings, we propose Predictability-Aware Canvas Selection and Supervision (PACS), an SFT method. PACS uses the model's own predictability to select each example's attention-visible input length and supervises reference tokens immediately before their confidence-guided revelation. Across reasoning and code-generation benchmarks, PACS consistently achieves the highest mean accuracy among the compared SFT methods, with gains of up to 12.84 percentage points over vanilla SFT on individual benchmarks. Code will be released upon publication.",
        "published": "2026-10-13",
        "authors": [
          "Miao Li",
          "Hanyang Jiang",
          "Hengyu Fu",
          "Baihe Huang",
          "Yating Liu",
          "Yuhang Cai",
          "Changkun Guan",
          "Zhongzhu Chen",
          "Jiecheng Lu",
          "Michael A. Boateng",
          "Pascal Van Hentenryck"
        ],
        "category": "cs.LG",
        "link": "https://arxiv.org/abs/2610.18001v1"
      },
      {
        "id": "2610.17948v1",
        "title": "HaloBlock: Faster and Better Decoding for Diffusion Large Language Models",
        "abstract": "Diffusion large language models (dLLMs) have attracted increasing research attention due to their bidirectional attention and arbitrary order decoding. However, there are two issues in dLLMs inference: the decoding difficulty is dominated by the tokens on the right side of the block, and the block-wise decoding inherently suffers from boundary contextual discontinuity. To tackle these limitations, we propose HaloBlock, a difficulty-aware adaptive halo mechanism for dLLM inference. We first leverage bidirectional cumulative entropy to characterize block decoding difficulty. Then, based on the bidirectional decoding difficulty, we adaptively allocate bidirectional halo regions on both sides of each block to mitigate boundary contextual discontinuity. The difficulty-aware design of HaloBlock smooths cross-block context propagation and lowers decoding difficulty for hard tokens, yielding better generation quality and faster convergence. Evaluated on LLaDA-8B-Instruct, Dream-7B-Instruct and LLaDA-1.5 across mathematical reasoning and code generation benchmarks, HaloBlock demonstrates consistent accuracy improvements and higher inference efficiency. Source codes are publicly available at https://github.com/PolyU-VCLab/HaloBlock.",
        "published": "2026-10-13",
        "authors": [
          "Zhaohui Zheng",
          "Chenhang He",
          "Jinrui Zhang",
          "Ming-Ming Cheng",
          "Lei Zhang"
        ],
        "category": "cs.LG",
        "link": "https://arxiv.org/abs/2610.17948v1"
      }
    ],
    "monthly_arxiv": [
      {
        "month": "2025-10",
        "count": 123
      },
      {
        "month": "2025-11",
        "count": 75
      },
      {
        "month": "2025-12",
        "count": 80
      },
      {
        "month": "2026-01",
        "count": 101
      },
      {
        "month": "2026-02",
        "count": 98
      },
      {
        "month": "2026-03",
        "count": 106
      },
      {
        "month": "2026-04",
        "count": 136
      },
      {
        "month": "2026-05",
        "count": 133
      },
      {
        "month": "2026-06",
        "count": 82
      },
      {
        "month": "2026-07",
        "count": 72
      },
      {
        "month": "2026-08",
        "count": 89
      },
      {
        "month": "2026-09",
        "count": 85
      }
    ],
    "themes": [
      {
        "title": "Surveys map the landscape and its gaps",
        "body": "Several review papers organize the field. Jiang and colleagues survey LLMs for code generation, and Huynh and Lin's comprehensive survey covers challenges, techniques, evaluation and applications. Bistarelli and colleagues apply the PRISMA methodology to 66 papers published between 2021 and 2023. They report Python as the preferred language and a significant research gap in addressing ethical constraints. They also compare models such as GPT-4 and CodeLlama on tasks from debugging to multi-turn program synthesis. Together these reviews give the baseline against which the newer work on evaluation and training can be read.",
        "papers": [
          "A survey on large language models for code generation",
          "Large language models for code generation: A comprehensive survey of challenges, techniques, evaluation, and applications",
          "Usage of large language model for code generation tasks: A review"
        ]
      },
      {
        "title": "Planning, prompting and adaptation of code models",
        "body": "A second line of work improves how models produce code without changing the task. Self-planning approaches build planning into code generation to help the model understand the problem, and a separate paper also studies planning with LLMs for code generation. Li and colleagues propose a large language model-aware selection approach for in-context learning. Weyssow and colleagues compare parameter-efficient fine-tuning across a large spectrum of small and large models on a Python code generation task. Gu and colleagues focus on domain-specific code. CodeT5+ presents open code LLMs for both understanding and generation.",
        "papers": [
          "Planning with large language models for code generation",
          "Self-planning code generation with large language models",
          "Large language model-aware in-context learning for code generation",
          "Exploring parameter-efficient fine-tuning techniques for code generation with large language models",
          "On the effectiveness of large language models in domain-specific code generation",
          "Codet5+: Open code large language models for code understanding and generation"
        ]
      },
      {
        "title": "Evaluation is moving beyond passing the given tests",
        "body": "EvalPlus argued that benchmark test cases can be limited in quantity and quality for assessing functional correctness. It augments datasets with test cases from an automatic test input generator that uses both LLM-based and mutation-based strategies. Recent preprints push the same concern further. One argues that rewarding passing tests lets a model stay wrong on inputs the tests do not cover, and turns to a program verifier that checks a formal specification for all inputs. Another introduces the DivAlgo benchmark to measure whether generated solutions are algorithmically diverse.",
        "papers": [
          "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
          "Verifier-Grounded Agentic RL for Provably Correct Code",
          "DivAlgo: A systematic evaluation of algorithmic diversity for code generation models"
        ]
      },
      {
        "title": "Learning signals and decoding for newer model families",
        "body": "Recent preprints concentrate on how models learn from outcomes and how they decode. Hindsight-Privileged RLVR assigns token-level credit from a single rollout, avoiding group sampling. Localized Context Distillation learns from interaction experience by regenerating only the critical spans of a student trajectory. For diffusion language models, one paper proposes predictability-aware supervised fine-tuning and another proposes HaloBlock for faster, better decoding. A further paper targets KV cache memory and bandwidth bottlenecks in long-output inference with mixed-precision key-cache quantization.",
        "papers": [
          "Learning from Interaction Experience via Localized Context Distillation",
          "Hindsight-Privileged RLVR: Single-Rollout Token Credit Without Group Sampling",
          "Match the Canvas, Not Just the Answer: Predictability-Aware SFT for Diffusion Language Models",
          "HaloBlock: Faster and Better Decoding for Diffusion Large Language Models",
          "PN-Mix: Pair-Norm Guided Mixed-Precision Key Cache Quantization for Efficient Long-Output LLM Inference"
        ]
      }
    ],
    "whats_new": [
      {
        "title": "Verifier-grounded and token-level reinforcement learning",
        "summary": "One preprint builds an agentic environment where a program verifier supplies the reward. It scores candidates by the weighted fraction of proof obligations discharged and adds an integrity audit against cheating. Another, HP-RLVR, assigns bounded token credit to both successful and failed attempts from a single rollout.",
        "papers": [
          "A5",
          "A2"
        ]
      },
      {
        "title": "Measuring algorithmic diversity of generated code",
        "summary": "DivAlgo is a large-scale benchmark of algorithmically diverse solutions to programming problems, paired with a lightweight similarity metric. Across 24 modern code generation models, uncontrolled output captured 31% of human diversity, and the best diverse generation strategy reached 70%.",
        "papers": [
          "A6"
        ]
      },
      {
        "title": "Learning from experience via localized distillation",
        "summary": "Localized Context Distillation locates the critical spans of a student trajectory that need revision, then has a teacher regenerate only those spans. It aims to avoid the exposure bias of off-policy distillation and the uninformative states of on-policy distillation.",
        "papers": [
          "A1"
        ]
      },
      {
        "title": "Diffusion language models: training and decoding",
        "summary": "Predictability-aware SFT addresses the mismatch between random masking in training and how diffusion models decode. HaloBlock uses bidirectional cumulative entropy to measure block decoding difficulty and adds adaptive halo regions to ease boundary discontinuity in dLLM inference.",
        "papers": [
          "A7",
          "A8"
        ]
      }
    ],
    "trend_note": "Monthly arXiv submissions matching this field ran from 123 in Oct 2025 to a peak of 136 in Apr 2026, with 133 in May 2026. They then eased to 72 in Jul 2026 and stood at 85 in Sep 2026.",
    "open_problems": [
      "Tests can pass while code stays wrong on uncovered inputs, motivating verifier-based rewards that are hard to game.",
      "Generated code is far less algorithmically diverse than human solutions, and steering does not restore that diversity.",
      "Reviews point to a significant research gap in addressing ethical constraints in LLM code generation.",
      "When every attempt fails or all receive the same outcome, reinforcement signals vanish, so credit assignment remains difficult."
    ],
    "faq": [
      {
        "q": "Which papers give an overview of LLMs for code generation?",
        "a": "Jiang and colleagues survey the field, Huynh and Lin offer a comprehensive survey of challenges, techniques, evaluation and applications, and Bistarelli and colleagues review 66 papers published between 2021 and 2023."
      },
      {
        "q": "Why do researchers question test-based evaluation?",
        "a": "EvalPlus notes that benchmark test cases can be limited in quantity and quality. A recent preprint adds that models can pass tests yet remain wrong on inputs the tests do not cover."
      },
      {
        "q": "What are the newest directions?",
        "a": "Recent preprints cover verifier-grounded reinforcement learning, single-rollout token credit, localized context distillation, algorithmic diversity benchmarking, and training and decoding for diffusion language models."
      }
    ],
    "total_citations": 7015
  },
  "analysis_md": "Foundational work covers surveys, planning, fine-tuning and test-based evaluation. Recent preprints move toward verifier-grounded reinforcement learning, algorithmic diversity benchmarks, and training and decoding for diffusion language models.\n\n## Surveys map the landscape and its gaps\n\nSeveral review papers organize the field. Jiang and colleagues survey LLMs for code generation, and Huynh and Lin's comprehensive survey covers challenges, techniques, evaluation and applications. Bistarelli and colleagues apply the PRISMA methodology to 66 papers published between 2021 and 2023. They report Python as the preferred language and a significant research gap in addressing ethical constraints. They also compare models such as GPT-4 and CodeLlama on tasks from debugging to multi-turn program synthesis. Together these reviews give the baseline against which the newer work on evaluation and training can be read.\n\nPapers: [A survey on large language models for code generation](https://dl.acm.org/doi/abs/10.1145/3747588); [Large language models for code generation: A comprehensive survey of challenges, techniques, evaluation, and applications](https://arxiv.org/abs/2503.01245); [Usage of large language model for code generation tasks: A review](https://link.springer.com/article/10.1007/s42979-025-04241-5)\n\n## Planning, prompting and adaptation of code models\n\nA second line of work improves how models produce code without changing the task. Self-planning approaches build planning into code generation to help the model understand the problem, and a separate paper also studies planning with LLMs for code generation. Li and colleagues propose a large language model-aware selection approach for in-context learning. Weyssow and colleagues compare parameter-efficient fine-tuning across a large spectrum of small and large models on a Python code generation task. Gu and colleagues focus on domain-specific code. CodeT5+ presents open code LLMs for both understanding and generation.\n\nPapers: [Planning with large language models for code generation](https://arxiv.org/abs/2303.05510); [Self-planning code generation with large language models](https://dl.acm.org/doi/abs/10.1145/3672456); [Large language model-aware in-context learning for code generation](https://dl.acm.org/doi/abs/10.1145/3715908); [Exploring parameter-efficient fine-tuning techniques for code generation with large language models](https://dl.acm.org/doi/abs/10.1145/3714461); [On the effectiveness of large language models in domain-specific code generation](https://dl.acm.org/doi/abs/10.1145/3697012); [Codet5+: Open code large language models for code understanding and generation](https://aclanthology.org/2023.emnlp-main.68/)\n\n## Evaluation is moving beyond passing the given tests\n\nEvalPlus argued that benchmark test cases can be limited in quantity and quality for assessing functional correctness. It augments datasets with test cases from an automatic test input generator that uses both LLM-based and mutation-based strategies. Recent preprints push the same concern further. One argues that rewarding passing tests lets a model stay wrong on inputs the tests do not cover, and turns to a program verifier that checks a formal specification for all inputs. Another introduces the DivAlgo benchmark to measure whether generated solutions are algorithmically diverse.\n\nPapers: [Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation](https://proceedings.neurips.cc/paper_files/paper/2023/hash/43e9d647ccd3e4b7b5baab53f0368686-Abstract-Conference.html); [Verifier-Grounded Agentic RL for Provably Correct Code](https://arxiv.org/abs/2610.19298v1); [DivAlgo: A systematic evaluation of algorithmic diversity for code generation models](https://arxiv.org/abs/2610.19157v1)\n\n## Learning signals and decoding for newer model families\n\nRecent preprints concentrate on how models learn from outcomes and how they decode. Hindsight-Privileged RLVR assigns token-level credit from a single rollout, avoiding group sampling. Localized Context Distillation learns from interaction experience by regenerating only the critical spans of a student trajectory. For diffusion language models, one paper proposes predictability-aware supervised fine-tuning and another proposes HaloBlock for faster, better decoding. A further paper targets KV cache memory and bandwidth bottlenecks in long-output inference with mixed-precision key-cache quantization.\n\nPapers: [Learning from Interaction Experience via Localized Context Distillation](https://arxiv.org/abs/2610.21427v1); [Hindsight-Privileged RLVR: Single-Rollout Token Credit Without Group Sampling](https://arxiv.org/abs/2610.20497v2); [Match the Canvas, Not Just the Answer: Predictability-Aware SFT for Diffusion Language Models](https://arxiv.org/abs/2610.18001v1); [HaloBlock: Faster and Better Decoding for Diffusion Large Language Models](https://arxiv.org/abs/2610.17948v1); [PN-Mix: Pair-Norm Guided Mixed-Precision Key Cache Quantization for Efficient Long-Output LLM Inference](https://arxiv.org/abs/2610.19605v1)\n\n## What's new on arXiv\n\nMonthly arXiv submissions matching this field ran from 123 in Oct 2025 to a peak of 136 in Apr 2026, with 133 in May 2026. They then eased to 72 in Jul 2026 and stood at 85 in Sep 2026.\n\n**Verifier-grounded and token-level reinforcement learning** — One preprint builds an agentic environment where a program verifier supplies the reward. It scores candidates by the weighted fraction of proof obligations discharged and adds an integrity audit against cheating. Another, HP-RLVR, assigns bounded token credit to both successful and failed attempts from a single rollout.\n\n**Measuring algorithmic diversity of generated code** — DivAlgo is a large-scale benchmark of algorithmically diverse solutions to programming problems, paired with a lightweight similarity metric. Across 24 modern code generation models, uncontrolled output captured 31% of human diversity, and the best diverse generation strategy reached 70%.\n\n**Learning from experience via localized distillation** — Localized Context Distillation locates the critical spans of a student trajectory that need revision, then has a teacher regenerate only those spans. It aims to avoid the exposure bias of off-policy distillation and the uninformative states of on-policy distillation.\n\n**Diffusion language models: training and decoding** — Predictability-aware SFT addresses the mismatch between random masking in training and how diffusion models decode. HaloBlock uses bidirectional cumulative entropy to measure block decoding difficulty and adds adaptive halo regions to ease boundary discontinuity in dLLM inference.\n\n- 2026-10-15 · [Learning from Interaction Experience via Localized Context Distillation](https://arxiv.org/abs/2610.21427v1) (cs.AI)\n- 2026-10-14 · [Hindsight-Privileged RLVR: Single-Rollout Token Credit Without Group Sampling](https://arxiv.org/abs/2610.20497v2) (cs.LG)\n- 2026-10-14 · [PN-Mix: Pair-Norm Guided Mixed-Precision Key Cache Quantization for Efficient Long-Output LLM Inference](https://arxiv.org/abs/2610.19605v1) (cs.LG)\n- 2026-10-13 · [Beyond ICD Codes: Semantic IDs for Generative Medical Coding](https://arxiv.org/abs/2610.19301v1) (cs.AI)\n- 2026-10-13 · [Verifier-Grounded Agentic RL for Provably Correct Code](https://arxiv.org/abs/2610.19298v1) (cs.LG)\n- 2026-10-13 · [DivAlgo: A systematic evaluation of algorithmic diversity for code generation models](https://arxiv.org/abs/2610.19157v1) (cs.SE)\n- 2026-10-13 · [Match the Canvas, Not Just the Answer: Predictability-Aware SFT for Diffusion Language Models](https://arxiv.org/abs/2610.18001v1) (cs.LG)\n- 2026-10-13 · [HaloBlock: Faster and Better Decoding for Diffusion Large Language Models](https://arxiv.org/abs/2610.17948v1) (cs.LG)\n\n## Foundational papers\n\n1. [A survey on large language models for code generation](https://dl.acm.org/doi/abs/10.1145/3747588) — J Jiang, F Wang, J Shen, S Kim, S Kim; 1589 citations. Surveys LLMs for code generation, describing them as transformative tools for automated code generation and mapping the landscape of models.\n2. [Planning with large language models for code generation](https://arxiv.org/abs/2303.05510) — S Zhang, Z Chen, Y Shen, M Ding et al., 2023; 361 citations\n3. [Self-planning code generation with large language models](https://dl.acm.org/doi/abs/10.1145/3672456) — X Jiang, Y Dong, L Wang, Z Fang, Q Shang et al.; 556 citations. Proposes a self-planning approach that builds planning into code generation to help the model understand the problem.\n4. [Large language models for code generation: A comprehensive survey of challenges, techniques, evaluation, and applications](https://arxiv.org/abs/2503.01245) — N Huynh, B Lin, 2025; 110 citations\n5. [Usage of large language model for code generation tasks: A review](https://link.springer.com/article/10.1007/s42979-025-04241-5) — S Bistarelli, M Fiore, I Mercanti, M Mongiello, 2021; 44 citations. A PRISMA literature review of 66 papers from 2021 to 2023, finding Python dominant and a significant research gap on ethical constraints.\n6. [Codet5+: Open code large language models for code understanding and generation](https://aclanthology.org/2023.emnlp-main.68/) — Y Wang, H Le, A Gotmare, N Bui, J Li et al., 2023; 1166 citations\n7. [Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation](https://proceedings.neurips.cc/paper_files/paper/2023/hash/43e9d647ccd3e4b7b5baab53f0368686-Abstract-Conference.html) — J Liu, CS Xia, Y Wang, L Zhang, 2023; 2721 citations. Introduces EvalPlus, which augments benchmarks with many automatically generated test cases to rigorously assess the functional correctness of LLM-synthesized code.\n8. [On the effectiveness of large language models in domain-specific code generation](https://dl.acm.org/doi/abs/10.1145/3697012) — X Gu, M Chen, Y Lin, Y Hu, H Zhang, C Wan et al.; 179 citations. Examines LLM proficiency in generating domain-specific code, a aspect that existing benchmarks do not emphasize.\n9. [Exploring parameter-efficient fine-tuning techniques for code generation with large language models](https://dl.acm.org/doi/abs/10.1145/3714461) — M Weyssow, X Zhou, K Kim, D Lo et al.; 180 citations. Studies parameter-efficient fine-tuning across a large spectrum of small and large models on a Python code generation task.\n10. [Large language model-aware in-context learning for code generation](https://dl.acm.org/doi/abs/10.1145/3715908) — J Li, C Tao, J Li♂, G Li, Z Jin, H Zhang et al.; 109 citations. Proposes a large language model-aware selection approach for in-context-learning-based code generation.\n\n## Open problems\n\n- Tests can pass while code stays wrong on uncovered inputs, motivating verifier-based rewards that are hard to game.\n- Generated code is far less algorithmically diverse than human solutions, and steering does not restore that diversity.\n- Reviews point to a significant research gap in addressing ethical constraints in LLM code generation.\n- When every attempt fails or all receive the same outcome, reinforcement signals vanish, so credit assignment remains difficult.\n\n## FAQ\n\n**Which papers give an overview of LLMs for code generation?**\nJiang and colleagues survey the field, Huynh and Lin offer a comprehensive survey of challenges, techniques, evaluation and applications, and Bistarelli and colleagues review 66 papers published between 2021 and 2023.\n\n**Why do researchers question test-based evaluation?**\nEvalPlus notes that benchmark test cases can be limited in quantity and quality. A recent preprint adds that models can pass tests yet remain wrong on inputs the tests do not cover.\n\n**What are the newest directions?**\nRecent preprints cover verifier-grounded reinforcement learning, single-rollout token credit, localized context distillation, algorithmic diversity benchmarking, and training and decoding for diffusion language models.\n",
  "sources": [
    {
      "title": "A survey on large language models for code generation",
      "url": "https://dl.acm.org/doi/abs/10.1145/3747588"
    },
    {
      "title": "Planning with large language models for code generation",
      "url": "https://arxiv.org/abs/2303.05510"
    },
    {
      "title": "Self-planning code generation with large language models",
      "url": "https://dl.acm.org/doi/abs/10.1145/3672456"
    },
    {
      "title": "Large language models for code generation: A comprehensive survey of challenges, techniques, evaluation, and applications",
      "url": "https://arxiv.org/abs/2503.01245"
    },
    {
      "title": "Usage of large language model for code generation tasks: A review",
      "url": "https://link.springer.com/article/10.1007/s42979-025-04241-5"
    },
    {
      "title": "Codet5+: Open code large language models for code understanding and generation",
      "url": "https://aclanthology.org/2023.emnlp-main.68/"
    },
    {
      "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
      "url": "https://proceedings.neurips.cc/paper_files/paper/2023/hash/43e9d647ccd3e4b7b5baab53f0368686-Abstract-Conference.html"
    },
    {
      "title": "On the effectiveness of large language models in domain-specific code generation",
      "url": "https://dl.acm.org/doi/abs/10.1145/3697012"
    },
    {
      "title": "Exploring parameter-efficient fine-tuning techniques for code generation with large language models",
      "url": "https://dl.acm.org/doi/abs/10.1145/3714461"
    },
    {
      "title": "Large language model-aware in-context learning for code generation",
      "url": "https://dl.acm.org/doi/abs/10.1145/3715908"
    },
    {
      "title": "Learning from Interaction Experience via Localized Context Distillation",
      "url": "https://arxiv.org/abs/2610.21427v1"
    },
    {
      "title": "Hindsight-Privileged RLVR: Single-Rollout Token Credit Without Group Sampling",
      "url": "https://arxiv.org/abs/2610.20497v2"
    },
    {
      "title": "PN-Mix: Pair-Norm Guided Mixed-Precision Key Cache Quantization for Efficient Long-Output LLM Inference",
      "url": "https://arxiv.org/abs/2610.19605v1"
    },
    {
      "title": "Beyond ICD Codes: Semantic IDs for Generative Medical Coding",
      "url": "https://arxiv.org/abs/2610.19301v1"
    },
    {
      "title": "Verifier-Grounded Agentic RL for Provably Correct Code",
      "url": "https://arxiv.org/abs/2610.19298v1"
    },
    {
      "title": "DivAlgo: A systematic evaluation of algorithmic diversity for code generation models",
      "url": "https://arxiv.org/abs/2610.19157v1"
    },
    {
      "title": "Match the Canvas, Not Just the Answer: Predictability-Aware SFT for Diffusion Language Models",
      "url": "https://arxiv.org/abs/2610.18001v1"
    },
    {
      "title": "HaloBlock: Faster and Better Decoding for Diffusion Large Language Models",
      "url": "https://arxiv.org/abs/2610.17948v1"
    }
  ],
  "related": [
    {
      "name": "LLM Agents & Planning: Literature Digest",
      "url": "https://agentflare.org/scholar/llm-agents-planning-literature-digest.html"
    },
    {
      "name": "Retrieval-Augmented Generation: Research Digest",
      "url": "https://agentflare.org/scholar/retrieval-augmented-generation-research-digest.html"
    },
    {
      "name": "AI Alignment & Safety: Research Digest",
      "url": "https://agentflare.org/scholar/ai-alignment-safety-research-digest.html"
    },
    {
      "name": "RLHF: Research Digest",
      "url": "https://agentflare.org/scholar/rlhf-research-digest.html"
    },
    {
      "name": "Multimodal Foundation Models: Research Digest",
      "url": "https://agentflare.org/scholar/multimodal-foundation-models-research-digest.html"
    },
    {
      "name": "Mechanistic Interpretability: Research Digest",
      "url": "https://agentflare.org/scholar/mechanistic-interpretability-research-digest.html"
    }
  ],
  "pricing": {
    "price_usd": 0.02,
    "method": "402",
    "endpoint": "https://esa.aisa.one/api/v1/access/verify",
    "autopay_hint": "set crawlerAutoPrice=true with the X-Crawler-Token header",
    "onboarding": "https://esa.aisa.one/agent/guide.html"
  },
  "powered_by": "AISA — agent-native search, settlement & delivery (https://aisa.one)"
}