publications | Dr. Liu Mingwei (刘名威)

2026

RealSec-bench: A Benchmark for Evaluating Secure Code Generation in Real-World Repositories

Yanlin Wang, Ziyao Zhang, Chong Wang, and 5 more authors

arXiv preprint arXiv:2601.22706, 2026

@article{wang2026realsec,
  title = {RealSec-bench: A Benchmark for Evaluating Secure Code Generation in Real-World Repositories},
  author = {Wang, Yanlin and Zhang, Ziyao and Wang, Chong and Xu, Xinyi and Liu, Mingwei and Wang, Yong and Chen, Jiachi and Zheng, Zibin},
  journal = {arXiv preprint arXiv:2601.22706},
  year = {2026},
}

TSE

Are decoder-only large language models the silver bullet for code search?

Yuxuan Chen, Mingwei Liu, Guangsheng Ou, and 4 more authors

IEEE Transactions on Software Engineering, 2026

Bib

@article{chen2026decoder,
  title = {Are decoder-only large language models the silver bullet for code search?},
  author = {Chen, Yuxuan and Liu, Mingwei and Ou, Guangsheng and Li, Anji and Dai, Dekun and Wang, Yanlin and Zheng, Zibin},
  journal = {IEEE Transactions on Software Engineering},
  year = {2026},
  publisher = {IEEE},
}

arXiv
Advances and Frontiers of LLM-based Issue Resolution in Software Engineering: A Comprehensive Survey

Caihua Li, Lianghong Guo, Yanlin Wang, and 8 more authors

arXiv preprint arXiv:2601.11655, 2026

Abs Bib

Issue resolution, a complex Software Engineering (SWE) task integral to real-world development, has emerged as a compelling challenge for artificial intelligence. The establishment of benchmarks like SWE-bench revealed this task as profoundly difficult for large language models, thereby significantly accelerating the evolution of autonomous coding agents. This paper presents a systematic survey of this emerging domain. We begin by examining data construction pipelines, covering automated collection and synthesis approaches. We then provide a comprehensive analysis of methodologies, spanning training-free frameworks with their modular components to training-based techniques, including supervised fine-tuning and reinforcement learning. Subsequently, we discuss critical analyses of data quality and agent behavior, alongside practical applications. Finally, we identify key challenges and outline promising directions for future research.
@article{li2026advances, title = {Advances and Frontiers of LLM-based Issue Resolution in Software Engineering: A Comprehensive Survey}, author = {Li, Caihua and Guo, Lianghong and Wang, Yanlin and Guo, Daya and Tao, Wei and Shan, Zhenyu and Liu, Mingwei and Chen, Jiachi and Song, Haoyu and Tang, Duyu and others}, journal = {arXiv preprint arXiv:2601.11655}, year = {2026}, }

arXiv

ShortCoder: Knowledge-Augmented Syntax Optimization for Token-Efficient Code Generation

Sicong Liu, Yanxian Huang, Mingwei Liu, and 6 more authors

arXiv preprint arXiv:2601.09703, 2026

Bib

@article{liu2026shortcoder,
  title = {ShortCoder: Knowledge-Augmented Syntax Optimization for Token-Efficient Code Generation},
  author = {Liu, Sicong and Huang, Yanxian and Liu, Mingwei and Chen, Jiachi and Shi, Ensheng and Ma, Yuchi and Zhang, Hongyu and Zhang, Yin and Wang, Yanlin},
  journal = {arXiv preprint arXiv:2601.09703},
  year = {2026},
}

2025

ASE 2025

DrainCode: Stealthy Energy Consumption Attacks on Retrieval-Augmented Code Generation via Context Poisoning

Yanlin Wang, Jiadong Wu, Tianyue Jiang, and 7 more authors

In 2025 40th IEEE/ACM International Conference on Automated Software Engineering (ASE), 2025

Bib

@inproceedings{wang2025draincode,
  title = {DrainCode: Stealthy Energy Consumption Attacks on Retrieval-Augmented Code Generation via Context Poisoning},
  author = {Wang, Yanlin and Wu, Jiadong and Jiang, Tianyue and Liu, Mingwei and Chen, Jiachi and Wang, Chong and Shi, Ensheng and Liu, Xilin and Ma, Yuchi and Zheng, Zibin},
  booktitle = {2025 40th IEEE/ACM International Conference on Automated Software Engineering (ASE)},
  pages = {778--790},
  year = {2025},
  organization = {IEEE},
}

arXiv

SimpleDevQA: Benchmarking Large Language Models on Development Knowledge QA

Jing Zhang, Lianghong Guo, Yanlin Wang, and 7 more authors

arXiv preprint arXiv:2512.08867, 2025

Bib

@article{zhang2025simpledevqa,
  title = {SimpleDevQA: Benchmarking Large Language Models on Development Knowledge QA},
  author = {Zhang, Jing and Guo, Lianghong and Wang, Yanlin and Liu, Mingwei and Chen, Jiachi and Ma, Yuchi and Shi, Ensheng and Zhuo, Terry Yue and Zhang, Hongyu and Zheng, Zibin},
  journal = {arXiv preprint arXiv:2512.08867},
  year = {2025},
}

arXiv

Framework-Aware Code Generation with API Knowledge Graph-Constructed Data: A Study on HarmonyOS

Mingwei Liu, Zheng Pei, Yanlin Wang, and 5 more authors

arXiv preprint arXiv:2512.00380, 2025

Bib

@article{liu2025framework,
  title = {Framework-Aware Code Generation with API Knowledge Graph-Constructed Data: A Study on HarmonyOS},
  author = {Liu, Mingwei and Pei, Zheng and Wang, Yanlin and Wang, Zihao and Li, Zikang and Lin, Enci and Peng, Xin and Zheng, Zibin},
  journal = {arXiv preprint arXiv:2512.00380},
  year = {2025},
}

ICSE 2026

Knowledge Matters: Injecting Project and Testing Knowledge into LLM-based Unit Test Generation

Anji Li, Mingwei Liu, Zhenxi Chen, and 5 more authors

arXiv preprint arXiv:2511.14224, 2025

Bib

@article{li2025ktester,
  title = {Knowledge Matters: Injecting Project and Testing Knowledge into LLM-based Unit Test Generation},
  author = {Li, Anji and Liu, Mingwei and Chen, Zhenxi and Pei, Zheng and Li, Zike and Dai, Dekun and Wang, Yanlin and Zheng, Zibin},
  journal = {arXiv preprint arXiv:2511.14224},
  year = {2025},
}

arXiv

Uncovering Pretraining Code in LLMs: A Syntax-Aware Attribution Approach

Yuanheng Li, Zhuoyang Chen, Xiaoyun Liu, and 5 more authors

arXiv preprint arXiv:2511.07033, 2025

Bib

@article{li2025uncovering,
  title = {Uncovering Pretraining Code in LLMs: A Syntax-Aware Attribution Approach},
  author = {Li, Yuanheng and Chen, Zhuoyang and Liu, Xiaoyun and Wang, Yuhao and Liu, Mingwei and Shi, Yang and Huang, Kaifeng and Zhao, Shengjie},
  journal = {arXiv preprint arXiv:2511.07033},
  year = {2025},
}

TSE

Cosqa+: Enhancing code search evaluation with a multi-choice benchmark and test-driven agents

Jing Gong, Yanghui Wu, Linxi Liang, and 4 more authors

IEEE Transactions on Software Engineering, 2025

Bib

@article{gong2025cosqa+,
  title = {Cosqa+: Enhancing code search evaluation with a multi-choice benchmark and test-driven agents},
  author = {Gong, Jing and Wu, Yanghui and Liang, Linxi and Wang, Yanlin and Chen, Jiachi and Liu, Mingwei and Zheng, Zibin},
  journal = {IEEE Transactions on Software Engineering},
  volume = {52},
  number = {1},
  pages = {206--220},
  year = {2025},
  publisher = {IEEE},
}

arXiv

EffiReasonTrans: RL-Optimized Reasoning for Code Translation

Yanlin Wang, Rongyi Ou, Yanli Wang, and 6 more authors

arXiv preprint arXiv:2510.18863, 2025

Bib

@article{wang2025effireasontrans,
  title = {EffiReasonTrans: RL-Optimized Reasoning for Code Translation},
  author = {Wang, Yanlin and Ou, Rongyi and Wang, Yanli and Liu, Mingwei and Chen, Jiachi and Shi, Ensheng and Liu, Xilin and Ma, Yuchi and Zheng, Zibin},
  journal = {arXiv preprint arXiv:2510.18863},
  year = {2025},
}

arXiv

Generating High-Quality Datasets for Code Editing via Open-Source Language Models

Zekai Zhang, Mingwei Liu, Zhenxi Chen, and 7 more authors

arXiv preprint arXiv:2509.25203, 2025

Bib

@article{zhang2025generating,
  title = {Generating High-Quality Datasets for Code Editing via Open-Source Language Models},
  author = {Zhang, Zekai and Liu, Mingwei and Chen, Zhenxi and Liang, Linxi and Chen, Yuxuan and Ou, Guangsheng and Wang, Yanlin and Li, Dan and Peng, Xin and Zheng, Zibin},
  journal = {arXiv preprint arXiv:2509.25203},
  year = {2025},
}

arXiv

EvolMathEval: Towards Evolvable Benchmarks for Mathematical Reasoning via Evolutionary Testing

Shengbo Wang, Mingwei Liu, Zike Li, and 4 more authors

arXiv preprint arXiv:2508.13003, 2025

Bib

@article{wang2025evolmatheval,
  title = {EvolMathEval: Towards Evolvable Benchmarks for Mathematical Reasoning via Evolutionary Testing},
  author = {Wang, Shengbo and Liu, Mingwei and Li, Zike and Li, Anji and Wang, Yanlin and Peng, Xin and Zheng, Zibin},
  journal = {arXiv preprint arXiv:2508.13003},
  year = {2025},
}

arXiv

A hierarchical and evolvable benchmark for fine-grained code instruction following with multi-turn feedback

Guoliang Duan, Mingwei Liu, Yanlin Wang, and 3 more authors

arXiv preprint arXiv:2507.00699, 2025

Bib

@article{duan2025hierarchical,
  title = {A hierarchical and evolvable benchmark for fine-grained code instruction following with multi-turn feedback},
  author = {Duan, Guoliang and Liu, Mingwei and Wang, Yanlin and Wang, Chong and Peng, Xin and Zheng, Zibin},
  journal = {arXiv preprint arXiv:2507.00699},
  year = {2025},
}

FSE 2025

Beyond functional correctness: Investigating coding style inconsistencies in large language models

Yanlin Wang, Tianyue Jiang, Mingwei Liu, and 5 more authors

Proceedings of the ACM on Software Engineering, 2025

Bib

@article{wang2025beyond,
  title = {Beyond functional correctness: Investigating coding style inconsistencies in large language models},
  author = {Wang, Yanlin and Jiang, Tianyue and Liu, Mingwei and Chen, Jiachi and Mao, Mingzhi and Liu, Xilin and Ma, Yuchi and Zheng, Zibin},
  journal = {Proceedings of the ACM on Software Engineering},
  volume = {2},
  number = {FSE},
  pages = {690--712},
  year = {2025},
  publisher = {ACM New York, NY, USA},
}

arXiv

Adadec: Uncertainty-guided adaptive decoding for llm-based code generation

Kaifeng He, Mingwei Liu, Chong Wang, and 4 more authors

arXiv e-prints, 2025

Bib

@article{he2025adadec,
  title = {Adadec: Uncertainty-guided adaptive decoding for llm-based code generation},
  author = {He, Kaifeng and Liu, Mingwei and Wang, Chong and Li, Zike and Wang, Yanlin and Peng, Xin and Zheng, Zibin},
  journal = {arXiv e-prints},
  pages = {arXiv--2506},
  year = {2025},
}

arXiv

Code Copycat Conundrum: Demystifying Repetition in LLM-based Code Generation

Mingwei Liu, Juntao Li, Ying Wang, and 8 more authors

arXiv preprint arXiv:2504.12608, 2025

Bib

@article{liu2025code,
  title = {Code Copycat Conundrum: Demystifying Repetition in LLM-based Code Generation},
  author = {Liu, Mingwei and Li, Juntao and Wang, Ying and Du, Xueying and Ou, Zuoyu and Chen, Qiuyuan and An, Bingxu and Wei, Zhao and Xu, Yong and Zou, Fangming and others},
  journal = {arXiv preprint arXiv:2504.12608},
  year = {2025},
}

arXiv

Feedbackeval: A benchmark for evaluating large language models in feedback-driven code repair tasks

Dekun Dai, MingWei Liu, Anji Li, and 5 more authors

arXiv preprint arXiv:2504.06939, 2025

Bib

@article{dai2025feedbackeval,
  title = {Feedbackeval: A benchmark for evaluating large language models in feedback-driven code repair tasks},
  author = {Dai, Dekun and Liu, MingWei and Li, Anji and Cao, Jialun and Wang, Yanlin and Wang, Chong and Peng, Xin and Zheng, Zibin},
  journal = {arXiv preprint arXiv:2504.06939},
  year = {2025},
}

arXiv

Enhancing the robustness of llm-generated code: Empirical study and framework

Zike Li, Mingwei Liu, Anji Li, and 4 more authors

arXiv e-prints, 2025

Bib

@article{li2025enhancing,
  title = {Enhancing the robustness of llm-generated code: Empirical study and framework},
  author = {Li, Zike and Liu, Mingwei and Li, Anji and He, Kaifeng and Wang, Yanlin and Peng, Xin and Zheng, Zibin},
  journal = {arXiv e-prints},
  pages = {arXiv--2503},
  year = {2025},
}

arXiv

What to retrieve for effective retrieval-augmented code generation? an empirical study and beyond

Wenchao Gu, Juntao Chen, Yanlin Wang, and 6 more authors

arXiv preprint arXiv:2503.20589, 2025

Bib

@article{gu2025retrieve,
  title = {What to retrieve for effective retrieval-augmented code generation? an empirical study and beyond},
  author = {Gu, Wenchao and Chen, Juntao and Wang, Yanlin and Jiang, Tianyue and Li, Xingzhe and Liu, Mingwei and Liu, Xilin and Ma, Yuchi and Zheng, Zibin},
  journal = {arXiv preprint arXiv:2503.20589},
  year = {2025},
}

arXiv

Enhancing llm-based code translation in repository context via triple knowledge-augmented

Guangsheng Ou, Mingwei Liu, Yuxuan Chen, and 5 more authors

arXiv e-prints, 2025

Bib

@article{ou2025enhancing,
  title = {Enhancing llm-based code translation in repository context via triple knowledge-augmented},
  author = {Ou, Guangsheng and Liu, Mingwei and Chen, Yuxuan and Du, Xueying and Wang, Shengbo and Zhang, Zekai and Peng, Xin and Zheng, Zibin},
  journal = {arXiv e-prints},
  pages = {arXiv--2503},
  year = {2025},
}

arXiv

RustEvo\^ 2: An Evolving Benchmark for API Evolution in LLM-based Rust Code Generation

Linxi Liang, Jing Gong, Mingwei Liu, and 5 more authors

arXiv preprint arXiv:2503.16922, 2025

Bib

@article{liang2025rustevo,
  title = {RustEvo\^{} 2: An Evolving Benchmark for API Evolution in LLM-based Rust Code Generation},
  author = {Liang, Linxi and Gong, Jing and Liu, Mingwei and Wang, Chong and Ou, Guangsheng and Wang, Yanlin and Peng, Xin and Zheng, Zibin},
  journal = {arXiv preprint arXiv:2503.16922},
  year = {2025},
}

arXiv

How should we build a benchmark? revisiting 274 code-related benchmarks for llms

Jialun Cao, Yuk-Kit Chan, Zixuan Ling, and 8 more authors

arXiv preprint arXiv:2501.10711, 2025

Bib

@article{cao2025should,
  title = {How should we build a benchmark? revisiting 274 code-related benchmarks for llms},
  author = {Cao, Jialun and Chan, Yuk-Kit and Ling, Zixuan and Wang, Wenxuan and Li, Shuqing and Liu, Mingwei and Qiao, Ruixi and Han, Yuting and Wang, Chaozheng and Yu, Boxi and others},
  journal = {arXiv preprint arXiv:2501.10711},
  year = {2025},
}

ASE 2025
RustRepoTrans: Repository-level Context Code Translation Benchmark Targeting Rust

Guangsheng Ou, Mingwei Liu, Yuxuan Chen, and 3 more authors

In 2025 40th IEEE/ACM International Conference on Automated Software Engineering (ASE), 2025

Abs Bib PDF Code

Recent advancements in large language models (LLMs) have demonstrated impressive capabilities in code translation, typically evaluated using benchmarks like CodeTransOcean and RepoTransBench. However, dependency-free benchmarks fail to capture real-world complexities by focusing primarily on simple function-level translations and overlooking repository-level context (e.g., dependencies). Full-repository translation benchmarks significantly exceed the current capabilities of existing models, resulting in performance bottlenecks that fail to provide actionable insights for guiding model development. Furthermore, existing benchmarks do not account for the scenario of incrementally translating new or modified modules from the source to the target language, which demands careful handling of repositorylevel contexts such as dependencies, cross-module references, and architectural divergence. Moreover, LLMs’ effectiveness in translating to newer, low-resource languages like Rust remains largely underexplored. To address these gaps, we introduce RustRepoTrans, the first repository-level context code translation benchmark targeting incremental translation, comprising 375 tasks translating into Rust from C, Java, and Python. Using this benchmark, we evaluate seven representative LLMs, analyzing their errors to assess limitations in complex translation scenarios. Among them, DeepSeek-R1 performs best with 51.5% Pass@1, excelling in both basic functionality and additional translation abilities, such as noise robustness and syntactical difference identification. However, even DeepSeek-R1 experiences a 22.2% performance drop (Pass@1 from 73.7% to 51.5%) when handling repositorylevel context compared to previous benchmarks without such context. Meanwhile, we propose a set of more fine-grained evaluation metrics and an enhanced evaluation framework, enabling a more comprehensive analysis of LLMs’ performance in repository-level context code translation tasks to provide finegrained insights that can effectively inform the development of code translation techniques.
@inproceedings{ou2025rustrepotrans, title = {RustRepoTrans: Repository-level Context Code Translation Benchmark Targeting Rust}, author = {Ou, Guangsheng and Liu, Mingwei and Chen, Yuxuan and Wang, Yanlin and Peng, Xin and Zheng, Zibin}, booktitle = {2025 40th IEEE/ACM International Conference on Automated Software Engineering (ASE)}, pages = {610--622}, year = {2025}, organization = {IEEE}, }

2024

ICSE 2024
Evaluating Large Language Models in Class-Level Code Generation

Xueying Du, Mingwei Liu, Kaixin Wang, and 7 more authors

In Proceedings of the 46th IEEE/ACM International Conference on Software Engineering, ICSE 2024, Lisbon , Portugal, April 14 - 20, 2024, 2024

Abs DOI arXiv Bib PDF Code

Recently, many large language models (LLMs) have been proposed, showing advanced proficiency in code generation. Meanwhile, many efforts have been dedicated to evaluating LLMs on code generation benchmarks such as HumanEval. Although being very helpful for comparing different LLMs, existing evaluation focuses on a simple code generation scenario (i.e., function-level or statement-level code generation), which mainly asks LLMs to generate one single code unit (e.g., a function or a statement) for the given natural language description. Such evaluation focuses on generating independent and often small-scale code units, thus leaving it unclear how LLMs perform in real-world software development scenarios. To fill this knowledge gap, we make the first attempt to evaluate LLMs in a more challenging code generation scenario, i.e., classlevel code generation. Compared with existing code generation benchmarks, it better reflects real-world software development scenarios due to it comprising broader contextual dependencies and multiple, interdependent units of code. We first manually construct the first class-level code generation benchmark ClassEval of 100 class-level Python code generation tasks with approximately 500 person-hours. Based on the new benchmark ClassEval, we then perform the first study of 11 state-of-the-art LLMs on class-level code generation. Based on our results, we find that all LLMs perform much worse on class-level code generation compared to the methodlevel. While GPT models still dominate other LLMs on class-level code generation, the performance rankings of other models on method-level code generation no longer holds for class-level code generation. Besides, most models (except GPT models) perform better when generating the class method by method; and they have the limited ability of generating dependent code. Based on our findings, we call for software engineering (SE) researchers’ expertise to build more LLM benchmarks based on practical and complicated software development scenarios
@inproceedings{DBLP:journals/corr/abs-2308-01861, author = {Du, Xueying and Liu, Mingwei and Wang, Kaixin and Wang, Hanlin and Liu, Junwei and Chen, Yixuan and Feng, Jiayi and Sha, Chaofeng and Peng, Xin and Lou, Yiling}, title = {Evaluating Large Language Models in Class-Level Code Generation}, booktitle = {Proceedings of the 46th {IEEE/ACM} International Conference on Software Engineering, {ICSE} 2024, Lisbon , Portugal, April 14 - 20, 2024}, pages = {1496--1508}, publisher = {{ACM}}, year = {2024}, url = {https://doi.org/10.1145/3597503.3639219}, doi = {10.1145/3597503.3639219}, timestamp = {Mon, 04 Dec 2023 11:02:39 +0100}, biburl = {https://dblp.org/rec/conf/sigsoft/0002YL0ZDY23.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }
ESEC/FSE 2024
Evaluating and Improving ChatGPT for Unit Test Generation

Zhiqiang Yuan, Mingwei Liu, Shiji Ding, and 4 more authors

In Proceedings of the 32st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC/FSE 2024, to appear, July 2024, Brazil, Brazil, 2024

Abs arXiv Bib PDF

Unit testing plays an essential role in detecting bugs in functionally-discrete program units (e.g., methods). Manually writing high-quality unit tests is time-consuming and laborious. Although the traditional techniques are able to generate tests with reasonable coverage, they are shown to exhibit low readability and still cannot be directly adopted by developers in practice. Recent work has shown the large potential of large language models (LLMs) in unit test generation. By being pre-trained on a massive developer-written code corpus, the models are capable of generating more human-like and meaningful test code. In this work, we perform the first empirical study to evaluate the capability of ChatGPT (i.e., one of the most representative LLMs with outstanding performance in code generation and comprehension) in unit test generation. In particular, we conduct both a quantitative analysis and a user study to systematically investigate the quality of its generated tests in terms of correctness, sufficiency, readability, and usability. We find that the tests generated by ChatGPT still suffer from correctness issues, including diverse compilation errors and execution failures (mostly caused by incorrect assertions); but the passing tests generated by ChatGPT almost resemble manually-written tests by achieving comparable coverage, readability, and even sometimes developers’ preference. Our findings indicate that generating unit tests with ChatGPT could be very promising if the correctness of its generated tests could be further improved. Inspired by our findings above, we further propose ChatTester, a novel ChatGPT-based unit test generation approach, which leverages ChatGPT itself to improve the quality of its generated tests. ChatTester incorporates an initial test generator and an iterative test refiner. Our evaluation demonstrates the effectiveness of ChatTester by generating 34.3% more compilable tests and 18.7% more tests with correct assertions than the default ChatGPT. In addition to ChatGPT, we further investigate the generalization capabilities of ChatTester by applying it to two recent open-source LLMs (i.e., CodeLlama-Instruct and CodeFuse) and our results show that ChatTester can also improve the quality of tests generated by these LLMs.
@inproceedings{FSE24CHATTESTER, author = {Yuan, Zhiqiang and Liu, Mingwei and Ding, Shiji and Wang, Kaixin and Chen, Yixuan and Peng, Xin and Lou, Yiling}, title = {Evaluating and Improving ChatGPT for Unit Test Generation}, booktitle = {Proceedings of the 32st {ACM} Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, {ESEC/FSE} 2024, to appear, July 2024, Brazil, Brazil}, pages = {0--0}, publisher = {{ACM}}, year = {2024}, }

JoS

Revisiting The Retrieval-Augmentation Strategy In Code Completion

Baihan Zou, Ying Wang, Xin Peng, and 5 more authors

Journal of Software, 2024

Bib PDF

@article{jos2024coderag,
  title = {Revisiting The Retrieval-Augmentation Strategy In Code Completion},
  author = {Zou, Baihan and Wang, Ying and Peng, Xin and Lou, Yiling and Liu, Lihua and Zhangxin, Dong and Lin, Fan and Liu, Mingwei},
  journal = {Journal of Software},
  volume = {0},
  number = {0},
  pages = {0--0},
  year = {2024},
}

arXiv
STALL+: Boosting LLM-based Repository-level Code Completion with Static Analysis

Junwei Liu, Yixuan Chen, Mingwei Liu, and 2 more authors

CoRR, 2024

Abs arXiv Bib PDF

Repository-level code completion is challenging as it involves complicated contexts from multiple files in the repository. To date, researchers have proposed two technical categories to enhance LLM-based repository-level code completion, i.e., retrieval-augmented generation (RAG) and static analysis integration. This work performs the first study on the static analysis integration in LLM-based repository-level code completion by investigating both the effectiveness and efficiency of static analysis integration strategies across different phases of code completion. We first implement a framework STALL+, which supports an extendable and customizable integration of multiple static analysis strategies into the complete pipeline of LLM-based repository-level code completion; and based on STALL+, we perform extensive experiments by including different code LLMs on the latest repository-level code completion benchmark CrossCodeEval. Our findings show that integrating file-level dependencies in prompting phase performs the best while the integration in post-processing phase performs the worse. Additionally, we observe different improvements from static analysis between dynamic languages and static languages, i.e., the best combination is prompting-phase with decoding-phase integration for Java while the best combination is prompting-phase with post-processing-phase integration for Python given the limitations of statically analyzing dynamic languages. Additionally, we find the complementarity between RAG and static analysis integration as well as their cost-effectiveness after combination.
@article{STALL, author = {Liu, Junwei and Chen, Yixuan and Liu, Mingwei and Peng, Xin and Lou, Yiling}, title = {STALL+: Boosting LLM-based Repository-level Code Completion with Static Analysis}, journal = {CoRR}, volume = {abs/2406.10018}, year = {2024}, eprinttype = {arXiv}, }
arXiv
Beyond Functional Correctness: Investigating Coding Style Inconsistencies in Large Language Models

Yanlin Wang, Tianyue Jiang, Mingwei Liu, and 2 more authors

arXiv preprint arXiv:2407.00456, 2024

Abs arXiv Bib PDF

Large language models (LLMs) have brought a paradigm shift to the field of code generation, offering the potential to enhance the software development process. However, previous research mainly focuses on the accuracy of code generation, while coding style differences between LLMs and human developers remain under-explored. In this paper, we empirically analyze the differences in coding style between the code generated by mainstream Code LLMs and the code written by human developers, and summarize coding style inconsistency taxonomy. Specifically, we first summarize the types of coding style inconsistencies by manually analyzing a large number of generation results. We then compare the code generated by Code LLMs with the code written by human programmers in terms of readability, conciseness, and robustness. The results reveal that LLMs and developers have different coding styles. Additionally, we study the possible causes of these inconsistencies and provide some solutions to alleviate the problem.
@article{wang2024beyond, title = {Beyond Functional Correctness: Investigating Coding Style Inconsistencies in Large Language Models}, author = {Wang, Yanlin and Jiang, Tianyue and Liu, Mingwei and Chen, Jiachi and Zheng, Zibin}, journal = {arXiv preprint arXiv:2407.00456}, year = {2024}, }

2023

TOSEM
XCoS: Explainable Code Search Based on Query Scoping and Knowledge Graph

Chong Wang, Xin Peng, Zhenchang Xing, and 4 more authors

ACM Trans. Softw. Eng. Methodol., 2023

Abs DOI Bib PDF

When searching code developers may express additional constraints (e.g., functional constraints and nonfunctional constraints) on the implementations of desired functionalities in the queries. Existing code search tools treat the queries as a whole and ignore the different implications of different parts of the queries. Moreover, these tools usually return a ranked list of candidate code snippets without any explanations. Therefore, the developers often find it hard to choose the desired results and build confidence on them. In this paper, we conduct a developer survey to better understand and address these issues and induct some insights from the survey results. Based on the insights, we propose XCoS, an explainable code search approach based on query scoping and knowledge graph. XCoS extracts a background knowledge graph from general knowledge bases like Wikidata and Wikipedia. Given a code search query, XCoS identifies different parts (i.e., functionalities, functional constraints, nonfunctional constraints) from it and use the expressions of functionalities and functional constraints to search the codebase. It then links both the query and the candidate code snippets to the concepts in the background knowledge graph and generates explanations based on the association paths between these two parts of concepts together with relevant descriptions. XCoS uses an interactive user interface that allows the user to better understand the associations between candidate code snippets and the query from different aspects and choose the desired results. Our evaluation shows that the quality of the extracted background knowledge and the concept linkings in codebase is generally high. Furthermore, the generated explanations are considered complete, concise, and readable and the approach can help developers find the desired code snippets more accurately and confidently.
@article{DBLP:journals/tosem/WangPXZLLM23, author = {Wang, Chong and Peng, Xin and Xing, Zhenchang and Zhang, Yue and Liu, Mingwei and Luo, Rong and Meng, Xiujie}, title = {XCoS: Explainable Code Search Based on Query Scoping and Knowledge Graph}, journal = {{ACM} Trans. Softw. Eng. Methodol.}, volume = {32}, number = {6}, pages = {140:1--140:28}, year = {2023}, url = {https://doi.org/10.1145/3593800}, doi = {10.1145/3593800}, timestamp = {Thu, 09 Nov 2023 00:00:00 +0100}, biburl = {https://dblp.org/rec/journals/tosem/WangPXZLLM23.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }
TSE
Task-Oriented ML/DL Library Recommendation Based on a Knowledge Graph

Mingwei Liu, Chengyuan Zhao, Xin Peng, and 3 more authors

IEEE Trans. Software Eng., 2023

Abs DOI Bib PDF

AI applications often use ML/DL (Machine Learning/Deep Learning) models to implement specific AI tasks. As application developers usually are not AI experts, they often choose to integrate existing implementations of ML/DL models as libraries for their AI tasks. As an active research area, AI attracts many researchers and produces a lot of papers every year. Many of the papers propose ML/DL models for specific tasks and provide their implementations. However, it is not easy for developers to find ML/DL libraries that are suitable for their tasks. The challenges lie in not only the fast development of AI application domains and techniques, but also the lack of detailed information of the libraries such as environmental dependencies and supporting resources. In this paper, we conduct an empirical study on ML/DL library seeking questions on Stack Overflow to understand the developers’ requirements for ML/DL libraries. Based on the findings of the study, we propose a task-oriented ML/DL library recommendation approach, called MLTaskKG. It constructs a knowledge graph that captures AI tasks, ML/DL models, model implementations, repositories, and their relationships by extracting knowledge from different sources such as ML/DL resource websites, papers, ML/DL frameworks, and repositories. Based on the knowledge graph, MLTaskKG recommends ML/DL libraries for developers by matching their requirements on tasks, model characteristics, and implementation information. Our evaluation shows that 92.8% of the tuples sampled from the resulting knowledge graph are correct, demonstrating the high quality of the knowledge graph. A further experiment shows that MLTaskKG can help developers find suitable ML/DL libraries using 47.6% shorter time and with 68.4% higher satisfaction.
@article{DBLP:journals/tse/LiuZPYWS23, author = {Liu, Mingwei and Zhao, Chengyuan and Peng, Xin and Yu, Simin and Wang, Haofen and Sha, Chaofeng}, title = {Task-Oriented {ML/DL} Library Recommendation Based on a Knowledge Graph}, journal = {{IEEE} Trans. Software Eng.}, volume = {49}, number = {8}, pages = {4081--4096}, year = {2023}, url = {https://doi.org/10.1109/TSE.2023.3285280}, doi = {10.1109/TSE.2023.3285280}, timestamp = {Thu, 31 Aug 2023 01:00:00 +0200}, biburl = {https://dblp.org/rec/journals/tse/LiuZPYWS23.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }
ICSME 2023
Knowledge Graph based Explainable Question Retrieval for Programming Tasks

Mingwei Liu, Simin Yu, Xin Peng, and 4 more authors

In IEEE International Conference on Software Maintenance and Evolution, ICSME 2023, Bogotá, Colombia, October 1-6, 2023, 2023

Abs DOI Bib PDF

Developers often seek solutions for their programming problems by retrieving existing questions on technical Q&A sites such as Stack Overflow. In many cases, they fail to find relevant questions due to the knowledge gap between the questions and the queries or feel it hard to choose the desired questions from the returned results due to the lack of explanations about the relevance. In this paper, we propose KGXQR, a knowledge graph based explainable question retrieval approach for programming tasks. It uses BERT-based sentence similarity to retrieve candidate Stack Overflow questions that are relevant to a given query. To bridge the knowledge gap and enhance the performance of question retrieval, it constructs a software development related concept knowledge graph and trains a question relevance prediction model to re-rank the candidate questions. The model is trained based on a combined sentence representation of BERT-based sentence embedding and graph-based concept embedding. To help understand the relevance of the returned Stack Overflow questions, KGXQR further generates explanations based on the association paths between the concepts involved in the query and the Stack Overflow questions. The evaluation shows that KGXQR outperforms the baselines in terms of accuracy, recall, MRR, and MAP and the generated explanations help the users to find the desired questions faster and more accurately.
@inproceedings{DBLP:conf/icsm/0002Y0DYXZ23, author = {Liu, Mingwei and Yu, Simin and Peng, Xin and Du, Xueying and Yang, Tianyong and Xu, Huanjun and Zhang, Gaoyang}, title = {Knowledge Graph based Explainable Question Retrieval for Programming Tasks}, booktitle = {{IEEE} International Conference on Software Maintenance and Evolution, {ICSME} 2023, Bogot{\'{a}}, Colombia, October 1-6, 2023}, pages = {123--135}, publisher = {{IEEE}}, year = {2023}, url = {https://doi.org/10.1109/ICSME58846.2023.00023}, doi = {10.1109/ICSME58846.2023.00023}, timestamp = {Tue, 09 Jan 2024 17:27:46 +0100}, biburl = {https://dblp.org/rec/conf/icsm/0002Y0DYXZ23.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }
ASE 2023
CodeGen4Libs: A Two-Stage Approach for Library-Oriented Code Generation

Mingwei Liu, Tianyong Yang, Yiling Lou, and 3 more authors

In 38th IEEE/ACM International Conference on Automated Software Engineering, ASE 2023, Luxembourg, September 11-15, 2023, 2023

Abs DOI Bib PDF Code

Automated code generation has been extensively studied in recent literature. In this work, we first survey 66 participants to motivate a more pragmatic code generation scenario, i.e., library-oriented code generation, where the generated code should implement the functionally of the natural language query with the given library. We then revisit existing learning-based code generation techniques and find they have limited effectiveness in such a library-oriented code generation scenario. To address this limitation, we propose a novel library-oriented code generation technique, CodeGen4Libs, which incorporates two stages: import generation and code generation. The import generation stage generates import statements for the natural language query with the given third-party libraries, while the code generation stage generates concrete code based on the generated imports and the query. To evaluate the effectiveness of our approach, we conduct extensive experiments on a dataset of 403,780 data items. Our results demonstrate that CodeGen4Libs outperforms baseline models in both import generation and code generation stages, achieving improvements of up to 97.4% on EM (Exact Match), 54.5% on BLEU, and 53.5% on Hit@All. Overall, our proposed CodeGen4Libs approach shows promising results in generating high-quality code with specific third-party libraries, which can improve the efficiency and effectiveness of software development.
@inproceedings{DBLP:conf/kbse/LiuYLDWP23, author = {Liu, Mingwei and Yang, Tianyong and Lou, Yiling and Du, Xueying and Wang, Ying and Peng, Xin}, title = {CodeGen4Libs: {A} Two-Stage Approach for Library-Oriented Code Generation}, booktitle = {38th {IEEE/ACM} International Conference on Automated Software Engineering, {ASE} 2023, Luxembourg, September 11-15, 2023}, pages = {434--445}, publisher = {{IEEE}}, year = {2023}, url = {https://doi.org/10.1109/ASE56229.2023.00159}, doi = {10.1109/ASE56229.2023.00159}, timestamp = {Thu, 16 Nov 2023 09:03:51 +0100}, biburl = {https://dblp.org/rec/conf/kbse/LiuYLDWP23.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }
ESEC/FSE 2023
KG4CraSolver: Recommending Crash Solutions via Knowledge Graph

Xueying Du, Yiling Lou, Mingwei Liu, and 2 more authors

In Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC/FSE 2023, San Francisco, CA, USA, December 3-9, 2023, 2023

Abs DOI Bib PDF

Fixing crashes is challenging, and developers often discuss their encountered crashes and refer to similar crashes and solutions on online Q&A forums (e.g., Stack Overflow). However, a crash often involves very complex contexts, which includes different contextual elements, e.g., purposes, environments, code, and crash traces. Existing crash solution recommendation or general solution recommendation techniques only use an incomplete context or treat the entire context as pure texts to search relevant solutions for a given crash, resulting in inaccurate recommendation results. In this work, we propose a novel crash solution knowledge graph (KG) to summarize the complete crash context and its solution with a graph-structured representation. To construct the crash solution KG automatically, we propose to leverage prompt learning to construct the KG from SO threads with a small set of labeled data. Based on the constructed KG, we further propose a novel KG-based crash solution recommendation technique KG4CraSolver, which precisely finds the relevant SO thread for an encountered crash by finely analyzing and matching the complete crash context based on the crash solution KG. The evaluation results show that the constructed KG is of high quality and KG4CraSolver outperforms baselines in terms of all metrics (e.g., 13.4%-113.4% MRR improvements). Moreover, we perform a user study and find that KG4CraSolver helps participants find crash solutions 34.4% faster and 63.3% more accurately.
@inproceedings{DBLP:conf/sigsoft/DuLL0Y23, author = {Du, Xueying and Lou, Yiling and Liu, Mingwei and Peng, Xin and Yang, Tianyong}, editor = {Chandra, Satish and Blincoe, Kelly and Tonella, Paolo}, title = {KG4CraSolver: Recommending Crash Solutions via Knowledge Graph}, booktitle = {Proceedings of the 31st {ACM} Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, {ESEC/FSE} 2023, San Francisco, CA, USA, December 3-9, 2023}, pages = {1242--1254}, publisher = {{ACM}}, year = {2023}, url = {https://doi.org/10.1145/3611643.3616317}, doi = {10.1145/3611643.3616317}, timestamp = {Mon, 04 Dec 2023 12:10:56 +0100}, biburl = {https://dblp.org/rec/conf/sigsoft/DuLL0Y23.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }
ESEC/FSE 2023
Recommending Analogical APIs via Knowledge Graph Embedding

Mingwei Liu, Yanjun Yang, Yiling Lou, and 4 more authors

In Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC/FSE 2023, San Francisco, CA, USA, December 3-9, 2023, 2023

Awarded Abs DOI Bib PDF

ACM SIGSOFT Distinguished Paper Award

Library migration, which replaces the current library with a different one to retain the same software behavior, is common in software evolution. An essential part of this is finding an analogous API for the desired functionality. However, due to the multitude of libraries/APIs, manually finding such an API is time-consuming and error-prone. Researchers created automated analogical API recommendation techniques, notably documentation-based methods. Despite potential, these methods have limitations, e.g., incomplete semantic understanding in documentation and scalability issues. In this study, we present KGE4AR, a novel documentation-based approach using knowledge graph (KG) embedding for recommending analogical APIs during library migration. KGE4AR introduces a unified API KG to comprehensively represent documentation knowledge, capturing high-level semantics. It further embeds this unified API KG into vectors for efficient, scalable similarity calculation. We assess KGE4AR with 35,773 Java libraries in two scenarios, with and without target libraries. KGE4AR notably outperforms state-of-the-art techniques (e.g., 47.1%-143.0% and 11.7%-80.6% MRR improvements), showcasing scalability with growing library counts.
@inproceedings{DBLP:conf/sigsoft/0002YL0ZDY23, author = {Liu, Mingwei and Yang, Yanjun and Lou, Yiling and Peng, Xin and Zhou, Zhong and Du, Xueying and Yang, Tianyong}, editor = {Chandra, Satish and Blincoe, Kelly and Tonella, Paolo}, title = {Recommending Analogical APIs via Knowledge Graph Embedding}, booktitle = {Proceedings of the 31st {ACM} Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, {ESEC/FSE} 2023, San Francisco, CA, USA, December 3-9, 2023}, pages = {1496--1508}, publisher = {{ACM}}, year = {2023}, url = {https://doi.org/10.1145/3611643.3616305}, doi = {10.1145/3611643.3616305}, timestamp = {Mon, 04 Dec 2023 11:02:39 +0100}, biburl = {https://dblp.org/rec/conf/sigsoft/0002YL0ZDY23.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }

arXiv

No More Manual Tests? Evaluating and Improving ChatGPT for Unit Test Generation

Zhiqiang Yuan, Yiling Lou, Mingwei Liu, and 4 more authors

CoRR, 2023

DOI arXiv Bib PDF

@article{DBLP:journals/corr/abs-2305-04207,
  author = {Yuan, Zhiqiang and Lou, Yiling and Liu, Mingwei and Ding, Shiji and Wang, Kaixin and Chen, Yixuan and Peng, Xin},
  title = {No More Manual Tests? Evaluating and Improving ChatGPT for Unit Test
   Generation},
  journal = {CoRR},
  volume = {abs/2305.04207},
  year = {2023},
  url = {https://doi.org/10.48550/arXiv.2305.04207},
  doi = {10.48550/ARXIV.2305.04207},
  eprinttype = {arXiv},
  timestamp = {Tue, 13 Jun 2023 01:00:00 +0200},
  biburl = {https://dblp.org/rec/journals/corr/abs-2305-04207.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org},
}

arXiv

Evaluating Instruction-Tuned Large Language Models on Code Comprehension and Generation

Zhiqiang Yuan, Junwei Liu, Qiancheng Zi, and 3 more authors

CoRR, 2023

DOI arXiv Bib PDF

@article{DBLP:journals/corr/abs-2308-01240,
  author = {Yuan, Zhiqiang and Liu, Junwei and Zi, Qiancheng and Liu, Mingwei and Peng, Xin and Lou, Yiling},
  title = {Evaluating Instruction-Tuned Large Language Models on Code Comprehension
   and Generation},
  journal = {CoRR},
  volume = {abs/2308.01240},
  year = {2023},
  url = {https://doi.org/10.48550/arXiv.2308.01240},
  doi = {10.48550/ARXIV.2308.01240},
  eprinttype = {arXiv},
  timestamp = {Mon, 21 Aug 2023 01:00:00 +0200},
  biburl = {https://dblp.org/rec/journals/corr/abs-2308-01240.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org},
}

arXiv

Resolving Crash Bugs via Large Language Models: An Empirical Study

Xueying Du, Mingwei Liu, Juntao Li, and 3 more authors

CoRR, 2023

DOI arXiv Bib PDF

@article{DBLP:journals/corr/abs-2312-10448,
  author = {Du, Xueying and Liu, Mingwei and Li, Juntao and Wang, Hanlin and Peng, Xin and Lou, Yiling},
  title = {Resolving Crash Bugs via Large Language Models: An Empirical Study},
  journal = {CoRR},
  volume = {abs/2312.10448},
  year = {2023},
  url = {https://doi.org/10.48550/arXiv.2312.10448},
  doi = {10.48550/ARXIV.2312.10448},
  eprinttype = {arXiv},
  timestamp = {Tue, 16 Jan 2024 00:00:00 +0100},
  biburl = {https://dblp.org/rec/journals/corr/abs-2312-10448.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org},
}

JoS

Survey on Representation Learning Methods of Knowledge Graph for Link Prediction

Xueying Du, Mingwei Liu, Liwei Shen, and 1 more author

Journal of Software, 2023

Bib PDF

@article{jos2023survey,
  title = {Survey on Representation Learning Methods of Knowledge Graph for Link Prediction},
  author = {Du, Xueying and Liu, Mingwei and Shen, Liwei and Peng, Xin},
  journal = {Journal of Software},
  volume = {35},
  number = {1},
  pages = {87--117},
  year = {2023},
}

2022

TSE
API-Related Developer Information Needs in Stack Overflow

Mingwei Liu, Xin Peng, Andrian Marcus, and 3 more authors

IEEE Trans. Software Eng., 2022

Abs DOI Bib PDF

Stack Overflow (SO) provides informal documentation for APIs in response to questions that express API related developer needs. Navigating the information available on SO and getting information related to a particular API and need is challenging due to the vast amount of questions and answers and the tag-driven structure of SO. In this paper we focus on identifying and classifying fine-grained developer needs expressed in sentences of API-related SO questions, as well as the specific information types used to express such needs, and the different roles APIs play in these questions and their answers. We derive a taxonomy, complementing existing ones, through an empirical study of 266 SO posts. We then develop and evaluate an approach for the automated identification of the fine-grained developer needs in SO threads, which takes a thread as input and outputs the corresponding developer needs, the types of information expressing them, and the roles of API elements relevant to the needs. To show a practical application of our taxonomy, we introduce and evaluate an approach for the automated retrieval of SO questions, based on these developer needs.
@article{DBLP:journals/tse/LiuPMXTZ22, author = {Liu, Mingwei and Peng, Xin and Marcus, Andrian and Xing, Shuangshuang and Treude, Christoph and Zhao, Chengyuan}, title = {API-Related Developer Information Needs in Stack Overflow}, journal = {{IEEE} Trans. Software Eng.}, volume = {48}, number = {11}, pages = {4485--4500}, year = {2022}, url = {https://doi.org/10.1109/TSE.2021.3120203}, doi = {10.1109/TSE.2021.3120203}, timestamp = {Tue, 13 Jun 2023 01:00:00 +0200}, biburl = {https://dblp.org/rec/journals/tse/LiuPMXTZ22.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }
ESEC/FSE 2022
How to formulate specific how-to questions in software development?

Mingwei Liu, Xin Peng, Andrian Marcus, and 4 more authors

In Proceedings of the 30th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC/FSE 2022, Singapore, Singapore, November 14-18, 2022, 2022

Abs DOI Bib PDF

Developers often ask how-to questions using search engines, technical Q&A communities, and interactive Q&A systems to seek help for specific programming tasks. However, they often do not formulate the questions in a specific way, making it hard for the systems to return the best answers. We propose an approach (TaskKG4Q) that interactively helps developers formulate a programming related how-to question. TaskKG4Q is using a programming task knowledge graph (task KG in short) mined from Stack Overflow questions, which provides a hierarchical conceptual structure for tasks in terms of [actions], [objects], and [constraints]. An empirical evaluation of the intrinsic quality of the task KG revealed that 75.0% of the annotated questions in the task KG are correct. The comparison between TaskKG4Q and two baselines revealed that TaskKG4Q can help developers formulate more specific how-to questions. More so, an empirical study with novice programmers revealed that they write more effective questions for finding answers to their programming tasks on Stack Overflow.
@inproceedings{DBLP:conf/sigsoft/Liu0MTXXY22, author = {Liu, Mingwei and Peng, Xin and Marcus, Andrian and Treude, Christoph and Xie, Jiazhan and Xu, Huanjun and Yang, Yanjun}, editor = {Roychoudhury, Abhik and Cadar, Cristian and Kim, Miryung}, title = {How to formulate specific how-to questions in software development?}, booktitle = {Proceedings of the 30th {ACM} Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, {ESEC/FSE} 2022, Singapore, Singapore, November 14-18, 2022}, pages = {306--318}, publisher = {{ACM}}, year = {2022}, url = {https://doi.org/10.1145/3540250.3549160}, doi = {10.1145/3540250.3549160}, timestamp = {Mon, 26 Jun 2023 01:00:00 +0200}, biburl = {https://dblp.org/rec/conf/sigsoft/Liu0MTXXY22.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }

2021

ESEC/FSE 2021
Learning-based extraction of first-order logic representations of API directives

Mingwei Liu, Xin Peng, Andrian Marcus, and 5 more authors

In ESEC/FSE ’21: 29th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, Athens, Greece, August 23-28, 2021, 2021

Abs DOI Bib PDF

Developers often rely on API documentation to learn API directives, i.e., constraints and guidelines related to API usage. Failing to follow API directives may cause defects or improper implementations. Since there are no industry-wide standards on how to document API directives, they take many forms and are often hard to understand by developers or challenging to parse with tools. In this paper, we propose a learning based approach for extracting first-order logic representations of API directives (FOL directives for short). The approach, called LEADFOL, uses a joint learning method to extract atomic formulas by identifying the predicates and arguments involved in directive sentences, and recognizes the logical relations between atomic formulas, by parsing the sentence structures. It then parses the arguments and uses a learning based method to link API references to their corresponding API elements. Finally, it groups the formulas of the same class or method together and transforms them into conjunctive normal form. Our evaluation shows that LEADFOL can accurately extract more FOL directives than a state-of-the-art approach and that the extracted FOL directives are useful in supporting code reviews.
@inproceedings{DBLP:conf/sigsoft/Liu0MTBLXZ21, author = {Liu, Mingwei and Peng, Xin and Marcus, Andrian and Treude, Christoph and Bai, Xuefang and Lyu, Gang and Xie, Jiazhan and Zhang, Xiaoxin}, editor = {Spinellis, Diomidis and Gousios, Georgios and Chechik, Marsha and Penta, Massimiliano Di}, title = {Learning-based extraction of first-order logic representations of {API} directives}, booktitle = {{ESEC/FSE} '21: 29th {ACM} Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, Athens, Greece, August 23-28, 2021}, pages = {491--502}, publisher = {{ACM}}, year = {2021}, url = {https://doi.org/10.1145/3468264.3468618}, doi = {10.1145/3468264.3468618}, timestamp = {Tue, 13 Jun 2023 01:00:00 +0200}, biburl = {https://dblp.org/rec/conf/sigsoft/Liu0MTBLXZ21.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }

JoS

Automatic Code Semantic Tag Generation Approach Based on Software Knowledge Graph

Shuangshuang Xing, Mingwei Liu, and Xin Peng

Journal of Software, 2021

Bib PDF

@article{jos2021automatic,
  title = {Automatic Code Semantic Tag Generation Approach Based on Software Knowledge Graph},
  author = {Xing, Shuangshuang and Liu, Mingwei and Peng, Xin},
  journal = {Journal of Software},
  volume = {33},
  number = {11},
  pages = {4027--4045},
  year = {2021},
}

2020

ICSME 2020

Source Code based On-demand Class Documentation Generation

Mingwei Liu, Xin Peng, Xiujie Meng, and 5 more authors

In IEEE International Conference on Software Maintenance and Evolution, ICSME 2020, Adelaide, Australia, September 28 - October 2, 2020, 2020

DOI Bib PDF

@inproceedings{DBLP:conf/icsm/LiuPMXXWLL20,
  author = {Liu, Mingwei and Peng, Xin and Meng, Xiujie and Xu, Huanjun and Xing, Shuangshuang and Wang, Xin and Liu, Yang and Lv, Gang},
  title = {Source Code based On-demand Class Documentation Generation},
  booktitle = {{IEEE} International Conference on Software Maintenance and Evolution,
                    {ICSME} 2020, Adelaide, Australia, September 28 - October 2, 2020},
  pages = {864--865},
  publisher = {{IEEE}},
  year = {2020},
  url = {https://doi.org/10.1109/ICSME46990.2020.00114},
  doi = {10.1109/ICSME46990.2020.00114},
  timestamp = {Fri, 04 Aug 2023 01:00:00 +0200},
  biburl = {https://dblp.org/rec/conf/icsm/LiuPMXXWLL20.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org},
}

ICSME 2020

Learning based and Context Aware Non-Informative Comment Detection

Mingwei Liu, Yanjun Yang, Xin Peng, and 4 more authors

In IEEE International Conference on Software Maintenance and Evolution, ICSME 2020, Adelaide, Australia, September 28 - October 2, 2020, 2020

DOI Bib PDF

@inproceedings{DBLP:conf/icsm/LiuYPWZWX20,
  author = {Liu, Mingwei and Yang, Yanjun and Peng, Xin and Wang, Chong and Zhao, Chengyuan and Wang, Xin and Xing, Shuangshuang},
  title = {Learning based and Context Aware Non-Informative Comment Detection},
  booktitle = {{IEEE} International Conference on Software Maintenance and Evolution,
                    {ICSME} 2020, Adelaide, Australia, September 28 - October 2, 2020},
  pages = {866--867},
  publisher = {{IEEE}},
  year = {2020},
  url = {https://doi.org/10.1109/ICSME46990.2020.00115},
  doi = {10.1109/ICSME46990.2020.00115},
  timestamp = {Tue, 13 Jun 2023 01:00:00 +0200},
  biburl = {https://dblp.org/rec/conf/icsm/LiuYPWZWX20.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org},
}

ASE 2020
Generating Concept based API Element Comparison Using a Knowledge Graph

Yang Liu, Mingwei Liu, Xin Peng, and 3 more authors

In 35th IEEE/ACM International Conference on Automated Software Engineering, ASE 2020, Melbourne, Australia, September 21-25, 2020, 2020

Abs DOI Bib PDF

Developers are concerned with the comparison of similar APIs in terms of their commonalities and (often subtle) differences. Our empirical study of Stack Overflow questions and API documentation confirms that API comparison questions are common and can often be answered by knowledge contained in API reference documentation. Our study also identifies eight types of API statements that are useful for API comparison. Based on these findings, we propose a knowledge graph based approach APIComp that automatically extracts API knowledge from API reference documentation to support the comparison of a pair of API classes or methods from different aspects. Our approach includes an offline phase for constructing an API knowledge graph, and an online phase for generating an API comparison result for a given pair of API elements. Our evaluation shows that the quality of different kinds of extracted knowledge in the API knowledge graph is generally high. Furthermore, the comparison results generated by APIComp are significantly better than those generated by a baseline approach based on heuristic rules and text similarity, and our generated API comparison results are useful for helping developers in API selection tasks.
@inproceedings{DBLP:conf/kbse/LiuLPTXZ20, author = {Liu, Yang and Liu, Mingwei and Peng, Xin and Treude, Christoph and Xing, Zhenchang and Zhang, Xiaoxin}, title = {Generating Concept based {API} Element Comparison Using a Knowledge Graph}, booktitle = {35th {IEEE/ACM} International Conference on Automated Software Engineering, {ASE} 2020, Melbourne, Australia, September 21-25, 2020}, pages = {834--845}, publisher = {{IEEE}}, year = {2020}, url = {https://doi.org/10.1145/3324884.3416628}, doi = {10.1145/3324884.3416628}, timestamp = {Mon, 26 Jun 2023 01:00:00 +0200}, biburl = {https://dblp.org/rec/conf/kbse/LiuLPTXZ20.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }
ESEC/FSE 2020
API method recommendation via explicit matching of functionality verb phrases

Wenkai Xie, Xin Peng, Mingwei Liu, and 4 more authors

In ESEC/FSE ’20: 28th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, Virtual Event, USA, November 8-13, 2020, 2020

Abs DOI Bib PDF

Due to the lexical gap between functionality descriptions and user queries, documentation-based API retrieval often produces poor results.Verb phrases and their phrase patterns are essential in both describing API functionalities and interpreting user queries. Thus we hypothesize that API retrieval can be facilitated by explicitly recognizing and matching between the fine-grained structures of functionality descriptions and user queries. To verify this hypothesis, we conducted a large-scale empirical study on the functionality descriptions of 14,733 JDK and Android API methods. We identified 356 different functionality verbs from the descriptions, which were grouped into 87 functionality categories, and we extracted 523 phrase patterns from the verb phrases of the descriptions. Building on these findings, we propose an API method recommendation approach based on explicit matching of functionality verb phrases in functionality descriptions and user queries, called PreMA. Our evaluation shows that PreMA can accurately recognize the functionality categories (92.8%) and phrase patterns (90.4%) of functionality description sentences; and when used for API retrieval tasks, PreMA can help participants complete their tasks more accurately and with fewer retries compared to a baseline approach.
@inproceedings{DBLP:conf/sigsoft/Xie0LTXZZ20, author = {Xie, Wenkai and Peng, Xin and Liu, Mingwei and Treude, Christoph and Xing, Zhenchang and Zhang, Xiaoxin and Zhao, Wenyun}, editor = {Devanbu, Prem and Cohen, Myra B. and Zimmermann, Thomas}, title = {{API} method recommendation via explicit matching of functionality verb phrases}, booktitle = {{ESEC/FSE} '20: 28th {ACM} Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, Virtual Event, USA, November 8-13, 2020}, pages = {1015--1026}, publisher = {{ACM}}, year = {2020}, url = {https://doi.org/10.1145/3368089.3409731}, doi = {10.1145/3368089.3409731}, timestamp = {Tue, 13 Jun 2023 01:00:00 +0200}, biburl = {https://dblp.org/rec/conf/sigsoft/Xie0LTXZZ20.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }

2019

ESEC/FSE 2019
A learning-based approach for automatic construction of domain glossary from source code and documentation

Chong Wang, Xin Peng, Mingwei Liu, and 4 more authors

In Proceedings of the ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC/SIGSOFT FSE 2019, Tallinn, Estonia, August 26-30, 2019, 2019

Abs DOI Bib PDF

A domain glossary that organizes domain-specific concepts and their aliases and relations is essential for knowledge acquisition and software development. Existing approaches use linguistic heuristics or term-frequency-based statistics to identify domain specific terms from software documentation, and thus the accuracy is often low. In this paper, we propose a learning-based approach for automatic construction of domain glossary from source code and software documentation. The approach uses a set of high-quality seed terms identified from code identifiers and natural language concept definitions to train a domain-specific prediction model to recognize glossary terms based on the lexical and semantic context of the sentences mentioning domain-specific concepts. It then merges the aliases of the same concepts to their canonical names, selects a set of explanation sentences for each concept, and identifies "is a", "has a", and "related to" relations between the concepts. We apply our approach to deep learning domain and Hadoop domain and harvest 5,382 and 2,069 concepts together with 16,962 and 6,815 relations respectively. Our evaluation validates the accuracy of the extracted domain glossary and its usefulness for the fusion and acquisition of knowledge from different documents of different projects.
@inproceedings{DBLP:conf/sigsoft/Wang0LXBXW19, author = {Wang, Chong and Peng, Xin and Liu, Mingwei and Xing, Zhenchang and Bai, Xuefang and Xie, Bing and Wang, Tuo}, editor = {Dumas, Marlon and Pfahl, Dietmar and Apel, Sven and Russo, Alessandra}, title = {A learning-based approach for automatic construction of domain glossary from source code and documentation}, booktitle = {Proceedings of the {ACM} Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, {ESEC/SIGSOFT} {FSE} 2019, Tallinn, Estonia, August 26-30, 2019}, pages = {97--108}, publisher = {{ACM}}, year = {2019}, url = {https://doi.org/10.1145/3338906.3338963}, doi = {10.1145/3338906.3338963}, timestamp = {Sat, 30 Sep 2023 09:56:36 +0200}, biburl = {https://dblp.org/rec/conf/sigsoft/Wang0LXBXW19.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }
ESEC/FSE 2019
Generating query-specific class API summaries

Mingwei Liu, Xin Peng, Andrian Marcus, and 4 more authors

In Proceedings of the ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC/SIGSOFT FSE 2019, Tallinn, Estonia, August 26-30, 2019, 2019

Abs DOI Bib PDF

Source code summaries are concise representations, in form of text and/or code, of complex code elements and are meant to help developers gain a quick understanding that in turns help them perform specific tasks. Generation of summaries that are task-specific is still a challenge in the automatic code summarization field. We propose an approach for generating on-demand, extrinsic hybrid summaries for API classes, relevant to a programming task, formulated as a natural language query. The summaries include the most relevant sentences extracted from the API reference documentation and the most relevant methods. External evaluators assessed the summaries generated for classes retrieved from JDK and Android libraries for several programming tasks. The majority found that the summaries are complete, concise, and readable. A comparison with summaries produce by three baseline approaches revealed that the information present only in our summaries is more relevant than the one present only in the baselines summaries. Finally, an extrinsic evaluation study showed that the summaries help the users evaluating the correctness of API retrieval results, faster and more accurately.
@inproceedings{DBLP:conf/sigsoft/Liu0MXXXL19, author = {Liu, Mingwei and Peng, Xin and Marcus, Andrian and Xing, Zhenchang and Xie, Wenkai and Xing, Shuangshuang and Liu, Yang}, editor = {Dumas, Marlon and Pfahl, Dietmar and Apel, Sven and Russo, Alessandra}, title = {Generating query-specific class {API} summaries}, booktitle = {Proceedings of the {ACM} Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering, {ESEC/SIGSOFT} {FSE} 2019, Tallinn, Estonia, August 26-30, 2019}, pages = {120--130}, publisher = {{ACM}}, year = {2019}, url = {https://doi.org/10.1145/3338906.3338971}, doi = {10.1145/3338906.3338971}, timestamp = {Fri, 04 Aug 2023 01:00:00 +0200}, biburl = {https://dblp.org/rec/conf/sigsoft/Liu0MXXXL19.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }

2018

ICSME 2018

Automatic Generation of API Documentations for Open-Source Projects

Xin Peng, Yifan Zhao, Mingwei Liu, and 4 more authors

In IEEE Third International Workshop on Dynamic Software Documentation, DySDoc@ICSME 2018, Madrid, Spain, September 25, 2018, 2018

DOI Bib PDF

@inproceedings{DBLP:conf/icsm/PengZLZLWX18,
  author = {Peng, Xin and Zhao, Yifan and Liu, Mingwei and Zhang, Fengyi and Liu, Yang and Wang, Xin and Xing, Zhenchang},
  title = {Automatic Generation of {API} Documentations for Open-Source Projects},
  booktitle = {{IEEE} Third International Workshop on Dynamic Software Documentation,
                    DySDoc@ICSME 2018, Madrid, Spain, September 25, 2018},
  pages = {7--8},
  publisher = {{IEEE}},
  year = {2018},
  url = {https://doi.org/10.1109/DySDoc3.2018.00010},
  doi = {10.1109/DYSDOC3.2018.00010},
  timestamp = {Mon, 26 Jun 2023 01:00:00 +0200},
  biburl = {https://dblp.org/rec/conf/icsm/PengZLZLWX18.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org},
}

ICSME 2018
Improving API Caveats Accessibility by Mining API Caveats Knowledge Graph

Hongwei Li, Sirui Li, Jiamou Sun, and 4 more authors

In 2018 IEEE International Conference on Software Maintenance and Evolution, ICSME 2018, Madrid, Spain, September 23-29, 2018, 2018

Awarded Abs DOI Bib PDF

IEEE TCSE Distinguished Paper Awards

API documentation provides important knowledge about the functionality and usage of APIs. In this paper, we focus on API caveats that developers should be aware of in order to avoid unintended use of an API. Our formative study of Stack Overflow questions suggests that API caveats are often scattered in multiple API documents, and are buried in lengthy textual descriptions. These characteristics make the API caveats less discoverable. When developers fail to notice API caveats, it is very likely to cause some unexpected programming errors. In this paper, we propose natural language processing(NLP) techniques to extract ten subcategories of API caveat sentences from API documentation and link these sentences to API entities in an API caveats knowledge graph. The API caveats knowledge graph can support information retrieval based or entity-centric search of API caveats. As a proof-of-concept, we construct an API caveats knowledge graph for Android APIs from the API documentation on the Android Developers website. We study the abundance of different subcategories of API caveats and use a sampling method to manually evaluate the quality of the API caveats knowledge graph. We also conduct a user study to validate whether and how the API caveats knowledge graph may improve the accessibility of API caveats in API documentation.
@inproceedings{DBLP:conf/icsm/LiLSXPLZ18, author = {Li, Hongwei and Li, Sirui and Sun, Jiamou and Xing, Zhenchang and Peng, Xin and Liu, Mingwei and Zhao, Xuejiao}, title = {Improving {API} Caveats Accessibility by Mining {API} Caveats Knowledge Graph}, booktitle = {2018 {IEEE} International Conference on Software Maintenance and Evolution, {ICSME} 2018, Madrid, Spain, September 23-29, 2018}, pages = {183--193}, publisher = {{IEEE} Computer Society}, year = {2018}, url = {https://doi.org/10.1109/ICSME.2018.00028}, doi = {10.1109/ICSME.2018.00028}, timestamp = {Tue, 13 Jun 2023 01:00:00 +0200}, biburl = {https://dblp.org/rec/conf/icsm/LiLSXPLZ18.bib}, bibsource = {dblp computer science bibliography, https://dblp.org}, }

Internetware 2018

Searching StackOverflow Questions with Multi-Faceted Categorization

Mingwei Liu, Xin Peng, Qingtao Jiang, and 3 more authors

In Proceedings of the Tenth Asia-Pacific Symposium on Internetware, Internetware 2018, Beijing, China, September 16-16, 2018, 2018

DOI Bib PDF

@inproceedings{DBLP:conf/internetware/Liu0JMYZ18,
  author = {Liu, Mingwei and Peng, Xin and Jiang, Qingtao and Marcus, Andrian and Yang, Junwen and Zhao, Wenyun},
  title = {Searching StackOverflow Questions with Multi-Faceted Categorization},
  booktitle = {Proceedings of the Tenth Asia-Pacific Symposium on Internetware, Internetware
   2018, Beijing, China, September 16-16, 2018},
  pages = {10:1--10:10},
  publisher = {{ACM}},
  year = {2018},
  url = {https://doi.org/10.1145/3275219.3275227},
  doi = {10.1145/3275219.3275227},
  timestamp = {Tue, 13 Jun 2023 01:00:00 +0200},
  biburl = {https://dblp.org/rec/conf/internetware/Liu0JMYZ18.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org},
}

TOSEM

Exploring Large Language Models in Resolving Environment-Related Crash Bugs: Localizing and Repairing

Xueying Du, Mingwei Liu, Hanlin Wang, and 3 more authors

ACM Transactions on Software Engineering and Methodology

Bib

@article{duexploring,
  title = {Exploring Large Language Models in Resolving Environment-Related Crash Bugs: Localizing and Repairing},
  author = {Du, Xueying and Liu, Mingwei and Wang, Hanlin and Li, Juntao and Peng, Xin and Lou, Yiling},
  journal = {ACM Transactions on Software Engineering and Methodology},
  publisher = {ACM New York, NY},
}