[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"similar-thunlp--LLMxMapReduce":3,"tool-thunlp--LLMxMapReduce":64},[4,17,27,35,43,56],{"id":5,"name":6,"github_repo":7,"description_zh":8,"stars":9,"difficulty_score":10,"last_commit_at":11,"category_tags":12,"status":16},3808,"stable-diffusion-webui","AUTOMATIC1111\u002Fstable-diffusion-webui","stable-diffusion-webui 是一个基于 Gradio 构建的网页版操作界面，旨在让用户能够轻松地在本地运行和使用强大的 Stable Diffusion 图像生成模型。它解决了原始模型依赖命令行、操作门槛高且功能分散的痛点，将复杂的 AI 绘图流程整合进一个直观易用的图形化平台。\n\n无论是希望快速上手的普通创作者、需要精细控制画面细节的设计师，还是想要深入探索模型潜力的开发者与研究人员，都能从中获益。其核心亮点在于极高的功能丰富度：不仅支持文生图、图生图、局部重绘（Inpainting）和外绘（Outpainting）等基础模式，还独创了注意力机制调整、提示词矩阵、负向提示词以及“高清修复”等高级功能。此外，它内置了 GFPGAN 和 CodeFormer 等人脸修复工具，支持多种神经网络放大算法，并允许用户通过插件系统无限扩展能力。即使是显存有限的设备，stable-diffusion-webui 也提供了相应的优化选项，让高质量的 AI 艺术创作变得触手可及。",162132,3,"2026-04-05T11:01:52",[13,14,15],"开发框架","图像","Agent","ready",{"id":18,"name":19,"github_repo":20,"description_zh":21,"stars":22,"difficulty_score":23,"last_commit_at":24,"category_tags":25,"status":16},1381,"everything-claude-code","affaan-m\u002Feverything-claude-code","everything-claude-code 是一套专为 AI 编程助手（如 Claude Code、Codex、Cursor 等）打造的高性能优化系统。它不仅仅是一组配置文件，而是一个经过长期实战打磨的完整框架，旨在解决 AI 代理在实际开发中面临的效率低下、记忆丢失、安全隐患及缺乏持续学习能力等核心痛点。\n\n通过引入技能模块化、直觉增强、记忆持久化机制以及内置的安全扫描功能，everything-claude-code 能显著提升 AI 在复杂任务中的表现，帮助开发者构建更稳定、更智能的生产级 AI 代理。其独特的“研究优先”开发理念和针对 Token 消耗的优化策略，使得模型响应更快、成本更低，同时有效防御潜在的攻击向量。\n\n这套工具特别适合软件开发者、AI 研究人员以及希望深度定制 AI 工作流的技术团队使用。无论您是在构建大型代码库，还是需要 AI 协助进行安全审计与自动化测试，everything-claude-code 都能提供强大的底层支持。作为一个曾荣获 Anthropic 黑客大奖的开源项目，它融合了多语言支持与丰富的实战钩子（hooks），让 AI 真正成长为懂上",138956,2,"2026-04-05T11:33:21",[13,15,26],"语言模型",{"id":28,"name":29,"github_repo":30,"description_zh":31,"stars":32,"difficulty_score":23,"last_commit_at":33,"category_tags":34,"status":16},2271,"ComfyUI","Comfy-Org\u002FComfyUI","ComfyUI 是一款功能强大且高度模块化的视觉 AI 引擎，专为设计和执行复杂的 Stable Diffusion 图像生成流程而打造。它摒弃了传统的代码编写模式，采用直观的节点式流程图界面，让用户通过连接不同的功能模块即可构建个性化的生成管线。\n\n这一设计巧妙解决了高级 AI 绘图工作流配置复杂、灵活性不足的痛点。用户无需具备编程背景，也能自由组合模型、调整参数并实时预览效果，轻松实现从基础文生图到多步骤高清修复等各类复杂任务。ComfyUI 拥有极佳的兼容性，不仅支持 Windows、macOS 和 Linux 全平台，还广泛适配 NVIDIA、AMD、Intel 及苹果 Silicon 等多种硬件架构，并率先支持 SDXL、Flux、SD3 等前沿模型。\n\n无论是希望深入探索算法潜力的研究人员和开发者，还是追求极致创作自由度的设计师与资深 AI 绘画爱好者，ComfyUI 都能提供强大的支持。其独特的模块化架构允许社区不断扩展新功能，使其成为当前最灵活、生态最丰富的开源扩散模型工具之一，帮助用户将创意高效转化为现实。",107662,"2026-04-03T11:11:01",[13,14,15],{"id":36,"name":37,"github_repo":38,"description_zh":39,"stars":40,"difficulty_score":23,"last_commit_at":41,"category_tags":42,"status":16},3704,"NextChat","ChatGPTNextWeb\u002FNextChat","NextChat 是一款轻量且极速的 AI 助手，旨在为用户提供流畅、跨平台的大模型交互体验。它完美解决了用户在多设备间切换时难以保持对话连续性，以及面对众多 AI 模型不知如何统一管理的痛点。无论是日常办公、学习辅助还是创意激发，NextChat 都能让用户随时随地通过网页、iOS、Android、Windows、MacOS 或 Linux 端无缝接入智能服务。\n\n这款工具非常适合普通用户、学生、职场人士以及需要私有化部署的企业团队使用。对于开发者而言，它也提供了便捷的自托管方案，支持一键部署到 Vercel 或 Zeabur 等平台。\n\nNextChat 的核心亮点在于其广泛的模型兼容性，原生支持 Claude、DeepSeek、GPT-4 及 Gemini Pro 等主流大模型，让用户在一个界面即可自由切换不同 AI 能力。此外，它还率先支持 MCP（Model Context Protocol）协议，增强了上下文处理能力。针对企业用户，NextChat 提供专业版解决方案，具备品牌定制、细粒度权限控制、内部知识库整合及安全审计等功能，满足公司对数据隐私和个性化管理的高标准要求。",87618,"2026-04-05T07:20:52",[13,26],{"id":44,"name":45,"github_repo":46,"description_zh":47,"stars":48,"difficulty_score":23,"last_commit_at":49,"category_tags":50,"status":16},2268,"ML-For-Beginners","microsoft\u002FML-For-Beginners","ML-For-Beginners 是由微软推出的一套系统化机器学习入门课程，旨在帮助零基础用户轻松掌握经典机器学习知识。这套课程将学习路径规划为 12 周，包含 26 节精炼课程和 52 道配套测验，内容涵盖从基础概念到实际应用的完整流程，有效解决了初学者面对庞大知识体系时无从下手、缺乏结构化指导的痛点。\n\n无论是希望转型的开发者、需要补充算法背景的研究人员，还是对人工智能充满好奇的普通爱好者，都能从中受益。课程不仅提供了清晰的理论讲解，还强调动手实践，让用户在循序渐进中建立扎实的技能基础。其独特的亮点在于强大的多语言支持，通过自动化机制提供了包括简体中文在内的 50 多种语言版本，极大地降低了全球不同背景用户的学习门槛。此外，项目采用开源协作模式，社区活跃且内容持续更新，确保学习者能获取前沿且准确的技术资讯。如果你正寻找一条清晰、友好且专业的机器学习入门之路，ML-For-Beginners 将是理想的起点。",84991,"2026-04-05T10:45:23",[14,51,52,53,15,54,26,13,55],"数据工具","视频","插件","其他","音频",{"id":57,"name":58,"github_repo":59,"description_zh":60,"stars":61,"difficulty_score":10,"last_commit_at":62,"category_tags":63,"status":16},3128,"ragflow","infiniflow\u002Fragflow","RAGFlow 是一款领先的开源检索增强生成（RAG）引擎，旨在为大语言模型构建更精准、可靠的上下文层。它巧妙地将前沿的 RAG 技术与智能体（Agent）能力相结合，不仅支持从各类文档中高效提取知识，还能让模型基于这些知识进行逻辑推理和任务执行。\n\n在大模型应用中，幻觉问题和知识滞后是常见痛点。RAGFlow 通过深度解析复杂文档结构（如表格、图表及混合排版），显著提升了信息检索的准确度，从而有效减少模型“胡编乱造”的现象，确保回答既有据可依又具备时效性。其内置的智能体机制更进一步，使系统不仅能回答问题，还能自主规划步骤解决复杂问题。\n\n这款工具特别适合开发者、企业技术团队以及 AI 研究人员使用。无论是希望快速搭建私有知识库问答系统，还是致力于探索大模型在垂直领域落地的创新者，都能从中受益。RAGFlow 提供了可视化的工作流编排界面和灵活的 API 接口，既降低了非算法背景用户的上手门槛，也满足了专业开发者对系统深度定制的需求。作为基于 Apache 2.0 协议开源的项目，它正成为连接通用大模型与行业专有知识之间的重要桥梁。",77062,"2026-04-04T04:44:48",[15,14,13,26,54],{"id":65,"github_repo":66,"name":67,"description_en":68,"description_zh":69,"ai_summary_zh":69,"readme_en":70,"readme_zh":71,"quickstart_zh":72,"use_case_zh":73,"hero_image_url":74,"owner_login":75,"owner_name":76,"owner_avatar_url":77,"owner_bio":78,"owner_company":68,"owner_location":68,"owner_email":79,"owner_twitter":68,"owner_website":80,"owner_url":81,"languages":82,"stars":91,"forks":92,"last_commit_at":93,"license":94,"difficulty_score":10,"env_os":95,"env_gpu":96,"env_ram":96,"env_deps":97,"category_tags":106,"github_topics":68,"view_count":107,"oss_zip_url":68,"oss_zip_packed_at":68,"status":16,"created_at":108,"updated_at":109,"faqs":110,"releases":146},162,"thunlp\u002FLLMxMapReduce","LLMxMapReduce",null,"LLMxMapReduce 是一个专为处理超长文本而设计的开源框架，灵感来源于大数据领域的经典 MapReduce 算法。它帮助当前主流大语言模型（LLM）更高效地理解与生成长篇内容，尤其擅长从海量输入中提取、整合关键信息，解决“长文难读、长文难写”的核心痛点。无论是撰写综述、分析报告，还是处理冗长文档，LLMxMapReduce 都能显著提升模型的表现。\n\n该工具特别适合研究人员、开发者和需要处理复杂长文本任务的专业用户。V1 版本通过结构化协议和置信度校准，让小模型也能媲美大模型的长上下文能力；V2 引入类似卷积神经网络的“分层聚合”机制，逐步融合局部信息形成全局理解；V3 更进一步，采用多智能体协作模式，支持自组织、模块化的学术综述生成。目前已集成 OpenAI 及兼容 API，开箱即用，并驱动了在线写作系统 SurveyGO。如果你正被长文本处理困扰，LLMxMapReduce 或许正是你需要的“长文加速器”。","# LLMxMapReduce: Simplified Long-Sequence Processing using Large Language Models\n\n\u003Ca href='https:\u002F\u002Fsurveygo.modelbest.cn\u002F'>\u003Cimg src='https:\u002F\u002Fimg.shields.io\u002Fbadge\u002FDemo-Page-pink'>\u003C\u002Fa> \u003Ca href='https:\u002F\u002Farxiv.org\u002Fabs\u002F2410.09342'>\u003Cimg src='https:\u002F\u002Fimg.shields.io\u002Fbadge\u002FV1-Paper-Green'>\u003C\u002Fa> \u003Ca href='https:\u002F\u002Farxiv.org\u002Fabs\u002F2504.05732'>\u003Cimg src='https:\u002F\u002Fimg.shields.io\u002Fbadge\u002FV2-Paper-blue'>\u003C\u002Fa> \u003Ca href='https:\u002F\u002Fhuggingface.co\u002Fdatasets\u002FR0k1e\u002FSurveyEval'>\u003Cimg src='https:\u002F\u002Fimg.shields.io\u002Fbadge\u002FSurveyEval-Benchmark-yellow'>\u003C\u002Fa> \u003Ca href='README_zh.md'>\u003Cimg src='https:\u002F\u002Fimg.shields.io\u002Fbadge\u002FChinese-Readme-red'>\u003C\u002Fa>\n\n# 🎉 News\n- [x] **`2025.09.13`** Release the code for LLMxMapReduce-V3.\n- [x] **`2025.04.22`** Release [SurveyGO](https:\u002F\u002Fsurveygo.znyx.com.cn\u002F), an online writting system driven by LLMxMapReduce-V2.\n- [x] **`2025.04.09`** Release the paper of LLMxMapReduce-V2 in [arXiv](https:\u002F\u002Farxiv.org\u002Fabs\u002F2504.05732).\n- [x] **`2025.02.21`** Add support for both OpenAI API and OpenAI-compatible APIs (e.g., vLLM).\n- [x] **`2024.10.12`** Release the paper of LLMxMapReduce-V1 in [arXiv](https:\u002F\u002Farxiv.org\u002Fabs\u002F2410.09342).\n- [x] **`2024.09.12`** Release the code for LLMxMapReduce-V1.\n\n# 📚 Overview\n**LLMxMapReduce** is a divide-and-conquer framework designed to enhance modern large language models (LLMs) in understanding and generating long sequences. Developed collaboratively by **AI9STARS**, **OpenBMB**, and **THUNLP**, this framework draws inspiration from the classic MapReduce algorithm introduced in the field of big data. Our goal is to build an LLM-driven distributed computing system capable of efficiently processing long sequences. Here are the main versions of LLMxMapReduce:\n\n* [**LLMxMapReduce-V1**](https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Fblob\u002Fmain\u002FLLMxMapReduce_V1): Utilizes a structured information protocol and in-context confidence calibration to enhance long-sequence understanding, enabling [MiniCPM3-4B](https:\u002F\u002Fgithub.com\u002FOpenBMB\u002FMiniCPM) to outperform 70B-scale models in long-context evaluations.\n* [**LLMxMapReduce-V2**](https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Ftree\u002Fmain\u002FLLMxMapReduce_V2): Introduces an entropy-driven convolutional test-time scaling mechanism to improve the integration of extremely large volumes of information, powering the online [SurveyGO](https:\u002F\u002Fsurveygo.modelbest.cn\u002F) system.\n* [**LLMxMapReduce-V3**](https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Fblob\u002Fmain\u002FLLMxMapReduce_V3): Introduces an interactive, modular, and self-organized multi-agent system for academic survey generation.\n\n# 📖 Introduction\n\n\nLong-form generation is crucial for a wide range of practical applications, typically categorized into short-to-long and long-to-long generation. While short-to-long generations have received considerable attention, generating long texts from extremely long resources remains relatively underexplored. The primary challenge in long-to-long generation lies in effectively integrating and analyzing relevant information from extensive inputs, which remains difficult for current large language models (LLMs). In this paper, we propose LLMxMapReduce-V2, a novel test-time scaling strategy designed to enhance the ability of LLMs to process extremely long inputs. Drawing inspiration from convolutional neural networks, which iteratively integrate local features into higher-level global representations, LLMxMapReduce-V2 utilizes stacked convolutional scaling layers to progressively expand the understanding of input materials. Both quantitative and qualitative experimental results demonstrate that our approach substantially enhances the ability of LLMs to process long inputs and generate coherent, informative long-form articles, outperforming several representative baselines.\n\n\u003Cdiv align=\"center\">\n  \u003Cimg src=\"https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Fthunlp_LLMxMapReduce_readme_ccc5cd7b79b1.jpg\" alt=\"$\\text{LLM}\\times \\text{MapReduce}$-V2 framework\">\n\u003C\u002Fdiv>\n\n# ⚡️ Getting Started\nThe following steps are about LLMxMapReduce-V2. If you want to use LLMxMapReduce-V1, you need to refer to [here](LLMxMapReduce_V1\u002FREADME.md). If you want to use LLMxMapReduce-V3, you need to refer to [here](LLMxMapReduce_V3\u002FREADME.md). \n\nTo get started, ensure all dependencies listed in requirements.txt are installed. You can do this by running:\n```bash\ncd LLMxMapReduce_V2\nconda create -n llm_mr_v2 python=3.11\nconda activate llm_mr_v2\npip install -r requirements.txt\npython -m playwright install --with-deps chromium\n```\n\nBefore evaluation, you need to download punkt_tab firstly.\n```python\nimport nltk\nnltk.download('punkt_tab')\n```\n## Env config\nPlease set your OPENAI_API_KEY and OPENAI_API_BASE in your environment variables before start the pipeline. If you use miniconda, replace `anaconda3` in `LD_LIBRARY_PATH` with `miniconda3`\n```bash\nexport LD_LIBRARY_PATH=${HOME}\u002Fanaconda3\u002Fenvs\u002Fllm_mr_v2\u002Flib\u002Fpython3.11\u002Fsite-packages\u002Fnvidia\u002Fnvjitlink\u002Flib:${LD_LIBRARY_PATH}\nexport PYTHONPATH=$(pwd):${PYTHONPATH}\nexport OPENAI_API_KEY=Your OpenAI Key. You need to set it when you choose the infer type as OpenAI.\nexport OPENAI_API_BASE=Your OpenAI base url\nexport GOOGLE_API_KEY=Your Google Cloud key. you need to set it when you choose the infer type as Google.\nexport SERP_API_KEY= Get SERP API key from https:\u002F\u002Fserpapi.com\n```\n\nWe provide both English and Chinese version of prompt. Default version is English. If you wish to use Chinese version, please set this env:\n``` bash\nexport PROMPT_LANGUAGE=\"zh\"\n```\n\n## Model Set\n⚠️ We strongly recommend using the Gemini Flash models. There may be unknown errors when using any other models. This project has high requirements for API consumption and concurrent volume, so it's not recommended to use locally deployed models.\n\nThe models used in the generation process are configured in the `.\u002FLLMxMapReduce_V2\u002Fconfig\u002Fmodel_config.json` file. Currently, we support both the OpenAI API and the Google API. You can specify the API to be used in the `infer_type` key. Additionally, you need to specify the model name in the `model` key.\n\nMoreover, the crawling process also requires large language model (LLM) inference. You can make changes in a similar manner in the `.\u002FLLMxMapReduce_V2\u002Fsrc\u002Fstart_pipeline.py` file. \n\n## Start LLMxMapReduce_V2 pipeline\nFollow the instructions and generate a report. The generated Markdown file is at .\u002Foutput\u002Fmd. \n```bash\ncd LLMxMapReduce_V2\nbash scripts\u002Fpipeline_start.sh TOPIC output_file_path.jsonl\n```\n\nIf you wish to use your own data, you need to set the `--input_file` and don't set `--topic` in scripts.\n\nThe input data should have following components at least:\n```json\n{\n  \"title\": \"The article title you wish to write\",\n  \"papers\": [\n    {\n      \"title\": \"The material title\",\n      \"abstract\": \"The abstract material. Optional, if not, part of the full text will be excerpted\",\n      \"txt\": \"The reference material full content\"\n    }\n  ]\n}\n```\n\nYou could use to use [this script](LLMxMapReduce_V2\u002Fscripts\u002Foutput_to_md.py) to convert data from `.jsonl` to multiple `.md` files.\n\n# 📃 Evaluation\nThe following steps are about LLMxMapReduce-V2. If you want to use LLMxMapReduce-V1, you need to refer to [here](LLMxMapReduce_V1\u002FREADME.md).\n\nFollow the steps below to set up the evaluation:\n## 1. Download the Dataset\nBefore running the evaluation, you need to download the `test` split of [SurveyEval dataset](https:\u002F\u002Fhuggingface.co\u002Fdatasets\u002FR0k1e\u002FSurveyEval). After downloading, store it in a `.jsonl` file.\n\n## 2. Run the Evaluation\nExecute the [scripts](LLMxMapReduce_V2\u002Fscripts\u002Feval_all.sh) to evaluate the generated result. \n```bash\ncd LLMxMapReduce_V2\nbash scripts\u002Feval_all.sh output_data_file_path.jsonl\n```\nAware that the evaluation process is token-consuming, you need to make sure you have enough balance.\n\n# 📊 Experiment Results\nOur experiments demonstrate the improved performance of LLM using the LLMxMapReduce-V2 framework on SurveyEval. Detailed results are provided below.\n\n| **Methods**           | **Struct.** | **Fait.** | **Rele.** | **Lang.** | **Crit.** | **Num.** | **Dens.** | **Prec.** | **Recall** |\n|-----------------------|-------------|-----------|-----------|-----------|-----------|----------|-----------|-----------|------------|\n| Vanilla               | 94.44       | 96.43     | **100.00**| **96.50** | 37.11     | 78.75    | **74.64** | 25.48     | 26.46      |\n| + Skeleton            | **98.95**   | **97.03** | **100.00**| 95.95     | **41.01** | **135.15**| 72.96     | **62.60** | **65.11**  |\n| AutoSurvey            | 86.00       | 93.10     | **100.00**| 92.90     | 68.39     | 423.35   | 31.97     | 50.12     | 51.73      |\n| LLMxMapReduce_V2       | **95.00**   | **97.22** | **100.00**| **94.34** | **71.99** | **474.90**| **52.23** | **95.50** | **95.80**  |\n\n# 📝 Citation\nIf you have used the content of this repository, please cite the paper and leave your star :).\n\n```\n@misc{wang2025llmtimesmapreducev2entropydrivenconvolutionaltesttime,\n      title={LLM$\\times$MapReduce-V2: Entropy-Driven Convolutional Test-Time Scaling for Generating Long-Form Articles from Extremely Long Resources}, \n      author={Haoyu Wang and Yujia Fu and Zhu Zhang and Shuo Wang and Zirui Ren and Xiaorong Wang and Zhili Li and Chaoqun He and Bo An and Zhiyuan Liu and Maosong Sun},\n      year={2025},\n      eprint={2504.05732},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https:\u002F\u002Farxiv.org\u002Fabs\u002F2504.05732}, \n}\n\n@misc{zhou2024llmtimesmapreducesimplifiedlongsequenceprocessing,\n      title={LLM$\\times$MapReduce: Simplified Long-Sequence Processing using Large Language Models}, \n      author={Zihan Zhou and Chong Li and Xinyi Chen and Shuo Wang and Yu Chao and Zhili Li and Haoyu Wang and Rongqiao An and Qi Shi and Zhixing Tan and Xu Han and Xiaodong Shi and Zhiyuan Liu and Maosong Sun},\n      year={2024},\n      eprint={2410.09342},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https:\u002F\u002Farxiv.org\u002Fabs\u002F2410.09342}, \n}\n```\n\n\n\n","# LLMxMapReduce：使用大语言模型简化长序列处理\n\n\u003Ca href='https:\u002F\u002Fsurveygo.modelbest.cn\u002F'>\u003Cimg src='https:\u002F\u002Fimg.shields.io\u002Fbadge\u002FDemo-Page-pink'>\u003C\u002Fa> \u003Ca href='https:\u002F\u002Farxiv.org\u002Fabs\u002F2410.09342'>\u003Cimg src='https:\u002F\u002Fimg.shields.io\u002Fbadge\u002FV1-Paper-Green'>\u003C\u002Fa> \u003Ca href='https:\u002F\u002Farxiv.org\u002Fabs\u002F2504.05732'>\u003Cimg src='https:\u002F\u002Fimg.shields.io\u002Fbadge\u002FV2-Paper-blue'>\u003C\u002Fa> \u003Ca href='https:\u002F\u002Fhuggingface.co\u002Fdatasets\u002FR0k1e\u002FSurveyEval'>\u003Cimg src='https:\u002F\u002Fimg.shields.io\u002Fbadge\u002FSurveyEval-Benchmark-yellow'>\u003C\u002Fa> \u003Ca href='README_zh.md'>\u003Cimg src='https:\u002F\u002Fimg.shields.io\u002Fbadge\u002FChinese-Readme-red'>\u003C\u002Fa>\n\n# 🎉 新闻\n- [x] **`2025.09.13`** 发布 LLMxMapReduce-V3 代码。\n- [x] **`2025.04.22`** 发布由 LLMxMapReduce-V2 驱动的在线写作系统 [SurveyGO](https:\u002F\u002Fsurveygo.znyx.com.cn\u002F)。\n- [x] **`2025.04.09`** 在 [arXiv](https:\u002F\u002Farxiv.org\u002Fabs\u002F2504.05732) 上发布 LLMxMapReduce-V2 论文。\n- [x] **`2025.02.21`** 新增支持 OpenAI API 及兼容 OpenAI 的 API（如 vLLM）。\n- [x] **`2024.10.12`** 在 [arXiv](https:\u002F\u002Farxiv.org\u002Fabs\u002F2410.09342) 上发布 LLMxMapReduce-V1 论文。\n- [x] **`2024.09.12`** 发布 LLMxMapReduce-V1 代码。\n\n# 📚 概览\n**LLMxMapReduce** 是一种分而治之（divide-and-conquer）框架，旨在增强现代大语言模型（Large Language Models, LLMs）对长序列的理解与生成能力。该框架由 **AI9STARS**、**OpenBMB** 和 **THUNLP** 联合开发，灵感源自大数据领域经典的 MapReduce 算法。我们的目标是构建一个由 LLM 驱动的分布式计算系统，以高效处理长序列。以下是 LLMxMapReduce 的主要版本：\n\n* [**LLMxMapReduce-V1**](https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Fblob\u002Fmain\u002FLLMxMapReduce_V1)：采用结构化信息协议和上下文内置信度校准，提升长序列理解能力，使 [MiniCPM3-4B](https:\u002F\u002Fgithub.com\u002FOpenBMB\u002FMiniCPM) 在长上下文评估中超越 70B 规模模型。\n* [**LLMxMapReduce-V2**](https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Ftree\u002Fmain\u002FLLMxMapReduce_V2)：引入熵驱动的卷积式测试时缩放机制，以更好地整合海量信息，为在线系统 [SurveyGO](https:\u002F\u002Fsurveygo.modelbest.cn\u002F) 提供动力。\n* [**LLMxMapReduce-V3**](https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Fblob\u002Fmain\u002FLLMxMapReduce_V3)：引入交互式、模块化、自组织的多智能体系统，用于学术综述生成。\n\n# 📖 简介\n\n长文本生成在众多实际应用中至关重要，通常可分为“短到长”和“长到长”两类生成任务。尽管“短到长”生成已受到广泛关注，“从极长资源生成长文本”的“长到长”任务仍相对未被充分探索。其核心挑战在于如何有效整合并分析海量输入中的相关信息，这对当前的大语言模型（LLMs）而言仍十分困难。本文提出 LLMxMapReduce-V2，一种新颖的测试时缩放策略，旨在增强 LLM 处理极长输入的能力。受卷积神经网络（CNN）启发——CNN 通过迭代将局部特征融合为更高层次的全局表征——LLMxMapReduce-V2 利用堆叠的卷积缩放层逐步扩展对输入材料的理解。定量与定性实验结果均表明，本方法显著提升了 LLM 处理长输入并生成连贯、信息丰富的长文的能力，优于多个代表性基线模型。\n\n\u003Cdiv align=\"center\">\n  \u003Cimg src=\"https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Fthunlp_LLMxMapReduce_readme_ccc5cd7b79b1.jpg\" alt=\"$\\text{LLM}\\times \\text{MapReduce}$-V2 framework\">\n\u003C\u002Fdiv>\n\n# ⚡️ 快速开始\n\n以下步骤针对 LLMxMapReduce-V2。若需使用 LLMxMapReduce-V1，请参阅 [此处](LLMxMapReduce_V1\u002FREADME.md)；若需使用 LLMxMapReduce-V3，请参阅 [此处](LLMxMapReduce_V3\u002FREADME.md)。\n\n首先，请确保安装 requirements.txt 中列出的所有依赖项：\n```bash\ncd LLMxMapReduce_V2\nconda create -n llm_mr_v2 python=3.11\nconda activate llm_mr_v2\npip install -r requirements.txt\npython -m playwright install --with-deps chromium\n```\n\n在评估前，需先下载 punkt_tab：\n```python\nimport nltk\nnltk.download('punkt_tab')\n```\n\n## 环境配置\n\n启动 pipeline 前，请在环境变量中设置 OPENAI_API_KEY 和 OPENAI_API_BASE。若使用 miniconda，请将 `LD_LIBRARY_PATH` 中的 `anaconda3` 替换为 `miniconda3`：\n```bash\nexport LD_LIBRARY_PATH=${HOME}\u002Fanaconda3\u002Fenvs\u002Fllm_mr_v2\u002Flib\u002Fpython3.11\u002Fsite-packages\u002Fnvidia\u002Fnvjitlink\u002Flib:${LD_LIBRARY_PATH}\nexport PYTHONPATH=$(pwd):${PYTHONPATH}\nexport OPENAI_API_KEY=您的 OpenAI 密钥。仅当 infer_type 设置为 OpenAI 时需要。\nexport OPENAI_API_BASE=您的 OpenAI 基础 URL\nexport GOOGLE_API_KEY=您的 Google Cloud 密钥。仅当 infer_type 设置为 Google 时需要。\nexport SERP_API_KEY= 从 https:\u002F\u002Fserpapi.com 获取 SERP API 密钥\n```\n\n我们提供英文和中文版本的提示词（prompt），默认为英文。如需使用中文版，请设置如下环境变量：\n``` bash\nexport PROMPT_LANGUAGE=\"zh\"\n```\n\n## 模型设置\n\n⚠️ 我们强烈推荐使用 Gemini Flash 系列模型。使用其他模型可能会出现未知错误。本项目对 API 调用量和并发量要求较高，不建议使用本地部署模型。\n\n生成过程中使用的模型配置位于 `.\u002FLLMxMapReduce_V2\u002Fconfig\u002Fmodel_config.json` 文件中。目前支持 OpenAI API 和 Google API，可通过 `infer_type` 键指定所用 API，并在 `model` 键中指定模型名称。\n\n此外，爬取过程同样需要大语言模型（LLM）推理，可在 `.\u002FLLMxMapReduce_V2\u002Fsrc\u002Fstart_pipeline.py` 文件中进行类似配置。\n\n## 启动 LLMxMapReduce_V2 流程\n\n按照说明生成报告，生成的 Markdown 文件位于 .\u002Foutput\u002Fmd 目录下：\n```bash\ncd LLMxMapReduce_V2\nbash scripts\u002Fpipeline_start.sh TOPIC output_file_path.jsonl\n```\n\n如需使用自定义数据，请设置 `--input_file` 参数，且不要设置 `--topic`。\n\n输入数据至少应包含以下字段：\n```json\n{\n  \"title\": \"您希望撰写的论文标题\",\n  \"papers\": [\n    {\n      \"title\": \"参考材料标题\",\n      \"abstract\": \"摘要内容（可选，若无则会截取部分正文）\",\n      \"txt\": \"参考材料全文\"\n    }\n  ]\n}\n```\n\n您可以使用 [此脚本](LLMxMapReduce_V2\u002Fscripts\u002Foutput_to_md.py) 将 `.jsonl` 数据转换为多个 `.md` 文件。\n\n# 📃 评估\n以下步骤适用于 LLMxMapReduce-V2。如需使用 LLMxMapReduce-V1，请参考 [此处](LLMxMapReduce_V1\u002FREADME.md)。\n\n请按以下步骤设置评估环境：\n\n## 1. 下载数据集\n在运行评估前，您需要下载 [SurveyEval 数据集](https:\u002F\u002Fhuggingface.co\u002Fdatasets\u002FR0k1e\u002FSurveyEval) 的 `test` 分割部分，并将其保存为 `.jsonl` 文件。\n\n## 2. 运行评估\n执行 [脚本](LLMxMapReduce_V2\u002Fscripts\u002Feval_all.sh) 来评估生成结果。\n```bash\ncd LLMxMapReduce_V2\nbash scripts\u002Feval_all.sh output_data_file_path.jsonl\n```\n请注意，评估过程消耗大量 token，请确保您的账户余额充足。\n\n# 📊 实验结果\n我们的实验表明，在 SurveyEval 上使用 LLMxMapReduce-V2 框架可显著提升大语言模型（LLM, Large Language Model）的性能。详细结果如下所示。\n\n| **方法**              | **结构**    | **事实性** | **相关性** | **语言**  | **批判性** | **数量** | **密度**  | **精确率** | **召回率** |\n|-----------------------|-------------|-----------|-----------|-----------|-----------|----------|-----------|-----------|------------|\n| Vanilla               | 94.44       | 96.43     | **100.00**| **96.50** | 37.11     | 78.75    | **74.64** | 25.48     | 26.46      |\n| + Skeleton            | **98.95**   | **97.03** | **100.00**| 95.95     | **41.01** | **135.15**| 72.96     | **62.60** | **65.11**  |\n| AutoSurvey            | 86.00       | 93.10     | **100.00**| 92.90     | 68.39     | 423.35   | 31.97     | 50.12     | 51.73      |\n| LLMxMapReduce_V2       | **95.00**   | **97.22** | **100.00**| **94.34** | **71.99** | **474.90**| **52.23** | **95.50** | **95.80**  |\n\n# 📝 引用\n如果您使用了本仓库的内容，请引用我们的论文并点个 Star :)。\n\n```\n@misc{wang2025llmtimesmapreducev2entropydrivenconvolutionaltesttime,\n      title={LLM$\\times$MapReduce-V2: Entropy-Driven Convolutional Test-Time Scaling for Generating Long-Form Articles from Extremely Long Resources}, \n      author={Haoyu Wang and Yujia Fu and Zhu Zhang and Shuo Wang and Zirui Ren and Xiaorong Wang and Zhili Li and Chaoqun He and Bo An and Zhiyuan Liu and Maosong Sun},\n      year={2025},\n      eprint={2504.05732},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https:\u002F\u002Farxiv.org\u002Fabs\u002F2504.05732}, \n}\n\n@misc{zhou2024llmtimesmapreducesimplifiedlongsequenceprocessing,\n      title={LLM$\\times$MapReduce: Simplified Long-Sequence Processing using Large Language Models}, \n      author={Zihan Zhou and Chong Li and Xinyi Chen and Shuo Wang and Yu Chao and Zhili Li and Haoyu Wang and Rongqiao An and Qi Shi and Zhixing Tan and Xu Han and Xiaodong Shi and Zhiyuan Liu and Maosong Sun},\n      year={2024},\n      eprint={2410.09342},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https:\u002F\u002Farxiv.org\u002Fabs\u002F2410.09342}, \n}\n```","# LLMxMapReduce 快速上手指南（中文版）\n\n---\n\n## 🧰 环境准备\n\n- **系统要求**：Linux 或 macOS（推荐 Ubuntu 20.04+）\n- **Python 版本**：3.11\n- **依赖管理工具**：推荐使用 `conda` 创建独立环境\n- **API 要求**：\n  - OpenAI API 或兼容接口（如 vLLM）\n  - Google Gemini API（强烈推荐使用 Gemini Flash 模型）\n  - SERP API（用于网络搜索，可选）\n- **国内加速建议**：\n  - 使用清华\u002F阿里云 pip 镜像加速安装：`pip install -i https:\u002F\u002Fpypi.tuna.tsinghua.edu.cn\u002Fsimple -r requirements.txt`\n  - 若使用 Hugging Face 数据集，可配置 HF_ENDPOINT 加速下载：`export HF_ENDPOINT=https:\u002F\u002Fhf-mirror.com`\n\n---\n\n## ⚙️ 安装步骤\n\n```bash\n# 克隆项目（若未克隆）\ngit clone https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce.git\ncd LLMxMapReduce\n\n# 进入 V2 目录（默认推荐版本）\ncd LLMxMapReduce_V2\n\n# 创建并激活 conda 环境\nconda create -n llm_mr_v2 python=3.11\nconda activate llm_mr_v2\n\n# 安装依赖（推荐使用国内镜像）\npip install -i https:\u002F\u002Fpypi.tuna.tsinghua.edu.cn\u002Fsimple -r requirements.txt\n\n# 安装浏览器驱动（用于网页抓取）\npython -m playwright install --with-deps chromium\n\n# 下载 NLTK 分词数据\npython -c \"import nltk; nltk.download('punkt_tab')\"\n```\n\n---\n\n## 🚀 基本使用\n\n### 1. 配置环境变量\n\n```bash\n# 设置 API 密钥（根据你使用的模型平台）\nexport OPENAI_API_KEY=你的OpenAI密钥\nexport OPENAI_API_BASE=https:\u002F\u002Fapi.openai.com\u002Fv1  # 或你的兼容API地址\n\n# 或使用 Google Gemini（推荐）\nexport GOOGLE_API_KEY=你的GoogleCloud密钥\n\n# 可选：设置中文提示词\nexport PROMPT_LANGUAGE=\"zh\"\n\n# 设置路径（miniconda用户请将 anaconda3 替换为 miniconda3）\nexport LD_LIBRARY_PATH=${HOME}\u002Fanaconda3\u002Fenvs\u002Fllm_mr_v2\u002Flib\u002Fpython3.11\u002Fsite-packages\u002Fnvidia\u002Fnvjitlink\u002Flib:${LD_LIBRARY_PATH}\nexport PYTHONPATH=$(pwd):${PYTHONPATH}\n```\n\n> 💡 推荐优先使用 `GOOGLE_API_KEY` + Gemini Flash 模型，稳定性更佳。\n\n### 2. 启动生成流程\n\n```bash\n# 生成指定主题的长文报告（输出为 .jsonl 和 .md）\nbash scripts\u002Fpipeline_start.sh \"人工智能最新进展\" .\u002Foutput\u002Fresult.jsonl\n```\n\n生成的 Markdown 文件位于：`.\u002Foutput\u002Fmd\u002F`\n\n### 3. 使用自定义输入数据\n\n准备 JSONL 格式文件，结构如下：\n\n```json\n{\n  \"title\": \"你想写的标题\",\n  \"papers\": [\n    {\n      \"title\": \"参考文献标题\",\n      \"abstract\": \"摘要（可选）\",\n      \"txt\": \"全文内容\"\n    }\n  ]\n}\n```\n\n运行命令：\n\n```bash\nbash scripts\u002Fpipeline_start.sh --input_file your_input.jsonl\n```\n\n---\n\n✅ 至此，你已成功运行 LLMxMapReduce-V2！生成结果可在 `.\u002Foutput\u002Fmd\u002F` 查看。如需评估效果，请参考项目内 `eval_all.sh` 脚本。","某高校科研团队正在撰写一篇关于“生成式AI在医疗影像诊断中的最新进展”的综述论文，需从近300篇顶会论文和临床报告中提取关键结论并整合成结构化长文。\n\n### 没有 LLMxMapReduce 时\n- 研究人员手动阅读上百篇PDF文献，耗时超过两周，且容易遗漏关键段落或重复内容。\n- 使用普通LLM处理整批文献时，因上下文长度限制（如32K tokens），模型频繁截断输入，导致分析碎片化、逻辑断裂。\n- 多人协作时各自总结不同章节，最终整合常出现观点冲突或结构混乱，需反复人工校对。\n- 输出结果缺乏层次感，重要发现被淹没在冗长文本中，审稿人反馈“信息密度低、主线不清晰”。\n- 遇到新加入的50篇补充材料时，必须从头重跑整个流程，迭代效率极低。\n\n### 使用 LLMxMapReduce 后\n- LLMxMapReduce自动将300篇文献按主题分块（Map阶段），由多个轻量级LLM并行提取摘要，再通过卷积式融合层（Reduce阶段）逐层提炼核心脉络，总耗时压缩至3天内。\n- 借助其“熵驱动的卷积缩放机制”，即使输入总量超百万tokens，也能动态聚焦高信息密度段落，输出连贯、无截断的完整综述框架。\n- 自动生成带层级标题与证据标注的初稿，团队只需微调重点章节，协作效率提升70%，版本冲突归零。\n- 最终输出具备学术论文标准结构（背景→方法分类→临床验证→争议点→未来方向），关键结论加粗+引用溯源，直接满足期刊格式要求。\n- 新增文献可单独注入Map阶段，系统自动增量更新全局摘要，无需全量重算，支持敏捷迭代。\n\nLLMxMapReduce让研究者从“文献搬运工”升级为“知识架构师”，用分布式思维突破LLM长文本处理的物理极限。","https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Fthunlp_LLMxMapReduce_00c362e6.png","thunlp","THUNLP","https:\u002F\u002Foss.gittoolsai.com\u002Favatars\u002Fthunlp_13da7b2e.png","Natural Language Processing Lab at Tsinghua University","thunlp@gmail.com","http:\u002F\u002Fnlp.csai.tsinghua.edu.cn","https:\u002F\u002Fgithub.com\u002Fthunlp",[83,87],{"name":84,"color":85,"percentage":86},"Python","#3572A5",99.2,{"name":88,"color":89,"percentage":90},"Shell","#89e051",0.8,869,62,"2026-04-01T14:14:19","Apache-2.0","Linux, macOS","未说明",{"notes":98,"python":99,"dependencies":100},"需配置 OpenAI 或 Google API 密钥，推荐使用 Gemini Flash 模型；不建议使用本地部署模型；首次运行需下载 nltk 的 punkt_tab 数据包。","3.11",[101,102,103,104,105],"playwright","nltk","openai","google-generativeai","serpapi",[26,13,15],4,"2026-03-27T02:49:30.150509","2026-04-06T06:52:05.116894",[111,116,121,126,131,136,141],{"id":112,"question_zh":113,"answer_zh":114,"source_url":115},325,"为什么本地运行效果远不如线上服务？","线上服务使用的是特定模型（如Gemini系列），而Qwen3等本地模型可能因能力不足导致质量差异。建议改用Gemini系列模型，并检查是否因模型推理能力不足或上下文窗口限制导致结果不佳。","https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Fissues\u002F46",{"id":117,"question_zh":118,"answer_zh":119,"source_url":120},326,"Pipeline卡在“Running”状态且Exec进度为0，如何解决？","1. 使用WSL2或Ubuntu系统，避免Windows下的编码问题；2. 使用作者推荐的模型（如Gemini），DeepSeek可能报错但仍可跑通；3. Gemini官方API易触发rate limits，建议使用国内代理；4. 检查日志确认是否成功爬取网页，失败会导致空转。","https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Fissues\u002F36",{"id":122,"question_zh":123,"answer_zh":124,"source_url":125},327,"运行到1-4阶段后长时间停滞不动，是什么原因？","可能是模型能力不足导致超出重试上限而失败。推荐使用Gemini系列模型（如gemini-2.0-flash），若仍失败请提供完整日志以便进一步排查。","https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Fissues\u002F28",{"id":127,"question_zh":128,"answer_zh":129,"source_url":130},328,"如何自定义使用其他大模型（如GPT、DeepSeek）？","支持自定义模型。若通过vLLM以OpenAI兼容API启动，需在配置中填写对应模型名称，设置infer_type为OpenAI，并修改环境变量OPENAI_API_BASE。建议选择推理能力强、上下文窗口大的模型。","https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Fissues\u002F12",{"id":132,"question_zh":133,"answer_zh":134,"source_url":135},329,"本地运行时无法生成流程图（flow chart），如何解决？","可能是所用模型返回格式不符合规范，导致正则匹配失败。建议更换更强力的模型（如Gemini系列）尝试，确保输出包含\u003Cfigure>标签和mermaid语法。","https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Fissues\u002F48",{"id":137,"question_zh":138,"answer_zh":139,"source_url":140},330,"如何配置搜索源（如arXiv）并指定输出语言？","目前代码中arXiv爬虫尚未开源，暂不支持自定义绑定搜索源与输出语言。Web端功能受限，需等待后续开源更新。","https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Fissues\u002F22",{"id":142,"question_zh":143,"answer_zh":144,"source_url":145},331,"使用本地JSONL文件时Pipeline卡住无报错，怎么办？","即使终端无报错，也请仔细检查日志中是否有Traceback或数据错误。数据异常可能导致Pipeline空转。建议提供完整日志或加入飞书用户群获取支持。","https:\u002F\u002Fgithub.com\u002Fthunlp\u002FLLMxMapReduce\u002Fissues\u002F20",[]]