[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"similar-microsoft--LoRA":3,"tool-microsoft--LoRA":61},[4,18,26,36,44,53],{"id":5,"name":6,"github_repo":7,"description_zh":8,"stars":9,"difficulty_score":10,"last_commit_at":11,"category_tags":12,"status":17},4358,"openclaw","openclaw\u002Fopenclaw","OpenClaw 是一款专为个人打造的本地化 AI 助手，旨在让你在自己的设备上拥有完全可控的智能伙伴。它打破了传统 AI 助手局限于特定网页或应用的束缚，能够直接接入你日常使用的各类通讯渠道，包括微信、WhatsApp、Telegram、Discord、iMessage 等数十种平台。无论你在哪个聊天软件中发送消息，OpenClaw 都能即时响应，甚至支持在 macOS、iOS 和 Android 设备上进行语音交互，并提供实时的画布渲染功能供你操控。\n\n这款工具主要解决了用户对数据隐私、响应速度以及“始终在线”体验的需求。通过将 AI 部署在本地，用户无需依赖云端服务即可享受快速、私密的智能辅助，真正实现了“你的数据，你做主”。其独特的技术亮点在于强大的网关架构，将控制平面与核心助手分离，确保跨平台通信的流畅性与扩展性。\n\nOpenClaw 非常适合希望构建个性化工作流的技术爱好者、开发者，以及注重隐私保护且不愿被单一生态绑定的普通用户。只要具备基础的终端操作能力（支持 macOS、Linux 及 Windows WSL2），即可通过简单的命令行引导完成部署。如果你渴望拥有一个懂你",349277,3,"2026-04-06T06:32:30",[13,14,15,16],"Agent","开发框架","图像","数据工具","ready",{"id":19,"name":20,"github_repo":21,"description_zh":22,"stars":23,"difficulty_score":10,"last_commit_at":24,"category_tags":25,"status":17},3808,"stable-diffusion-webui","AUTOMATIC1111\u002Fstable-diffusion-webui","stable-diffusion-webui 是一个基于 Gradio 构建的网页版操作界面，旨在让用户能够轻松地在本地运行和使用强大的 Stable Diffusion 图像生成模型。它解决了原始模型依赖命令行、操作门槛高且功能分散的痛点，将复杂的 AI 绘图流程整合进一个直观易用的图形化平台。\n\n无论是希望快速上手的普通创作者、需要精细控制画面细节的设计师，还是想要深入探索模型潜力的开发者与研究人员，都能从中获益。其核心亮点在于极高的功能丰富度：不仅支持文生图、图生图、局部重绘（Inpainting）和外绘（Outpainting）等基础模式，还独创了注意力机制调整、提示词矩阵、负向提示词以及“高清修复”等高级功能。此外，它内置了 GFPGAN 和 CodeFormer 等人脸修复工具，支持多种神经网络放大算法，并允许用户通过插件系统无限扩展能力。即使是显存有限的设备，stable-diffusion-webui 也提供了相应的优化选项，让高质量的 AI 艺术创作变得触手可及。",162132,"2026-04-05T11:01:52",[14,15,13],{"id":27,"name":28,"github_repo":29,"description_zh":30,"stars":31,"difficulty_score":32,"last_commit_at":33,"category_tags":34,"status":17},1381,"everything-claude-code","affaan-m\u002Feverything-claude-code","everything-claude-code 是一套专为 AI 编程助手（如 Claude Code、Codex、Cursor 等）打造的高性能优化系统。它不仅仅是一组配置文件，而是一个经过长期实战打磨的完整框架，旨在解决 AI 代理在实际开发中面临的效率低下、记忆丢失、安全隐患及缺乏持续学习能力等核心痛点。\n\n通过引入技能模块化、直觉增强、记忆持久化机制以及内置的安全扫描功能，everything-claude-code 能显著提升 AI 在复杂任务中的表现，帮助开发者构建更稳定、更智能的生产级 AI 代理。其独特的“研究优先”开发理念和针对 Token 消耗的优化策略，使得模型响应更快、成本更低，同时有效防御潜在的攻击向量。\n\n这套工具特别适合软件开发者、AI 研究人员以及希望深度定制 AI 工作流的技术团队使用。无论您是在构建大型代码库，还是需要 AI 协助进行安全审计与自动化测试，everything-claude-code 都能提供强大的底层支持。作为一个曾荣获 Anthropic 黑客大奖的开源项目，它融合了多语言支持与丰富的实战钩子（hooks），让 AI 真正成长为懂上",157379,2,"2026-04-15T23:32:42",[14,13,35],"语言模型",{"id":37,"name":38,"github_repo":39,"description_zh":40,"stars":41,"difficulty_score":32,"last_commit_at":42,"category_tags":43,"status":17},2271,"ComfyUI","Comfy-Org\u002FComfyUI","ComfyUI 是一款功能强大且高度模块化的视觉 AI 引擎，专为设计和执行复杂的 Stable Diffusion 图像生成流程而打造。它摒弃了传统的代码编写模式，采用直观的节点式流程图界面，让用户通过连接不同的功能模块即可构建个性化的生成管线。\n\n这一设计巧妙解决了高级 AI 绘图工作流配置复杂、灵活性不足的痛点。用户无需具备编程背景，也能自由组合模型、调整参数并实时预览效果，轻松实现从基础文生图到多步骤高清修复等各类复杂任务。ComfyUI 拥有极佳的兼容性，不仅支持 Windows、macOS 和 Linux 全平台，还广泛适配 NVIDIA、AMD、Intel 及苹果 Silicon 等多种硬件架构，并率先支持 SDXL、Flux、SD3 等前沿模型。\n\n无论是希望深入探索算法潜力的研究人员和开发者，还是追求极致创作自由度的设计师与资深 AI 绘画爱好者，ComfyUI 都能提供强大的支持。其独特的模块化架构允许社区不断扩展新功能，使其成为当前最灵活、生态最丰富的开源扩散模型工具之一，帮助用户将创意高效转化为现实。",108322,"2026-04-10T11:39:34",[14,15,13],{"id":45,"name":46,"github_repo":47,"description_zh":48,"stars":49,"difficulty_score":32,"last_commit_at":50,"category_tags":51,"status":17},6121,"gemini-cli","google-gemini\u002Fgemini-cli","gemini-cli 是一款由谷歌推出的开源 AI 命令行工具，它将强大的 Gemini 大模型能力直接集成到用户的终端环境中。对于习惯在命令行工作的开发者而言，它提供了一条从输入提示词到获取模型响应的最短路径，无需切换窗口即可享受智能辅助。\n\n这款工具主要解决了开发过程中频繁上下文切换的痛点，让用户能在熟悉的终端界面内直接完成代码理解、生成、调试以及自动化运维任务。无论是查询大型代码库、根据草图生成应用，还是执行复杂的 Git 操作，gemini-cli 都能通过自然语言指令高效处理。\n\n它特别适合广大软件工程师、DevOps 人员及技术研究人员使用。其核心亮点包括支持高达 100 万 token 的超长上下文窗口，具备出色的逻辑推理能力；内置 Google 搜索、文件操作及 Shell 命令执行等实用工具；更独特的是，它支持 MCP（模型上下文协议），允许用户灵活扩展自定义集成，连接如图像生成等外部能力。此外，个人谷歌账号即可享受免费的额度支持，且项目基于 Apache 2.0 协议完全开源，是提升终端工作效率的理想助手。",100752,"2026-04-10T01:20:03",[52,13,15,14],"插件",{"id":54,"name":55,"github_repo":56,"description_zh":57,"stars":58,"difficulty_score":32,"last_commit_at":59,"category_tags":60,"status":17},4721,"markitdown","microsoft\u002Fmarkitdown","MarkItDown 是一款由微软 AutoGen 团队打造的轻量级 Python 工具，专为将各类文件高效转换为 Markdown 格式而设计。它支持 PDF、Word、Excel、PPT、图片（含 OCR）、音频（含语音转录）、HTML 乃至 YouTube 链接等多种格式的解析，能够精准提取文档中的标题、列表、表格和链接等关键结构信息。\n\n在人工智能应用日益普及的今天，大语言模型（LLM）虽擅长处理文本，却难以直接读取复杂的二进制办公文档。MarkItDown 恰好解决了这一痛点，它将非结构化或半结构化的文件转化为模型“原生理解”且 Token 效率极高的 Markdown 格式，成为连接本地文件与 AI 分析 pipeline 的理想桥梁。此外，它还提供了 MCP（模型上下文协议）服务器，可无缝集成到 Claude Desktop 等 LLM 应用中。\n\n这款工具特别适合开发者、数据科学家及 AI 研究人员使用，尤其是那些需要构建文档检索增强生成（RAG）系统、进行批量文本分析或希望让 AI 助手直接“阅读”本地文件的用户。虽然生成的内容也具备一定可读性，但其核心优势在于为机器",93400,"2026-04-06T19:52:38",[52,14],{"id":62,"github_repo":63,"name":64,"description_en":65,"description_zh":66,"ai_summary_zh":66,"readme_en":67,"readme_zh":68,"quickstart_zh":69,"use_case_zh":70,"hero_image_url":71,"owner_login":72,"owner_name":73,"owner_avatar_url":74,"owner_bio":75,"owner_company":76,"owner_location":76,"owner_email":77,"owner_twitter":78,"owner_website":79,"owner_url":80,"languages":81,"stars":86,"forks":87,"last_commit_at":88,"license":89,"difficulty_score":32,"env_os":90,"env_gpu":91,"env_ram":91,"env_deps":92,"category_tags":97,"github_topics":98,"view_count":32,"oss_zip_url":76,"oss_zip_packed_at":76,"status":17,"created_at":109,"updated_at":110,"faqs":111,"releases":112},8016,"microsoft\u002FLoRA","LoRA","Code for loralib, an implementation of \"LoRA: Low-Rank Adaptation of Large Language Models\"","LoRA（低秩适应）是一种专为大型语言模型设计的高效微调技术。它旨在解决传统全量微调方法在训练大模型时面临的巨大挑战：需要更新海量参数，导致存储成本高昂、任务切换困难且计算资源消耗巨大。\n\nLoRA 的核心思路是“冻结”预训练模型的原始权重，仅通过训练一对低秩分解矩阵来学习特定任务的适配信息。这种独特的技术亮点在于，它能将可训练参数量减少数个数量级（例如从数十亿降至数百万），却能在 GLUE 等基准测试中取得与全量微调相当甚至更优的性能。此外，由于不改变原始模型结构，LoRA 在推理阶段不会引入任何额外延迟，并允许用户在不同任务间快速切换，只需加载微小的适配权重即可。\n\n这项技术非常适合 AI 研究人员、开发者以及希望在大模型基础上进行定制化应用的企业团队。无论是学术探索还是工业部署，LoRA 都能让用户以更低的算力门槛和存储成本，高效地将通用大模型转化为解决特定领域问题的专家模型。目前，LoRA 已被集成到 Hugging Face 的 PEFT 库中，成为业界主流的参数高效微调方案之一。","# LoRA: Low-Rank Adaptation of Large Language Models\n\nThis repo contains the source code of the Python package `loralib` and several examples of how to integrate it with PyTorch models, such as those in Hugging Face.\nWe only support PyTorch for now.\nSee our paper for a detailed description of LoRA.\n\n**LoRA: Low-Rank Adaptation of Large Language Models** \u003Cbr>\n*Edward J. Hu\\*, Yelong Shen\\*, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen* \u003Cbr>\nPaper: https:\u002F\u002Farxiv.org\u002Fabs\u002F2106.09685 \u003Cbr>\nVideo explainer: https:\u002F\u002Fwww.youtube.com\u002Fwatch?v=DhRoTONcyZE \u003Cbr>\n\n*Update 2\u002F2023: LoRA is now supported by the [State-of-the-art Parameter-Efficient Fine-Tuning (PEFT)](https:\u002F\u002Fgithub.com\u002Fhuggingface\u002Fpeft) library by Hugging Face.*\n\nLoRA reduces the number of trainable parameters by learning pairs of rank-decompostion matrices while freezing the original weights.\nThis vastly reduces the storage requirement for large language models adapted to specific tasks and enables efficient task-switching during deployment all without introducing inference latency.\nLoRA also outperforms several other adaptation methods including adapter, prefix-tuning, and fine-tuning.\n\nWe obtain result comparable or superior to full finetuning on the GLUE benchmark using [RoBERTa (Liu et al., 2019)](https:\u002F\u002Farxiv.org\u002Fabs\u002F1907.11692) base and large and [DeBERTa (He et al., 2020)](https:\u002F\u002Farxiv.org\u002Fabs\u002F2006.03654) XXL 1.5B, while only training and storing a fraction of the parameters. Click the numbers below to download the RoBERTa and DeBERTa LoRA checkpoints.\n\n|   |         | RoBERTa base \u003Cbr> Fine-tune  |  RoBERTa base \u003Cbr> LoRA  | DeBERTa XXL \u003Cbr> Fine-tune | DeBERTa XXL \u003Cbr> LoRA  |\n|---|-------------------------|----------------|--------------------------|-----------------|-----------------|\n|   | # of Trainable Params.  | 125M | 0.8M | 1.5B | 4.7M     |\n|   | MNLI (m-Acc\u002Fmm-Acc)     | \u003Cb>87.6\u003C\u002Fb> | [\u003Cb>87.5\u003C\u002Fb>±.3\u002F86.9±.3](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_mnli.bin) |91.7\u002F\u003Cb>91.9\u003C\u002Fb>| [\u003Cb>91.9\u003C\u002Fb>±.1\u002F\u003Cb>91.9\u003C\u002Fb>±.2](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_mnli.bin)       |\n|   | SST2 (Acc)              | 94.8 | [\u003Cb>95.1\u003C\u002Fb>±.2](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_sst2.bin) | \u003Cb>97.2\u003C\u002Fb>    | [96.9±.2](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_sst2.bin)                    |\n|   | MRPC (Acc)              | \u003Cb>90.2\u003C\u002Fb> | [\u003Cb>89.7\u003C\u002Fb>±.7](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_mrpc.bin) | 92.0           | [\u003Cb>92.6\u003C\u002Fb>±.6](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_mrpc.bin)             |\n|   | CoLA (Matthew's Corr)   | \u003Cb>63.6\u003C\u002Fb> | [\u003Cb>63.4\u003C\u002Fb>±1.2](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_cola.bin) | \u003Cb>72.0\u003C\u002Fb>    | [\u003Cb>72.4\u003C\u002Fb>±1.1](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_cola.bin)           |\n|   | QNLI (Acc)              | 92.8 | [\u003Cb>93.3\u003C\u002Fb>±.3](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_qnli.bin) | \u003Cb>96.0\u003C\u002Fb>    | [\u003Cb>96.0\u003C\u002Fb>±.1](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_qnli.bin)            |\n|   | QQP (Acc)               | \u003Cb>91.9\u003C\u002Fb> | [90.8±.1](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_qqp.bin) | 92.7           | [\u003Cb>92.9\u003C\u002Fb>±.1](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_qqp.bin)           |\n|   | RTE (Acc)               | 78.7 | [\u003Cb>86.6\u003C\u002Fb>±.7](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_rte.bin) | 93.9           | [\u003Cb>94.9\u003C\u002Fb>±.4](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_rte.bin)           |\n|   | STSB (Pearson\u002FSpearman Corr) | 91.2 | [\u003Cb>91.5\u003C\u002Fb>±.2\u002F\u003Cb>91.3\u003C\u002Fb>±.2](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_stsb.bin) |\u003Cb>92.9\u003C\u002Fb>\u002F92.6| [\u003Cb>93.0\u003C\u002Fb>±.2\u002F\u003Cb>92.9\u003C\u002Fb>±.3](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_stsb.bin)      |\n|   | Average  | 86.40 | \u003Cb>87.24\u003C\u002Fb> | 91.06 | \u003Cb>91.32\u003C\u002Fb> |\n\n\u003Ci>Note: You still need the original pre-trained checkpoint from [Hugging Face](https:\u002F\u002Fhuggingface.co\u002F) to use the LoRA checkpoints.\u003C\u002Fi>\n\nFine-tuning numbers are taken from [Liu et al. (2019)](https:\u002F\u002Farxiv.org\u002Fabs\u002F1907.11692) and [He et al. (2020)](https:\u002F\u002Farxiv.org\u002Fabs\u002F2006.03654).  We include confidence intervals on results from our experiments. Please follow the instructions in `examples\u002FNLU\u002F` to reproduce our results.\n\nOn GPT-2, LoRA compares favorably to both full finetuning and other efficient tuning methods, such as [adapter (Houlsby et al., 2019)](https:\u002F\u002Farxiv.org\u002Fabs\u002F1902.00751) and [prefix tuning (Li and Liang, 2021)](https:\u002F\u002Farxiv.org\u002Fabs\u002F2101.00190). We evaluated on E2E NLG Challenge, DART, and WebNLG:\n\n|   | Method              | # of Trainable Params | E2E (BLEU)   | DART (BLEU)  | WebNLG (BLEU-U\u002FS\u002FA)            |\n|---|---------------------|-----------------------|--------------|--------------|--------------------------------|\n|   | GPT-2 M (Fine-Tune) | 354.92M               | 68.2         | 46.0         | 30.4\u002F\u003Cb>63.2\u003C\u002Fb>\u002F47.6          |\n|   | GPT-2 M (Adapter)   | 0.37M                 | 66.3         | 42.4         | 45.1\u002F54.5\u002F50.2                 |\n|   | GPT-2 M (Prefix)    | 0.35M                 | 69.7         | 45.7         | 44.1\u002F63.1\u002F54.4                 |\n|   | GPT-2 M (LoRA)      | 0.35M                 |\u003Cb>70.4\u003C\u002Fb>±.1|\u003Cb>47.1\u003C\u002Fb>±.2| \u003Cb>46.7\u003C\u002Fb>±.4\u002F62.1±.2\u002F\u003Cb>55.3\u003C\u002Fb>±.2 |\n|   | GPT-2 L (Fine-Tune) | 774.03M               | 68.5         | 46.5         | 41.7\u002F\u003Cb>64.6\u003C\u002Fb>\u002F54.2          |\n|   | GPT-2 L (Adapter)   | 0.88M                 | 69.1±.1      | 45.7±.1      | \u003Cb>49.8\u003C\u002Fb>±.0\u002F61.1±.0\u002F56.0±.0 |\n|   | GPT-2 L (Prefix)    | 0.77M                 | 70.3         | 46.5         | 47.0\u002F64.2\u002F56.4                 |\n|   | GPT-2 L (LoRA)      | 0.77M                 |\u003Cb>70.4\u003C\u002Fb>±.1|\u003Cb>47.5\u003C\u002Fb>±.1| 48.4±.3\u002F\u003Cb>64.0\u003C\u002Fb>±.3\u002F\u003Cb>57.0\u003C\u002Fb>±.1 |\n\nNon-LoRA baselines, except for adapter on GPT-2 large, are taken from [Li and Liang (2021)](https:\u002F\u002Farxiv.org\u002Fabs\u002F2101.00190). We include confidence intervals on results from our experiments.\n\nDownload the GPT-2 LoRA checkpoints:\n * [GPT-2 Medium E2E](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FGPT-2\u002Fgpt2_md_lora_e2e.pt) (1.5 MB)\n * [GPT-2 Medium DART](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FGPT-2\u002Fgpt2_md_lora_dart.pt) (1.5 MB)\n * [GPT-2 Medium WebNLG](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FGPT-2\u002Fgpt2_md_lora_webnlg.pt) (1.5 MB)\n * [GPT-2 Large E2E](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FGPT-2\u002Fgpt2_lg_lora_e2e.pt) (2.3 MB)\n * [GPT-2 Large DART](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FGPT-2\u002Fgpt2_lg_lora_dart.pt) (2.3 MB)\n * [GPT-2 Large WebNLG](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FGPT-2\u002Fgpt2_lg_lora_webnlg.pt) (2.3 MB)\n\nPlease follow the instructions in `examples\u002FNLG\u002F` to reproduce our result.\n## Repository Overview\n\n\u003Ci>(The initial release of this repo has been archived in the branch \"snapshot-9-15-2021\")\u003C\u002Fi>\n\nThere are several directories in this repo:\n* [loralib\u002F](loralib) contains the source code for the package `loralib`, which needs to be installed to run the examples we provide;\n* [examples\u002FNLG\u002F](examples\u002FNLG) contains an example implementation of LoRA in GPT-2 using our package, which can be used to reproduce the result in our paper;\n* [examples\u002FNLU\u002F](examples\u002FNLU) contains an example implementation of LoRA in RoBERTa and DeBERTa using our package, which produces competitive results on the GLUE benchmark;\n* See how we use `loralib` in [GPT-2](examples\u002FNLG\u002Fsrc\u002Fmodel.py), [RoBERTa](examples\u002FNLU\u002Fsrc\u002Ftransformers\u002Fmodels\u002Froberta\u002Fmodeling_roberta.py), and [DeBERTa v2](examples\u002FNLU\u002Fsrc\u002Ftransformers\u002Fmodels\u002Fdeberta_v2\u002Fmodeling_deberta_v2.py)\n\n## Quickstart\n\n 1. Installing `loralib` is simply\n ```bash\n pip install loralib\n # Alternatively\n # pip install git+https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\n ```\n\n 2. You can choose to adapt some layers by replacing them with counterparts implemented in `loralib`. We only support `nn.Linear`, `nn.Embedding`, and `nn.Conv2d` for now. We also support a `MergedLinear` for cases where a single `nn.Linear` represents more than one layers, such as in some implementations of the attention `qkv` projection (see Additional Notes for more).\n ```python\n # ===== Before =====\n # layer = nn.Linear(in_features, out_features)\n\n # ===== After ======\n import loralib as lora\n # Add a pair of low-rank adaptation matrices with rank r=16\n layer = lora.Linear(in_features, out_features, r=16)\n ```\n\n 3. Before the training loop begins, mark only LoRA parameters as trainable.\n ```python\n import loralib as lora\n model = BigModel()\n # This sets requires_grad to False for all parameters without the string \"lora_\" in their names\n lora.mark_only_lora_as_trainable(model)\n # Training loop\n for batch in dataloader:\n    ...\n ```\n 4. When saving a checkpoint, generate a `state_dict` that only contains LoRA parameters.\n ```python\n # ===== Before =====\n # torch.save(model.state_dict(), checkpoint_path)\n # ===== After =====\n torch.save(lora.lora_state_dict(model), checkpoint_path)\n ```\n 5. When loading a checkpoint using `load_state_dict`, be sure to set `strict=False`.\n ```python\n # Load the pretrained checkpoint first\n model.load_state_dict(torch.load('ckpt_pretrained.pt'), strict=False)\n # Then load the LoRA checkpoint\n model.load_state_dict(torch.load('ckpt_lora.pt'), strict=False)\n ```\n\n#### Now training can proceed as usual.\n\n## Additional Notes\n\n1. While we focus on a simple yet effect setup, namely adapting only the `q` and `v` projection in a Transformer, in our examples, LoRA can be apply to any subsets of pre-trained weights. We encourage you to explore different configurations, such as adapting the embedding layer by replacing `nn.Embedding` with `lora.Embedding` and\u002For adapting the MLP layers. It's very likely that the optimal configuration varies for different model architectures and tasks.\n\n2. Some Transformer implementation uses a single `nn.Linear` for the projection matrices for query, key, and value. If one wishes to constrain the rank of the updates to the individual matrices, one has to either break it up into three separate matrices or use `lora.MergedLinear`. Make sure to modify the checkpoint accordingly if you choose to break up the layer.\n```python\n# ===== Before =====\n# qkv_proj = nn.Linear(d_model, 3*d_model)\n# ===== After =====\n# Break it up (remember to modify the pretrained checkpoint accordingly)\nq_proj = lora.Linear(d_model, d_model, r=8)\nk_proj = nn.Linear(d_model, d_model)\nv_proj = lora.Linear(d_model, d_model, r=8)\n# Alternatively, use lora.MergedLinear (recommended)\nqkv_proj = lora.MergedLinear(d_model, 3*d_model, r=8, enable_lora=[True, False, True])\n```\n3. Training bias vectors in tandem with LoRA might be a cost-efficient way to squeeze out extra task performance (if you tune the learning rate carefully). While we did not study its effect thoroughly in our paper, we make it easy to try in `lora`. You can mark some biases as trainable by passing \"all\" or \"lora_only\" to `bias=` when calling `mark_only_lora_as_trainable`. Remember to pass the corresponding `bias=` argument to `lora_state_dict` when saving a checkpoint.\n```python\n# ===== Before =====\n# lora.mark_only_lora_as_trainable(model) # Not training any bias vectors\n# ===== After =====\n# Training all bias vectors associated with modules we apply LoRA to \nlora.mark_only_lora_as_trainable(model, bias='lora_only')\n# Alternatively, we can train *all* bias vectors in the model, including LayerNorm biases\nlora.mark_only_lora_as_trainable(model, bias='all')\n# When saving a checkpoint, use the same bias= ('all' or 'lora_only')\ntorch.save(lora.lora_state_dict(model, bias='all'), checkpoint_path)\n```\n4. Calling `model.eval()` will trigger the merging of LoRA parameters with the corresponding pretrained ones, which eliminates additional latency for subsequent forward passes. Calling `model.train()` again will undo the merge. This can be disabled by passing `merge_weights=False` to LoRA layers.\n\n## Contact\nPlease contact us or post an issue if you have any questions.\n\nFor questions related to the package `loralib`:\n* Edward Hu (edward@edwardjhu.com)\n* Phillip Wallis (phwallis@microsoft.com)\n* Weizhu Chen (wzchen@microsoft.com)\n\nThe GPT-2 example:\n* Phillip Wallis (phwallis@microsoft.com)\n* Yelong Shen (yeshe@microsoft.com)\n\nThe RoBERTa\u002FDeBERTa example:\n* Lu Wang (luw@microsoft.com)\n\n## Acknowledgements\nWe thank in alphabetical order Jianfeng Gao, Jade Huang, Jiayuan Huang, Lisa Xiang Li, Xiaodong Liu, Yabin Liu, Benjamin Van Durme, Luis Vargas, Haoran Wei, Peter Welinder, and Greg Yang for providing valuable feedback.\n\n## Citation\n```BibTeX\n@inproceedings{\nhu2022lora,\ntitle={Lo{RA}: Low-Rank Adaptation of Large Language Models},\nauthor={Edward J Hu and Yelong Shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Lu Wang and Weizhu Chen},\nbooktitle={International Conference on Learning Representations},\nyear={2022},\nurl={https:\u002F\u002Fopenreview.net\u002Fforum?id=nZeVKeeFYf9}\n}\n```\n\n## Contributing\n\nThis project welcomes contributions and suggestions.  Most contributions require you to agree to a\nContributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us\nthe rights to use your contribution. For details, visit https:\u002F\u002Fcla.opensource.microsoft.com.\n\nWhen you submit a pull request, a CLA bot will automatically determine whether you need to provide\na CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions\nprovided by the bot. You will only need to do this once across all repos using our CLA.\n\nThis project has adopted the [Microsoft Open Source Code of Conduct](https:\u002F\u002Fopensource.microsoft.com\u002Fcodeofconduct\u002F).\nFor more information see the [Code of Conduct FAQ](https:\u002F\u002Fopensource.microsoft.com\u002Fcodeofconduct\u002Ffaq\u002F) or\ncontact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.\n","# LoRA：大型语言模型的低秩适应\n\n本仓库包含 Python 包 `loralib` 的源代码，以及如何将其与 PyTorch 模型（例如 Hugging Face 中的模型）集成的若干示例。目前我们仅支持 PyTorch。\n\n有关 LoRA 的详细说明，请参阅我们的论文。\n\n**LoRA：大型语言模型的低秩适应** \u003Cbr>\n*Edward J. Hu\\*, Yelong Shen\\*, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen* \u003Cbr>\n论文：https:\u002F\u002Farxiv.org\u002Fabs\u002F2106.09685 \u003Cbr>\n视频讲解：https:\u002F\u002Fwww.youtube.com\u002Fwatch?v=DhRoTONcyZE \u003Cbr>\n\n*更新：2023年2月，LoRA 现已由 Hugging Face 的 [最先进的参数高效微调（PEFT）](https:\u002F\u002Fgithub.com\u002Fhuggingface\u002Fpeft) 库支持。*\n\nLoRA 通过学习一对秩分解矩阵来减少可训练参数的数量，同时冻结原始权重。这大大降低了针对特定任务进行适配的大型语言模型所需的存储空间，并在部署过程中实现了高效的多任务切换，且不会引入推理延迟。此外，LoRA 在性能上优于其他多种适配方法，包括 Adapter、前缀调优和全量微调。\n\n我们在 GLUE 基准测试中，使用 [RoBERTa (Liu et al., 2019)](https:\u002F\u002Farxiv.org\u002Fabs\u002F1907.11692) base 和 large 模型，以及 [DeBERTa (He et al., 2020)](https:\u002F\u002Farxiv.org\u002Fabs\u002F2006.03654) XXL 1.5B 模型，获得了与全量微调相当或更优的结果，而仅需训练和存储极小部分的参数。点击下方链接即可下载 RoBERTa 和 DeBERTa 的 LoRA 检查点。\n\n|   |         | RoBERTa base \u003Cbr> 全量微调  |  RoBERTa base \u003Cbr> LoRA  | DeBERTa XXL \u003Cbr> 全量微调 | DeBERTa XXL \u003Cbr> LoRA  |\n|---|-------------------------|----------------|--------------------------|-----------------|-----------------|\n|   | 可训练参数数量        | 1.25亿 | 80万 | 15亿 | 470万     |\n|   | MNLI (m-Acc\u002Fmm-Acc)     | \u003Cb>87.6\u003C\u002Fb> | [\u003Cb>87.5\u003C\u002Fb>±0.3\u002F86.9±0.3](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_mnli.bin) |91.7\u002F\u003Cb>91.9\u003C\u002Fb>| [\u003Cb>91.9\u003C\u002Fb>±0.1\u002F\u003Cb>91.9\u003C\u002Fb>±0.2](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_mnli.bin)       |\n|   | SST2 (Acc)              | 94.8 | [\u003Cb>95.1\u003C\u002Fb>±0.2](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_sst2.bin) | \u003Cb>97.2\u003C\u002Fb>    | [96.9±0.2](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_sst2.bin)                    |\n|   | MRPC (Acc)              | \u003Cb>90.2\u003C\u002Fb> | [\u003Cb>89.7\u003C\u002Fb>±0.7](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_mrpc.bin) | 92.0           | [\u003Cb>92.6\u003C\u002Fb>±0.6](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_mrpc.bin)             |\n|   | CoLA (Matthew's Corr)   | \u003Cb>63.6\u003C\u002Fb> | [\u003Cb>63.4\u003C\u002Fb>±1.2](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_cola.bin) | \u003Cb>72.0\u003C\u002Fb>    | [\u003Cb>72.4\u003C\u002Fb>±1.1](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_cola.bin)           |\n|   | QNLI (Acc)              | 92.8 | [\u003Cb>93.3\u003C\u002Fb>±0.3](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_qnli.bin) | \u003Cb>96.0\u003C\u002Fb>    | [\u003Cb>96.0\u003C\u002Fb>±0.1](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_qnli.bin)            |\n|   | QQP (Acc)               | \u003Cb>91.9\u003C\u002Fb> | [90.8±0.1](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_qqp.bin) | 92.7           | [\u003Cb>92.9\u003C\u002Fb>±0.1](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_qqp.bin)           |\n|   | RTE (Acc)               | 78.7 | [\u003Cb>86.6\u003C\u002Fb>±0.7](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_rte.bin) | 93.9           | [\u003Cb>94.9\u003C\u002Fb>±0.4](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_rte.bin)           |\n|   | STSB (Pearson\u002FSpearman 相关性) | 91.2 | [\u003Cb>91.5\u003C\u002Fb>±0.2\u002F\u003Cb>91.3\u003C\u002Fb>±0.2](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FRoBERTa-base\u002Froberta_base_lora_stsb.bin) |\u003Cb>92.9\u003C\u002Fb>\u002F92.6| [\u003Cb>93.0\u003C\u002Fb>±0.2\u002F\u003Cb>92.9\u003C\u002Fb>±0.3](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FDeBERTa\u002Fdeberta_v2_xxlarge_lora_stsb.bin)      |\n|   | 平均值                  | 86.40 | \u003Cb>87.24\u003C\u002Fb> | 91.06 | \u003Cb>91.32\u003C\u002Fb> |\n\n\u003Ci>注：使用 LoRA 检查点时，仍需从 [Hugging Face](https:\u002F\u002Fhuggingface.co\u002F) 获取原始预训练检查点。\u003C\u002Fi>\n\n全量微调的数据来自 [Liu et al. (2019)](https:\u002F\u002Farxiv.org\u002Fabs\u002F1907.11692) 和 [He et al. (2020)](https:\u002F\u002Farxiv.org\u002Fabs\u002F2006.03654)。我们为实验结果提供了置信区间。请按照 `examples\u002FNLU\u002F` 中的说明操作，以复现我们的结果。\n\n在 GPT-2 上，LoRA 的表现优于全量微调以及其他高效调优方法，例如 [Adapter (Houlsby et al., 2019)](https:\u002F\u002Farxiv.org\u002Fabs\u002F1902.00751) 和 [前缀调优 (Li and Liang, 2021)](https:\u002F\u002Farxiv.org\u002Fabs\u002F2101.00190)。我们在 E2E NLG Challenge、DART 和 WebNLG 数据集上进行了评估：\n\n|   | 方法              | 可训练参数数量 | E2E (BLEU)   | DART (BLEU)  | WebNLG (BLEU-U\u002FS\u002FA)            |\n|---|---------------------|-----------------------|--------------|--------------|--------------------------------|\n|   | GPT-2 M (全量微调) | 3.5492亿              | 68.2         | 46.0         | 30.4\u002F\u003Cb>63.2\u003C\u002Fb>\u002F47.6          |\n|   | GPT-2 M (Adapter)   | 37万                  | 66.3         | 42.4         | 45.1\u002F54.5\u002F50.2                 |\n|   | GPT-2 M (前缀调优) | 35万                  | 69.7         | 45.7         | 44.1\u002F63.1\u002F54.4                 |\n|   | GPT-2 M (LoRA)      | 35万                  |\u003Cb>70.4\u003C\u002Fb>±0.1|\u003Cb>47.1\u003C\u002Fb>±0.2| \u003Cb>46.7\u003C\u002Fb>±0.4\u002F62.1±0.2\u002F\u003Cb>55.3\u003C\u002Fb>±0.2 |\n|   | GPT-2 L (全量微调) | 7.7403亿              | 68.5         | 46.5         | 41.7\u002F\u003Cb>64.6\u003C\u002Fb>\u002F54.2          |\n|   | GPT-2 L (Adapter)   | 88万                  | 69.1±0.1      | 45.7±0.1      | \u003Cb>49.8\u003C\u002Fb>±0.0\u002F61.1±0.0\u002F56.0±0.0 |\n|   | GPT-2 L (前缀调优) | 77万                  | 70.3         | 46.5         | 47.0\u002F64.2\u002F56.4                 |\n|   | GPT-2 L (LoRA)      | 77万                  |\u003Cb>70.4\u003C\u002Fb>±0.1|\u003Cb>47.5\u003C\u002Fb>±0.1| 48.4±0.3\u002F\u003Cb>64.0\u003C\u002Fb>±0.3\u002F\u003Cb>57.0\u003C\u002Fb>±0.1 |\n\n非 LoRA 对照组数据除 GPT-2 large 上的 Adapter 外，均来自 [Li and Liang (2021)](https:\u002F\u002Farxiv.org\u002Fabs\u002F2101.00190)。我们为实验结果提供了置信区间。\n\n下载 GPT-2 LoRA 检查点：\n * [GPT-2 Medium E2E](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FGPT-2\u002Fgpt2_md_lora_e2e.pt) (1.5 MB)\n * [GPT-2 Medium DART](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FGPT-2\u002Fgpt2_md_lora_dart.pt) (1.5 MB)\n * [GPT-2 Medium WebNLG](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FGPT-2\u002Fgpt2_md_lora_webnlg.pt) (1.5 MB)\n * [GPT-2 Large E2E](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FGPT-2\u002Fgpt2_lg_lora_e2e.pt) (2.3 MB)\n * [GPT-2 Large DART](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FGPT-2\u002Fgpt2_lg_lora_dart.pt) (2.3 MB)\n * [GPT-2 Large WebNLG](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\u002Freleases\u002Fdownload\u002FGPT-2\u002Fgpt2_lg_lora_webnlg.pt) (2.3 MB)\n\n请按照 `examples\u002FNLG\u002F` 中的说明来复现我们的结果。\n\n\n## 仓库概览\n\n\u003Ci>(本仓库的初始版本已归档在分支 \"snapshot-9-15-2021\" 中)\u003C\u002Fi>\n\n该仓库包含以下几个目录：\n* [loralib\u002F](loralib) 包含 `loralib` 软件包的源代码，需要安装此包才能运行我们提供的示例；\n* [examples\u002FNLG\u002F](examples\u002FNLG) 包含使用我们的软件包在 GPT-2 中实现 LoRA 的示例，可用于复现论文中的结果；\n* [examples\u002FNLU\u002F](examples\u002FNLU) 包含使用我们的软件包在 RoBERTa 和 DeBERTa 中实现 LoRA 的示例，在 GLUE 基准测试上取得了具有竞争力的结果；\n* 请参阅我们在 [GPT-2](examples\u002FNLG\u002Fsrc\u002Fmodel.py)、[RoBERTa](examples\u002FNLU\u002Fsrc\u002Ftransformers\u002Fmodels\u002Froberta\u002Fmodeling_roberta.py) 和 [DeBERTa v2](examples\u002FNLU\u002Fsrc\u002Ftransformers\u002Fmodels\u002Fdeberta_v2\u002Fmodeling_deberta_v2.py) 中如何使用 `loralib`\n\n## 快速入门\n\n 1. 安装 `loralib` 非常简单：\n ```bash\n pip install loralib\n # 或者\n # pip install git+https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\n ```\n\n 2. 您可以选择用 `loralib` 中实现的对应层替换部分层以进行适配。目前我们仅支持 `nn.Linear`、`nn.Embedding` 和 `nn.Conv2d`。此外，我们还提供 `MergedLinear`，用于处理单个 `nn.Linear` 表示多个层的情况，例如某些注意力机制中 `qkv` 投影的实现（更多信息请参见附注）。\n ```python\n # ===== 之前 =====\n # layer = nn.Linear(in_features, out_features)\n\n # ===== 之后 ======\n import loralib as lora\n # 添加一对低秩适配矩阵，秩为 r=16\n layer = lora.Linear(in_features, out_features, r=16)\n ```\n\n 3. 在训练循环开始前，仅将 LoRA 参数标记为可训练。\n ```python\n import loralib as lora\n model = BigModel()\n # 这会将所有名称中不包含 \"lora_\" 字样的参数的 requires_grad 设置为 False\n lora.mark_only_lora_as_trainable(model)\n # 训练循环\n for batch in dataloader:\n    ...\n ```\n 4. 保存检查点时，生成仅包含 LoRA 参数的 `state_dict`。\n ```python\n # ===== 之前 =====\n # torch.save(model.state_dict(), checkpoint_path)\n # ===== 之后 =====\n torch.save(lora.lora_state_dict(model), checkpoint_path)\n ```\n 5. 使用 `load_state_dict` 加载检查点时，请务必设置 `strict=False`。\n ```python\n # 先加载预训练检查点\n model.load_state_dict(torch.load('ckpt_pretrained.pt'), strict=False)\n # 然后加载 LoRA 检查点\n model.load_state_dict(torch.load('ckpt_lora.pt'), strict=False)\n ```\n\n#### 现在可以像往常一样继续训练了。\n\n## 附注\n\n1. 尽管我们在示例中专注于一个简单而有效的设置——即仅对 Transformer 中的 `q` 和 `v` 投影进行适配——但 LoRA 实际上可以应用于预训练权重的任何子集。我们鼓励您尝试不同的配置，例如用 `lora.Embedding` 替换 `nn.Embedding` 层，或对 MLP 层进行适配。对于不同的模型架构和任务，最佳配置很可能会有所不同。\n\n2. 有些 Transformer 实现使用单个 `nn.Linear` 来完成查询、键和值的投影矩阵。如果希望限制对各个矩阵更新的秩，就需要将其拆分为三个独立的矩阵，或者使用 `lora.MergedLinear`。如果您选择拆分该层，请务必相应地修改检查点。\n```python\n# ===== 之前 =====\n# qkv_proj = nn.Linear(d_model, 3*d_model)\n# ===== 之后 =====\n# 拆分成三个独立的矩阵（记得相应地修改预训练检查点）\nq_proj = lora.Linear(d_model, d_model, r=8)\nk_proj = nn.Linear(d_model, d_model)\nv_proj = lora.Linear(d_model, d_model, r=8)\n# 或者使用 lora.MergedLinear（推荐）\nqkv_proj = lora.MergedLinear(d_model, 3*d_model, r=8, enable_lora=[True, False, True])\n```\n3. 与 LoRA 同时训练偏置向量可能是提升任务性能的一种经济高效的方式（前提是仔细调整学习率）。虽然我们在论文中并未深入研究其效果，但在 `lora` 中我们已经使其易于尝试。您可以通过在调用 `mark_only_lora_as_trainable` 时将 `bias=` 参数设置为 \"all\" 或 \"lora_only\" 来标记部分偏置为可训练。保存检查点时，请确保将相应的 `bias=` 参数传递给 `lora_state_dict`。\n```python\n# ===== 之前 =====\n# lora.mark_only_lora_as_trainable(model) # 不训练任何偏置向量\n# ===== 之后 =====\n# 训练所有与应用 LoRA 的模块相关的偏置向量\nlora.mark_only_lora_as_trainable(model, bias='lora_only')\n# 或者，我们可以训练模型中的 *所有* 偏置向量，包括 LayerNorm 的偏置\nlora.mark_only_lora_as_trainable(model, bias='all')\n# 保存检查点时，使用相同的 bias=（'all' 或 'lora_only'）\ntorch.save(lora.lora_state_dict(model, bias='all'), checkpoint_path)\n```\n4. 调用 `model.eval()` 会触发 LoRA 参数与其对应的预训练参数合并，从而消除后续前向传播的额外延迟。再次调用 `model.train()` 则会取消合并。这一行为可以通过在 LoRA 层中传入 `merge_weights=False` 来禁用。\n\n## 联系方式\n如有任何问题，请联系我们或提交 issue。\n\n关于 `loralib` 软件包的问题：\n* Edward Hu (edward@edwardjhu.com)\n* Phillip Wallis (phwallis@microsoft.com)\n* Weizhu Chen (wzchen@microsoft.com)\n\n关于 GPT-2 示例的问题：\n* Phillip Wallis (phwallis@microsoft.com)\n* Yelong Shen (yeshe@microsoft.com)\n\n关于 RoBERTa\u002FDeBERTa 示例的问题：\n* Lu Wang (luw@microsoft.com)\n\n## 致谢\n我们按字母顺序感谢 Jianfeng Gao、Jade Huang、Jiayuan Huang、Lisa Xiang Li、Xiaodong Liu、Yabin Liu、Benjamin Van Durme、Luis Vargas、Haoran Wei、Peter Welinder 和 Greg Yang 提供的宝贵反馈。\n\n## 引用\n```BibTeX\n@inproceedings{\nhu2022lora,\ntitle={Lo{RA}: Low-Rank Adaptation of Large Language Models},\nauthor={Edward J Hu and Yelong Shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Lu Wang and Weizhu Chen},\nbooktitle={International Conference on Learning Representations},\nyear={2022},\nurl={https:\u002F\u002Fopenreview.net\u002Fforum?id=nZeVKeeFYf9}\n}\n```\n\n## 贡献说明\n\n本项目欢迎各类贡献和建议。大多数贡献都需要您签署一份贡献者许可协议（CLA），以声明您有权且确实将您的贡献使用权授予我们。有关详细信息，请访问 https:\u002F\u002Fcla.opensource.microsoft.com。\n\n当您提交拉取请求时，CLA 机器人会自动判断您是否需要提供 CLA，并相应地为该 PR 添加标记（例如状态检查、评论等）。请按照机器人提供的指示操作即可。对于使用我们 CLA 的所有仓库，您只需完成一次此操作。\n\n本项目已采纳 [微软开源行为准则](https:\u002F\u002Fopensource.microsoft.com\u002Fcodeofconduct\u002F)。如需更多信息，请参阅 [行为准则常见问题解答](https:\u002F\u002Fopensource.microsoft.com\u002Fcodeofconduct\u002Ffaq\u002F)；如有任何其他问题或意见，请发送邮件至 [opencode@microsoft.com](mailto:opencode@microsoft.com)。","# LoRA 快速上手指南\n\nLoRA (Low-Rank Adaptation) 是一种用于大语言模型的高效微调技术。它通过冻结预训练权重并学习低秩分解矩阵，大幅减少可训练参数量，在降低存储需求的同时实现高效的任務切换，且不会增加推理延迟。\n\n## 环境准备\n\n*   **操作系统**: Linux, macOS, Windows\n*   **编程语言**: Python 3.7+\n*   **核心依赖**:\n    *   PyTorch (目前仅支持 PyTorch)\n    *   Hugging Face Transformers (推荐，用于加载预训练模型)\n*   **硬件要求**: 支持 CUDA 的 GPU（推荐）或 CPU\n\n## 安装步骤\n\n你可以直接通过 pip 安装官方发布的 `loralib` 包：\n\n```bash\npip install loralib\n```\n\n或者从 GitHub 源码安装最新版本：\n\n```bash\npip install git+https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002FLoRA\n```\n\n> **提示**：国内用户若下载缓慢，可尝试使用清华或阿里镜像源加速：\n> `pip install loralib -i https:\u002F\u002Fpypi.tuna.tsinghua.edu.cn\u002Fsimple`\n\n## 基本使用\n\n以下是将 LoRA 集成到现有 PyTorch 模型中的最小化流程。\n\n### 1. 替换网络层\n将需要适配的标准层（如 `nn.Linear`）替换为 `loralib` 提供的对应层，并指定秩（rank, `r`）。\n\n```python\nimport torch.nn as nn\nimport loralib as lora\n\n# ===== 修改前 =====\n# layer = nn.Linear(in_features, out_features)\n\n# ===== 修改后 =====\n# 添加一对秩 r=16 的低秩适应矩阵\nlayer = lora.Linear(in_features, out_features, r=16)\n```\n*注：目前支持 `nn.Linear`, `nn.Embedding`, `nn.Conv2d` 以及针对注意力机制优化的 `lora.MergedLinear`。*\n\n### 2. 冻结参数并标记 LoRA 可训练\n在训练循环开始前，冻结模型所有原始参数，仅将包含 \"lora_\" 的参数设为可训练。\n\n```python\nimport loralib as lora\n\nmodel = BigModel() # 你的预训练模型\n\n# 将所有不含 \"lora_\" 的参数的 requires_grad 设为 False\nlora.mark_only_lora_as_trainable(model)\n\n# 开始训练循环\nfor batch in dataloader:\n    # ... 常规训练代码 ...\n    pass\n```\n\n### 3. 保存检查点\n保存时，仅导出 LoRA 相关的参数，以节省存储空间。\n\n```python\n# ===== 修改前 =====\n# torch.save(model.state_dict(), checkpoint_path)\n\n# ===== 修改后 =====\n# 仅保存 LoRA 参数的 state_dict\ntorch.save(lora.lora_state_dict(model), checkpoint_path)\n```\n\n### 4. 加载检查点\n加载时需要分两步：先加载预训练底座权重，再加载 LoRA 权重。**注意**：加载 LoRA 权重时必须设置 `strict=False`。\n\n```python\n# 1. 加载预训练底座权重\nmodel.load_state_dict(torch.load('ckpt_pretrained.pt'), strict=False)\n\n# 2. 加载 LoRA 微调权重\nmodel.load_state_dict(torch.load('ckpt_lora.pt'), strict=False)\n```\n\n完成上述步骤后，即可像往常一样进行训练或推理。","某金融科技公司希望将通用的 15 亿参数大语言模型（如 DeBERTa）快速适配到“信贷风险报告生成”这一垂直领域，以辅助分析师撰写专业文档。\n\n### 没有 LoRA 时\n- **显存资源爆炸**：全量微调需要加载完整的模型权重及优化器状态，单卡显存无法支撑，必须租用昂贵的多卡集群，硬件成本极高。\n- **存储与部署沉重**：每针对一个新业务（如反欺诈或合规审查）微调一次，就需保存一份完整的 1.5B 参数副本，导致磁盘空间迅速耗尽且版本管理混乱。\n- **任务切换低效**：在生产环境中切换不同任务模型时，需要重新加载巨大的权重文件，造成严重的服务延迟，无法满足实时性要求。\n- **训练周期漫长**：由于参数量巨大，梯度更新和收敛过程缓慢，从数据准备到模型上线往往需要数天时间，难以响应紧急业务需求。\n\n### 使用 LoRA 后\n- **显存占用骤降**：LoRA 冻结了原始模型权重，仅训练极低秩的分解矩阵（仅需约 470 万参数），使得单张消费级显卡即可完成原本需要集群才能跑通的训练任务。\n- **轻量级存储管理**：每个特定任务只需保存几兆大小的 LoRA 适配器文件，无需复制主模型，极大节省了存储空间并简化了多版本迭代管理。\n- **零延迟动态切换**：部署时可动态加载不同的 LoRA 模块到同一个基座模型上，实现毫秒级的任务热切换，且不引入任何额外的推理延迟。\n- **高效迭代上线**：可训练参数减少了两个数量级，训练速度显著提升，团队能在几小时内完成从实验到部署的全流程，快速响应市场变化。\n\nLoRA 通过极小化的参数调整，在保持与大模型全量微调相当性能的同时，彻底解决了垂直领域落地中的资源瓶颈与部署效率难题。","https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Fmicrosoft_LoRA_2cafcc8e.png","microsoft","Microsoft","https:\u002F\u002Foss.gittoolsai.com\u002Favatars\u002Fmicrosoft_4900709c.png","Open source projects and samples from Microsoft",null,"opensource@microsoft.com","OpenAtMicrosoft","https:\u002F\u002Fopensource.microsoft.com","https:\u002F\u002Fgithub.com\u002Fmicrosoft",[82],{"name":83,"color":84,"percentage":85},"Python","#3572A5",100,13432,891,"2026-04-15T11:34:20","MIT","","未说明",{"notes":93,"python":91,"dependencies":94},"该工具仅支持 PyTorch 框架。使用时需先加载原始预训练模型权重（如来自 Hugging Face），再加载 LoRA 适配器权重。支持对 nn.Linear、nn.Embedding 和 nn.Conv2d 层进行适配。具体硬件需求取决于所使用的基础大语言模型规模（如 RoBERTa、DeBERTa 或 GPT-2）。",[95,96],"torch","loralib",[35,14],[99,100,101,102,103,104,105,106,107,108],"gpt-2","adaptation","language-model","gpt-3","low-rank","pytorch","deep-learning","roberta","deberta","lora","2026-03-27T02:49:30.150509","2026-04-16T16:09:14.698359",[],[113,118,123,128],{"id":114,"version":115,"summary_zh":116,"released_at":117},281163,"DeBERTa","DeBERTa XXL 的 LoRA 检查点。请参阅 NLU\u002FREADME.md，了解复现我们实验结果的步骤。","2021-09-16T22:05:55",{"id":119,"version":120,"summary_zh":121,"released_at":122},281164,"GPT-2","GPT-2 中型和大型模型的 LoRA 检查点。请参阅 NLG\u002FREADME.md，了解复现我们实验结果的步骤。","2021-09-16T21:54:56",{"id":124,"version":125,"summary_zh":126,"released_at":127},281165,"RoBERTa-large","RoBERTa Large 模型的 LoRA 检查点。请参阅 NLU\u002FREADME.md，了解复现我们实验结果的步骤。","2021-09-16T21:52:32",{"id":129,"version":130,"summary_zh":131,"released_at":132},281166,"RoBERTa-base","用于 RoBERTa base 的 LoRA 检查点。请参阅 NLU\u002FREADME.md，了解复现我们实验结果的步骤。","2021-09-16T21:48:51"]