[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"similar-appvision-ai--fast-bert":3,"tool-appvision-ai--fast-bert":64},[4,17,25,39,48,56],{"id":5,"name":6,"github_repo":7,"description_zh":8,"stars":9,"difficulty_score":10,"last_commit_at":11,"category_tags":12,"status":16},1381,"everything-claude-code","affaan-m\u002Feverything-claude-code","everything-claude-code 是一套专为 AI 编程助手（如 Claude Code、Codex、Cursor 等）打造的高性能优化系统。它不仅仅是一组配置文件，而是一个经过长期实战打磨的完整框架，旨在解决 AI 代理在实际开发中面临的效率低下、记忆丢失、安全隐患及缺乏持续学习能力等核心痛点。\n\n通过引入技能模块化、直觉增强、记忆持久化机制以及内置的安全扫描功能，everything-claude-code 能显著提升 AI 在复杂任务中的表现，帮助开发者构建更稳定、更智能的生产级 AI 代理。其独特的“研究优先”开发理念和针对 Token 消耗的优化策略，使得模型响应更快、成本更低，同时有效防御潜在的攻击向量。\n\n这套工具特别适合软件开发者、AI 研究人员以及希望深度定制 AI 工作流的技术团队使用。无论您是在构建大型代码库，还是需要 AI 协助进行安全审计与自动化测试，everything-claude-code 都能提供强大的底层支持。作为一个曾荣获 Anthropic 黑客大奖的开源项目，它融合了多语言支持与丰富的实战钩子（hooks），让 AI 真正成长为懂上",138956,2,"2026-04-05T11:33:21",[13,14,15],"开发框架","Agent","语言模型","ready",{"id":18,"name":19,"github_repo":20,"description_zh":21,"stars":22,"difficulty_score":10,"last_commit_at":23,"category_tags":24,"status":16},3704,"NextChat","ChatGPTNextWeb\u002FNextChat","NextChat 是一款轻量且极速的 AI 助手，旨在为用户提供流畅、跨平台的大模型交互体验。它完美解决了用户在多设备间切换时难以保持对话连续性，以及面对众多 AI 模型不知如何统一管理的痛点。无论是日常办公、学习辅助还是创意激发，NextChat 都能让用户随时随地通过网页、iOS、Android、Windows、MacOS 或 Linux 端无缝接入智能服务。\n\n这款工具非常适合普通用户、学生、职场人士以及需要私有化部署的企业团队使用。对于开发者而言，它也提供了便捷的自托管方案，支持一键部署到 Vercel 或 Zeabur 等平台。\n\nNextChat 的核心亮点在于其广泛的模型兼容性，原生支持 Claude、DeepSeek、GPT-4 及 Gemini Pro 等主流大模型，让用户在一个界面即可自由切换不同 AI 能力。此外，它还率先支持 MCP（Model Context Protocol）协议，增强了上下文处理能力。针对企业用户，NextChat 提供专业版解决方案，具备品牌定制、细粒度权限控制、内部知识库整合及安全审计等功能，满足公司对数据隐私和个性化管理的高标准要求。",87618,"2026-04-05T07:20:52",[13,15],{"id":26,"name":27,"github_repo":28,"description_zh":29,"stars":30,"difficulty_score":10,"last_commit_at":31,"category_tags":32,"status":16},2268,"ML-For-Beginners","microsoft\u002FML-For-Beginners","ML-For-Beginners 是由微软推出的一套系统化机器学习入门课程，旨在帮助零基础用户轻松掌握经典机器学习知识。这套课程将学习路径规划为 12 周，包含 26 节精炼课程和 52 道配套测验，内容涵盖从基础概念到实际应用的完整流程，有效解决了初学者面对庞大知识体系时无从下手、缺乏结构化指导的痛点。\n\n无论是希望转型的开发者、需要补充算法背景的研究人员，还是对人工智能充满好奇的普通爱好者，都能从中受益。课程不仅提供了清晰的理论讲解，还强调动手实践，让用户在循序渐进中建立扎实的技能基础。其独特的亮点在于强大的多语言支持，通过自动化机制提供了包括简体中文在内的 50 多种语言版本，极大地降低了全球不同背景用户的学习门槛。此外，项目采用开源协作模式，社区活跃且内容持续更新，确保学习者能获取前沿且准确的技术资讯。如果你正寻找一条清晰、友好且专业的机器学习入门之路，ML-For-Beginners 将是理想的起点。",84991,"2026-04-05T10:45:23",[33,34,35,36,14,37,15,13,38],"图像","数据工具","视频","插件","其他","音频",{"id":40,"name":41,"github_repo":42,"description_zh":43,"stars":44,"difficulty_score":45,"last_commit_at":46,"category_tags":47,"status":16},3128,"ragflow","infiniflow\u002Fragflow","RAGFlow 是一款领先的开源检索增强生成（RAG）引擎，旨在为大语言模型构建更精准、可靠的上下文层。它巧妙地将前沿的 RAG 技术与智能体（Agent）能力相结合，不仅支持从各类文档中高效提取知识，还能让模型基于这些知识进行逻辑推理和任务执行。\n\n在大模型应用中，幻觉问题和知识滞后是常见痛点。RAGFlow 通过深度解析复杂文档结构（如表格、图表及混合排版），显著提升了信息检索的准确度，从而有效减少模型“胡编乱造”的现象，确保回答既有据可依又具备时效性。其内置的智能体机制更进一步，使系统不仅能回答问题，还能自主规划步骤解决复杂问题。\n\n这款工具特别适合开发者、企业技术团队以及 AI 研究人员使用。无论是希望快速搭建私有知识库问答系统，还是致力于探索大模型在垂直领域落地的创新者，都能从中受益。RAGFlow 提供了可视化的工作流编排界面和灵活的 API 接口，既降低了非算法背景用户的上手门槛，也满足了专业开发者对系统深度定制的需求。作为基于 Apache 2.0 协议开源的项目，它正成为连接通用大模型与行业专有知识之间的重要桥梁。",77062,3,"2026-04-04T04:44:48",[14,33,13,15,37],{"id":49,"name":50,"github_repo":51,"description_zh":52,"stars":53,"difficulty_score":45,"last_commit_at":54,"category_tags":55,"status":16},519,"PaddleOCR","PaddlePaddle\u002FPaddleOCR","PaddleOCR 是一款基于百度飞桨框架开发的高性能开源光学字符识别工具包。它的核心能力是将图片、PDF 等文档中的文字提取出来，转换成计算机可读取的结构化数据，让机器真正“看懂”图文内容。\n\n面对海量纸质或电子文档，PaddleOCR 解决了人工录入效率低、数字化成本高的问题。尤其在人工智能领域，它扮演着连接图像与大型语言模型（LLM）的桥梁角色，能将视觉信息直接转化为文本输入，助力智能问答、文档分析等应用场景落地。\n\nPaddleOCR 适合开发者、算法研究人员以及有文档自动化需求的普通用户。其技术优势十分明显：不仅支持全球 100 多种语言的识别，还能在 Windows、Linux、macOS 等多个系统上运行，并灵活适配 CPU、GPU、NPU 等各类硬件。作为一个轻量级且社区活跃的开源项目，PaddleOCR 既能满足快速集成的需求，也能支撑前沿的视觉语言研究，是处理文字识别任务的理想选择。",74913,"2026-04-05T10:44:17",[15,33,13,37],{"id":57,"name":58,"github_repo":59,"description_zh":60,"stars":61,"difficulty_score":45,"last_commit_at":62,"category_tags":63,"status":16},2181,"OpenHands","OpenHands\u002FOpenHands","OpenHands 是一个专注于 AI 驱动开发的开源平台，旨在让智能体（Agent）像人类开发者一样理解、编写和调试代码。它解决了传统编程中重复性劳动多、环境配置复杂以及人机协作效率低等痛点，通过自动化流程显著提升开发速度。\n\n无论是希望提升编码效率的软件工程师、探索智能体技术的研究人员，还是需要快速原型验证的技术团队，都能从中受益。OpenHands 提供了灵活多样的使用方式：既可以通过命令行（CLI）或本地图形界面在个人电脑上轻松上手，体验类似 Devin 的流畅交互；也能利用其强大的 Python SDK 自定义智能体逻辑，甚至在云端大规模部署上千个智能体并行工作。\n\n其核心技术亮点在于模块化的软件智能体 SDK，这不仅构成了平台的引擎，还支持高度可组合的开发模式。此外，OpenHands 在 SWE-bench 基准测试中取得了 77.6% 的优异成绩，证明了其解决真实世界软件工程问题的能力。平台还具备完善的企业级功能，支持与 Slack、Jira 等工具集成，并提供细粒度的权限管理，适合从个人开发者到大型企业的各类用户场景。",70612,"2026-04-05T11:12:22",[15,14,13,36],{"id":65,"github_repo":66,"name":67,"description_en":68,"description_zh":69,"ai_summary_zh":70,"readme_en":71,"readme_zh":72,"quickstart_zh":73,"use_case_zh":74,"hero_image_url":75,"owner_login":76,"owner_name":77,"owner_avatar_url":78,"owner_bio":79,"owner_company":80,"owner_location":80,"owner_email":80,"owner_twitter":80,"owner_website":80,"owner_url":81,"languages":82,"stars":99,"forks":100,"last_commit_at":101,"license":102,"difficulty_score":103,"env_os":104,"env_gpu":105,"env_ram":104,"env_deps":106,"category_tags":114,"github_topics":115,"view_count":10,"oss_zip_url":80,"oss_zip_packed_at":80,"status":16,"created_at":119,"updated_at":120,"faqs":121,"releases":150},3257,"appvision-ai\u002Ffast-bert","fast-bert","Super easy library for BERT based NLP models","fast-bert 是一个专为自然语言处理（NLP）任务设计的深度学习库，旨在让基于 BERT、RoBERTa、XLNet 等先进模型的文本分类工作变得极其简单。它主要解决了传统大模型训练门槛高、代码复杂且难以快速部署的痛点，帮助开发者将精力集中在数据与业务逻辑上，而非繁琐的工程实现。\n\n这款工具非常适合希望快速上手前沿 NLP 技术的开发人员、数据科学家以及算法研究人员。无论是需要处理多类别还是多标签的文本分类任务，fast-bert 都能提供流畅的支持。其核心亮点在于借鉴了 fast.ai 的设计理念，提供了高度封装的 API，用户只需几行代码即可完成从数据加载、模型微调到超参数调整的全过程。此外，fast-bert 还集成了学习率查找器（Learning Rate Finder）以辅助优化训练效果，并支持 LAMB 优化器来显著加速大规模模型的训练进程。\n\n除了本地训练，fast-bert 还具备完善的模型保存与部署功能，甚至支持直接部署到 AWS Sagemaker 云端环境。依托于 Hugging Face 强大的底层库，fast-bert 在保持技术先进性的同时，极大地降低","fast-bert 是一个专为自然语言处理（NLP）任务设计的深度学习库，旨在让基于 BERT、RoBERTa、XLNet 等先进模型的文本分类工作变得极其简单。它主要解决了传统大模型训练门槛高、代码复杂且难以快速部署的痛点，帮助开发者将精力集中在数据与业务逻辑上，而非繁琐的工程实现。\n\n这款工具非常适合希望快速上手前沿 NLP 技术的开发人员、数据科学家以及算法研究人员。无论是需要处理多类别还是多标签的文本分类任务，fast-bert 都能提供流畅的支持。其核心亮点在于借鉴了 fast.ai 的设计理念，提供了高度封装的 API，用户只需几行代码即可完成从数据加载、模型微调到超参数调整的全过程。此外，fast-bert 还集成了学习率查找器（Learning Rate Finder）以辅助优化训练效果，并支持 LAMB 优化器来显著加速大规模模型的训练进程。\n\n除了本地训练，fast-bert 还具备完善的模型保存与部署功能，甚至支持直接部署到 AWS Sagemaker 云端环境。依托于 Hugging Face 强大的底层库，fast-bert 在保持技术先进性的同时，极大地降低了工程复杂度，是让社区更便捷地应用尖端深度学习技术的得力助手。","# Fast-Bert\n\n[![License Apache 2.0](https:\u002F\u002Fimg.shields.io\u002Fbadge\u002Flicense-Apache%202.0-blue.svg)](https:\u002F\u002Fgithub.com\u002Fdeepmipt\u002FDeepPavlov\u002Fblob\u002Fmaster\u002FLICENSE)\n[![PyPI version](https:\u002F\u002Fbadge.fury.io\u002Fpy\u002Ffast-bert.svg)](https:\u002F\u002Fbadge.fury.io\u002Fpy\u002Ffast-bert)\n![Python 3.6, 3.7](https:\u002F\u002Fimg.shields.io\u002Fbadge\u002Fpython-3.6%20%7C%203.7-green.svg)\n\n**New - Learning Rate Finder for Text Classification Training (borrowed with thanks from https:\u002F\u002Fgithub.com\u002Fdavidtvs\u002Fpytorch-lr-finder)**\n\n\n**Supports LAMB optimizer for faster training.**\nPlease refer to https:\u002F\u002Farxiv.org\u002Fabs\u002F1904.00962 for the paper on LAMB optimizer.\n\n**Supports BERT and XLNet for both Multi-Class and Multi-Label text classification.**\n\nFast-Bert is the deep learning library that allows developers and data scientists to train and deploy BERT and XLNet based models for natural language processing tasks beginning with Text Classification.\n\nThe work on FastBert is built on solid foundations provided by the excellent [Hugging Face BERT PyTorch library](https:\u002F\u002Fgithub.com\u002Fhuggingface\u002Fpytorch-pretrained-BERT) and is inspired by [fast.ai](https:\u002F\u002Fgithub.com\u002Ffastai\u002Ffastai) and strives to make the cutting edge deep learning technologies accessible for the vast community of machine learning practitioners.\n\nWith FastBert, you will be able to:\n\n1. Train (more precisely fine-tune) BERT, RoBERTa and XLNet text classification models on your custom dataset.\n\n2. Tune model hyper-parameters such as epochs, learning rate, batch size, optimiser schedule and more.\n\n3. Save and deploy trained model for inference (including on AWS Sagemaker).\n\nFast-Bert will support both multi-class and multi-label text classification for the following and in due course, it will support other NLU tasks such as Named Entity Recognition, Question Answering and Custom Corpus fine-tuning.\n\n1.  **[BERT](https:\u002F\u002Fgithub.com\u002Fgoogle-research\u002Fbert)** (from Google) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https:\u002F\u002Farxiv.org\u002Fabs\u002F1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova.\n\n2)  **[XLNet](https:\u002F\u002Fgithub.com\u002Fzihangdai\u002Fxlnet\u002F)** (from Google\u002FCMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https:\u002F\u002Farxiv.org\u002Fabs\u002F1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.\n\n3)  **[RoBERTa](https:\u002F\u002Farxiv.org\u002Fabs\u002F1907.11692)** (from Facebook), a Robustly Optimized BERT Pretraining Approach by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du et al.\n\n4)  **DistilBERT (from HuggingFace)**, released together with the blogpost [Smaller, faster, cheaper, lighter: Introducing DistilBERT, a distilled version of BERT](https:\u002F\u002Fmedium.com\u002Fhuggingface\u002Fdistilbert-8cf3380435b5) by Victor Sanh, Lysandre Debut and Thomas Wolf.\n\n## Installation\n\nThis repo is tested on Python 3.6+.\n\n### With pip\n\nPyTorch-Transformers can be installed by pip as follows:\n\n```bash\npip install fast-bert\n```\n\n### From source\n\nClone the repository and run:\n\n```bash\npip install [--editable] .\n```\n\nor\n\n```bash\npip install git+https:\u002F\u002Fgithub.com\u002Fkaushaltrivedi\u002Ffast-bert.git\n```\n\nYou will also need to install NVIDIA Apex.\n\n```bash\ngit clone https:\u002F\u002Fgithub.com\u002FNVIDIA\u002Fapex\ncd apex\npip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" .\u002F\n```\n\n## Usage\n\n## Text Classification\n\n### 1. Create a DataBunch object\n\nThe databunch object takes training, validation and test csv files and converts the data into internal representation for BERT, RoBERTa, DistilBERT or XLNet. The object also instantiates the correct data-loaders based on device profile and batch_size and max_sequence_length.\n\n```python\n\nfrom fast_bert.data_cls import BertDataBunch\n\ndatabunch = BertDataBunch(DATA_PATH, LABEL_PATH,\n                          tokenizer='bert-base-uncased',\n                          train_file='train.csv',\n                          val_file='val.csv',\n                          label_file='labels.csv',\n                          text_col='text',\n                          label_col='label',\n                          batch_size_per_gpu=16,\n                          max_seq_length=512,\n                          multi_gpu=True,\n                          multi_label=False,\n                          model_type='bert')\n```\n\n#### File format for train.csv and val.csv\n\n| index | text                                                                                                                                                                                                                                                                                                                                | label |\n| ----- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |\n| 0     | Looking through the other comments, I'm amazed that there aren't any warnings to potential viewers of what they have to look forward to when renting this garbage. First off, I rented this thing with the understanding that it was a competently rendered Indiana Jones knock-off.                                                | neg   |\n| 1     | I've watched the first 17 episodes and this series is simply amazing! I haven't been this interested in an anime series since Neon Genesis Evangelion. This series is actually based off an h-game, which I'm not sure if it's been done before or not, I haven't played the game, but from what I've heard it follows it very well | pos   |\n| 2     | his movie is nothing short of a dark, gritty masterpiece. I may be bias, as the Apartheid era is an area I've always felt for.                                                                                                                                                                                                      | pos   |\n\nIn case the column names are different than the usual text and labels, you will have to provide those names in the databunch text_col and label_col parameters.\n\n**labels.csv** will contain a list of all unique labels. In this case the file will contain:\n\n```csv\npos\nneg\n```\n\nFor multi-label classification, **labels.csv** will contain all possible labels:\n\n```\ntoxic\nsevere_toxic\nobscene\nthreat\ninsult\nidentity_hate\n```\n\nThe file **train.csv** will then contain one column for each label, with each column value being either 0 or 1. Don't forget to change `multi_label=True` for multi-label classification in `BertDataBunch`.\n\n| id  | text                                                                       | toxic | severe_toxic | obscene | threat | insult | identity_hate |\n| --- | -------------------------------------------------------------------------- | ----- | ------------ | ------- | ------ | ------ | ------------- |\n| 0   | Why the edits made under my username Hardcore Metallica Fan were reverted? | 0     | 0            | 0       | 0      | 0      | 0             |\n| 0   | I will mess you up                                                         | 1     | 0            | 0       | 1      | 0      | 0             |\n\nlabel_col will be a list of label column names. In this case it will be:\n\n```python\n['toxic','severe_toxic','obscene','threat','insult','identity_hate']\n```\n\n#### Tokenizer\n\nYou can either create a tokenizer object and pass it to DataBunch or you can pass the model name as tokenizer and DataBunch will automatically download and instantiate an appropriate tokenizer object.\n\nFor example for using XLNet base cased model, set tokenizer parameter to 'xlnet-base-cased'. DataBunch will automatically download and instantiate XLNetTokenizer with the vocabulary for xlnet-base-cased model.\n\n#### Model Type\n\nFast-Bert supports XLNet, RoBERTa and BERT based classification models. Set model type parameter value to **'bert'**, **roberta** or **'xlnet'** in order to initiate an appropriate databunch object.\n\n### 2. Create a Learner Object\n\nBertLearner is the ‘learner’ object that holds everything together. It encapsulates the key logic for the lifecycle of the model such as training, validation and inference.\n\nThe learner object will take the databunch created earlier as as input alongwith some of the other parameters such as location for one of the pretrained models, FP16 training, multi_gpu and multi_label options.\n\nThe learner class contains the logic for training loop, validation loop, optimiser strategies and key metrics calculation. This help the developers focus on their custom use-cases without worrying about these repetitive activities.\n\nAt the same time the learner object is flexible enough to be customised either via using flexible parameters or by creating a subclass of BertLearner and redefining relevant methods.\n\n```python\n\nfrom fast_bert.learner_cls import BertLearner\nfrom fast_bert.metrics import accuracy\nimport logging\n\nlogger = logging.getLogger()\ndevice_cuda = torch.device(\"cuda\")\nmetrics = [{'name': 'accuracy', 'function': accuracy}]\n\nlearner = BertLearner.from_pretrained_model(\n\t\t\t\t\t\tdatabunch,\n\t\t\t\t\t\tpretrained_path='bert-base-uncased',\n\t\t\t\t\t\tmetrics=metrics,\n\t\t\t\t\t\tdevice=device_cuda,\n\t\t\t\t\t\tlogger=logger,\n\t\t\t\t\t\toutput_dir=OUTPUT_DIR,\n\t\t\t\t\t\tfinetuned_wgts_path=None,\n\t\t\t\t\t\twarmup_steps=500,\n\t\t\t\t\t\tmulti_gpu=True,\n\t\t\t\t\t\tis_fp16=True,\n\t\t\t\t\t\tmulti_label=False,\n\t\t\t\t\t\tlogging_steps=50)\n```\n\n| parameter           | description                                                                                                                                                                                                                    |\n| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |\n| databunch           | Databunch object created earlier                                                                                                                                                                                               |\n| pretrained_path     | Directory for the location of the pretrained model files or the name of one of the pretrained models i.e. bert-base-uncased, xlnet-large-cased, etc                                                                            |\n| metrics             | List of metrics functions that you want the model to calculate on the validation set, e.g. accuracy, beta, etc                                                                                                                 |\n| device              | torch.device of type _cuda_ or _cpu_                                                                                                                                                                                           |\n| logger              | logger object                                                                                                                                                                                                                  |\n| output_dir          | Directory for model to save trained artefacts, tokenizer vocabulary and tensorboard files                                                                                                                                      |\n| finetuned_wgts_path | provide the location for fine-tuned language model (experimental feature)                                                                                                                                                      |\n| warmup_steps        | number of training warms steps for the scheduler                                                                                                                                                                               |\n| multi_gpu           | multiple GPUs available e.g. if running on AWS p3.8xlarge instance                                                                                                                                                             |\n| is_fp16             | FP16 training                                                                                                                                                                                                                  |\n| multi_label         | multilabel classification                                                                                                                                                                                                      |\n| logging_steps       | number of steps between each tensorboard metrics calculation. Set it to 0 to disable tensor flow logging. Keeping this value too low will lower the training speed as model will be evaluated each time the metrics are logged |\n\n### 3. Find the optimal learning rate\n\nThe learning rate is one of the most important hyperparameters for model training.  We have incorporated the learining rate finder that was proposed by Leslie Smith and then built into the fastai library. \n\n```python\nlearner.lr_find(start_lr=1e-5,optimizer_type='lamb')\n```\n\nThe code is heavily borrowed from David Silva's [pytorch-lr-finder library](https:\u002F\u002Fgithub.com\u002Fdavidtvs\u002Fpytorch-lr-finder). \n\n![Learning rate range test](https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Fappvision-ai_fast-bert_readme_8dd886374272.png)\n\n### 4. Train the model\n\n```python\nlearner.fit(epochs=6,\n\t\t\tlr=6e-5,\n\t\t\tvalidate=True, \t# Evaluate the model after each epoch\n\t\t\tschedule_type=\"warmup_cosine\",\n\t\t\toptimizer_type=\"lamb\")\n```\n\nFast-Bert now supports LAMB optmizer. Due to the speed of training, we have set LAMB as the default optimizer. You can switch back to AdamW by setting optimizer_type to 'adamw'.\n\n### 5. Save trained model artifacts\n\n```python\nlearner.save_model()\n```\n\nModel artefacts will be persisted in the output_dir\u002F'model_out' path provided to the learner object. Following files will be persisted:\n\n| File name               | description                                      |\n| ----------------------- | ------------------------------------------------ |\n| pytorch_model.bin       | trained model weights                            |\n| spiece.model            | sentence tokenizer vocabulary (for xlnet models) |\n| vocab.txt               | workpiece tokenizer vocabulary (for bert models) |\n| special_tokens_map.json | special tokens mappings                          |\n| config.json             | model config                                     |\n| added_tokens.json       | list of new tokens                               |\n\nAs the model artefacts are all stored in the same folder, you will be able to instantiate the learner object to run inference by pointing pretrained_path to this location.\n\n### 6. Model Inference\n\nIf you already have a Learner object with trained model instantiated, just call predict_batch method on the learner object with the list of text data:\n\n```python\ntexts = ['I really love the Netflix original movies',\n\t\t 'this movie is not worth watching']\npredictions = learner.predict_batch(texts)\n```\n\nIf you have persistent trained model and just want to run inference logic on that trained model, use the second approach, i.e. the predictor object.\n\n```python\nfrom fast_bert.prediction import BertClassificationPredictor\n\nMODEL_PATH = OUTPUT_DIR\u002F'model_out'\n\npredictor = BertClassificationPredictor(\n\t\t\t\tmodel_path=MODEL_PATH,\n\t\t\t\tlabel_path=LABEL_PATH, # location for labels.csv file\n\t\t\t\tmulti_label=False,\n\t\t\t\tmodel_type='xlnet',\n\t\t\t\tdo_lower_case=False,\n\t\t\t\tdevice=None) # set custom torch.device, defaults to cuda if available\n\n# Single prediction\nsingle_prediction = predictor.predict(\"just get me result for this text\")\n\n# Batch predictions\ntexts = [\n\t\"this is the first text\",\n\t\"this is the second text\"\n\t]\n\nmultiple_predictions = predictor.predict_batch(texts)\n```\n\n## Language Model Fine-tuning\n\nA useful approach to use BERT based models on custom datasets is to first finetune the language model task for the custom dataset, an apporach followed by fast.ai's ULMFit. The idea is to start with a pre-trained model and further train the model on the raw text of the custom dataset. We will use the masked LM task to finetune the language model.\n\nThis section will describe the usage of FastBert to finetune the language model.\n\n### 1. Import the necessary libraries\n\nThe necessary objects are stored in the files with '\\_lm' suffix.\n\n```python\n# Language model Databunch\nfrom fast_bert.data_lm import BertLMDataBunch\n# Language model learner\nfrom fast_bert.learner_lm import BertLMLearner\n\nfrom pathlib import Path\nfrom box import Box\n```\n\n### 2. Define parameters and setup datapaths\n\n```python\n# Box is a nice wrapper to create an object from a json dict\nargs = Box({\n    \"seed\": 42,\n    \"task_name\": 'imdb_reviews_lm',\n    \"model_name\": 'roberta-base',\n    \"model_type\": 'roberta',\n    \"train_batch_size\": 16,\n    \"learning_rate\": 4e-5,\n    \"num_train_epochs\": 20,\n    \"fp16\": True,\n    \"fp16_opt_level\": \"O2\",\n    \"warmup_steps\": 1000,\n    \"logging_steps\": 0,\n    \"max_seq_length\": 512,\n    \"multi_gpu\": True if torch.cuda.device_count() > 1 else False\n})\n\nDATA_PATH = Path('..\u002Flm_data\u002F')\nLOG_PATH = Path('..\u002Flogs')\nMODEL_PATH = Path('..\u002Flm_model_{}\u002F'.format(args.model_type))\n\nDATA_PATH.mkdir(exist_ok=True)\nMODEL_PATH.mkdir(exist_ok=True)\nLOG_PATH.mkdir(exist_ok=True)\n\n\n```\n\n### 3. Create DataBunch object\n\nThe BertLMDataBunch class contains a static method 'from_raw_corpus' that will take the list of raw texts and create DataBunch for the language model learner.\n\nThe method will at first preprocess the text list by removing html tags, extra spaces and more and then create files `lm_train.txt` and `lm_val.txt`. These files will be used for training and evaluating the language model finetuning task.\n\nThe next step will be to featurize the texts. The text will be tokenized, numericalized and split into blocks on 512 tokens (including special tokens).\n\n```python\ndatabunch_lm = BertLMDataBunch.from_raw_corpus(\n\t\t\t\t\tdata_dir=DATA_PATH,\n\t\t\t\t\ttext_list=texts,\n\t\t\t\t\ttokenizer=args.model_name,\n\t\t\t\t\tbatch_size_per_gpu=args.train_batch_size,\n\t\t\t\t\tmax_seq_length=args.max_seq_length,\n                    multi_gpu=args.multi_gpu,\n                    model_type=args.model_type,\n                    logger=logger)\n```\n\nAs this step can take some time based on the size of your custom dataset's text, the featurized data will be cached in pickled files in the data_dir\u002Flm_cache folder.\n\nThe next time, instead of using from_raw_corpus method, you may want to directly instantiate the DataBunch object as shown below:\n\n```python\ndatabunch_lm = BertLMDataBunch(\n\t\t\t\t\t\tdata_dir=DATA_PATH,\n\t\t\t\t\t\ttokenizer=args.model_name,\n                        batch_size_per_gpu=args.train_batch_size,\n                        max_seq_length=args.max_seq_length,\n                        multi_gpu=args.multi_gpu,\n                        model_type=args.model_type,\n                        logger=logger)\n```\n\n### 4. Create the LM Learner object\n\nBertLearner is the ‘learner’ object that holds everything together. It encapsulates the key logic for the lifecycle of the model such as training, validation and inference.\n\nThe learner object will take the databunch created earlier as as input alongwith some of the other parameters such as location for one of the pretrained models, FP16 training, multi_gpu and multi_label options.\n\nThe learner class contains the logic for training loop, validation loop, and optimizer strategies. This help the developers focus on their custom use-cases without worrying about these repetitive activities.\n\nAt the same time the learner object is flexible enough to be customized either via using flexible parameters or by creating a subclass of BertLearner and redefining relevant methods.\n\n```python\nlearner = BertLMLearner.from_pretrained_model(\n\t\t\t\t\t\t\tdataBunch=databunch_lm,\n\t\t\t\t\t\t\tpretrained_path=args.model_name,\n\t\t\t\t\t\t\toutput_dir=MODEL_PATH,\n\t\t\t\t\t\t\tmetrics=[],\n\t\t\t\t\t\t\tdevice=device,\n\t\t\t\t\t\t\tlogger=logger,\n\t\t\t\t\t\t\tmulti_gpu=args.multi_gpu,\n\t\t\t\t\t\t\tlogging_steps=args.logging_steps,\n\t\t\t\t\t\t\tfp16_opt_level=args.fp16_opt_level)\n```\n\n### 5. Train the model\n\n```python\nlearner.fit(epochs=6,\n\t\t\tlr=6e-5,\n\t\t\tvalidate=True, \t# Evaluate the model after each epoch\n\t\t\tschedule_type=\"warmup_cosine\",\n\t\t\toptimizer_type=\"lamb\")\n```\n\nFast-Bert now supports LAMB optmizer. Due to the speed of training, we have set LAMB as the default optimizer. You can switch back to AdamW by setting optimizer_type to 'adamw'.\n\n### 6. Save trained model artifacts\n\n```python\nlearner.save_model()\n```\n\nModel artefacts will be persisted in the output_dir\u002F'model_out' path provided to the learner object. Following files will be persisted:\n\n| File name               | description                                      |\n| ----------------------- | ------------------------------------------------ |\n| pytorch_model.bin       | trained model weights                            |\n| spiece.model            | sentence tokenizer vocabulary (for xlnet models) |\n| vocab.txt               | workpiece tokenizer vocabulary (for bert models) |\n| special_tokens_map.json | special tokens mappings                          |\n| config.json             | model config                                     |\n| added_tokens.json       | list of new tokens                               |\n\nThe pytorch_model.bin contains the finetuned weights and you can point the classification task learner object to this file throgh the `finetuned_wgts_path` parameter.\n\n## Amazon Sagemaker Support\n\nThe purpose of this library is to let you train and deploy production grade models. As transformer models require expensive GPUs to train, I have added support for training and deploying model on AWS SageMaker.\n\nThe repository contains the docker image and code for building BERT based classification models in Amazon SageMaker.\n\nPlease refer to my blog [Train and Deploy the Mighty BERT based NLP models using FastBert and Amazon SageMaker](https:\u002F\u002Ftowardsdatascience.com\u002Ftrain-and-deploy-mighty-transformer-nlp-models-using-fastbert-and-aws-sagemaker-cc4303c51cf3) that provides detailed explanation on using SageMaker with FastBert.\n\n## Citation\n\nPlease include a mention of [this library](https:\u002F\u002Fgithub.com\u002Fkaushaltrivedi\u002Ffast-bert) and HuggingFace [pytorch-transformers](https:\u002F\u002Fgithub.com\u002Fhuggingface\u002Fpytorch-transformers) library and a link to the present repository if you use this work in a published or open-source project.\n\nAlso include my blogs on this topic:\n\n- [Introducing FastBert — A simple Deep Learning library for BERT Models](https:\u002F\u002Fmedium.com\u002Fhuggingface\u002Fintroducing-fastbert-a-simple-deep-learning-library-for-bert-models-89ff763ad384)\n- [Multi-label Text Classification using BERT – The Mighty Transformer](https:\u002F\u002Fmedium.com\u002Fhuggingface\u002Fmulti-label-text-classification-using-bert-the-mighty-transformer-69714fa3fb3d)\n\n- [Train and Deploy the Mighty BERT based NLP models using FastBert and Amazon SageMaker](https:\u002F\u002Ftowardsdatascience.com\u002Ftrain-and-deploy-mighty-transformer-nlp-models-using-fastbert-and-aws-sagemaker-cc4303c51cf3)\n","# Fast-Bert\n\n[![Apache 2.0 许可证](https:\u002F\u002Fimg.shields.io\u002Fbadge\u002Flicense-Apache%202.0-blue.svg)](https:\u002F\u002Fgithub.com\u002Fdeepmipt\u002FDeepPavlov\u002Fblob\u002Fmaster\u002FLICENSE)\n[![PyPI 版本](https:\u002F\u002Fbadge.fury.io\u002Fpy\u002Ffast-bert.svg)](https:\u002F\u002Fbadge.fury.io\u002Fpy\u002Ffast-bert)\n![Python 3.6, 3.7](https:\u002F\u002Fimg.shields.io\u002Fbadge\u002Fpython-3.6%20%7C%203.7-green.svg)\n\n**新增 - 用于文本分类训练的学习率查找工具（感谢借用了 https:\u002F\u002Fgithub.com\u002Fdavidtvs\u002Fpytorch-lr-finder 的实现）**\n\n\n**支持 LAMB 优化器，以加快训练速度。**\n有关 LAMB 优化器的论文，请参阅 https:\u002F\u002Farxiv.org\u002Fabs\u002F1904.00962。\n\n**同时支持 BERT 和 XLNet，适用于多分类和多标签文本分类任务。**\n\nFast-Bert 是一个深度学习库，使开发者和数据科学家能够训练和部署基于 BERT 和 XLNet 的模型，用于从文本分类开始的自然语言处理任务。\n\nFastBert 的开发建立在优秀的 [Hugging Face BERT PyTorch 库](https:\u002F\u002Fgithub.com\u002Fhuggingface\u002Fpytorch-pretrained-BERT) 所提供的坚实基础上，并受到 [fast.ai](https:\u002F\u002Fgithub.com\u002Ffastai\u002Ffastai) 的启发，致力于让前沿的深度学习技术能够被更广泛的机器学习从业者所使用。\n\n通过 Fast-Bert，您将能够：\n\n1. 在您的自定义数据集上训练（更准确地说是微调）BERT、RoBERTa 和 XLNet 文本分类模型。\n\n2. 调整模型超参数，例如训练轮数、学习率、批量大小、优化器调度等。\n\n3. 保存并部署训练好的模型以进行推理（包括在 AWS Sagemaker 上部署）。\n\nFast-Bert 将同时支持多分类和多标签文本分类任务；未来还将支持其他自然语言处理任务，如命名实体识别、问答系统以及自定义语料库的微调。\n\n1.  **[BERT](https:\u002F\u002Fgithub.com\u002Fgoogle-research\u002Fbert)**（由 Google 发布），其论文为 Jacob Devlin、Ming-Wei Chang、Kenton Lee 和 Kristina Toutanova 撰写的《BERT：面向语言理解的深度双向 Transformer 预训练》（https:\u002F\u002Farxiv.org\u002Fabs\u002F1810.04805）。\n\n2)  **[XLNet](https:\u002F\u002Fgithub.com\u002Fzihangdai\u002Fxlnet\u002F)**（由 Google 和 CMU 共同发布），其论文为 Zhilin Yang*、Zihang Dai*、Yiming Yang、Jaime Carbonell、Ruslan Salakhutdinov 和 Quoc V. Le 撰写的《XLNet：通用自回归语言理解预训练》（https:\u002F\u002Farxiv.org\u002Fabs\u002F1906.08237）。\n\n3)  **[RoBERTa](https:\u002F\u002Farxiv.org\u002Fabs\u002F1907.11692)**（由 Facebook 发布），这是一种由 Yinhan Liu、Myle Ott、Naman Goyal、Jingfei Du 等人提出的鲁棒优化的 BERT 预训练方法。\n\n4)  **DistilBERT（由 HuggingFace 发布）**，其发布伴随着 Victor Sanh、Lysandre Debut 和 Thomas Wolf 撰写的博客文章《更小、更快、更便宜、更轻：推出 DistilBERT，即 BERT 的蒸馏版本》（https:\u002F\u002Fmedium.com\u002Fhuggingface\u002Fdistilbert-8cf3380435b5）。\n\n## 安装\n\n此仓库已在 Python 3.6 及以上版本上测试通过。\n\n### 使用 pip\n\n可以通过以下命令使用 pip 安装 PyTorch-Transformers：\n\n```bash\npip install fast-bert\n```\n\n### 从源代码安装\n\n克隆仓库并运行：\n\n```bash\npip install [--editable] .\n```\n\n或者\n\n```bash\npip install git+https:\u002F\u002Fgithub.com\u002Fkaushaltrivedi\u002Ffast-bert.git\n```\n\n此外，您还需要安装 NVIDIA Apex。\n\n```bash\ngit clone https:\u002F\u002Fgithub.com\u002FNVIDIA\u002Fapex\ncd apex\npip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" .\u002F\n```\n\n## 使用方法\n\n## 文本分类\n\n### 1. 创建 DataBunch 对象\n\nDataBunch 对象接收训练集、验证集和测试集的 CSV 文件，并将数据转换为 BERT、RoBERTa、DistilBERT 或 XLNet 所需的内部表示形式。该对象还会根据设备配置、批大小和最大序列长度实例化相应 的数据加载器。\n\n```python\nfrom fast_bert.data_cls import BertDataBunch\n\ndatabunch = BertDataBunch(DATA_PATH, LABEL_PATH,\n                          tokenizer='bert-base-uncased',\n                          train_file='train.csv',\n                          val_file='val.csv',\n                          label_file='labels.csv',\n                          text_col='text',\n                          label_col='label',\n                          batch_size_per_gpu=16,\n                          max_seq_length=512,\n                          multi_gpu=True,\n                          multi_label=False,\n                          model_type='bert')\n```\n\n#### train.csv 和 val.csv 的文件格式\n\n| 索引 | 文本                                                                                                                                                                                                                                                                                                                                | 标签 |\n| ----- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |\n| 0     | 看了其他评论，我惊讶于竟然没有任何警告提醒潜在观众，在租借这部烂片时他们会遇到什么。首先，我租这部电影时以为它只是个制作尚可的《夺宝奇兵》模仿作品。                                                | neg   |\n| 1     | 我已经看了前 17 集，这部剧真是太棒了！自从《新世纪福音战士》以来，我还没有对哪部动画如此着迷过。这部剧其实改编自一款成人游戏，我不确定之前是否有人这样做过。虽然我没玩过那款游戏，但据我所知，它与原作非常契合。 | pos   |\n| 2     | 这部电影绝对称得上是一部黑暗而 gritty 的杰作。也许我有些偏颇，因为种族隔离时代一直是我非常关注的领域。                                                                                                                                                                                                      | pos   |\n\n如果列名不是通常的 “text” 和 “labels”，则需要在 DataBunch 的 text_col 和 label_col 参数中指定这些列名。\n\n**labels.csv** 将包含所有唯一标签的列表。在本例中，该文件内容如下：\n\n```csv\npos\nneg\n```\n\n对于多标签分类，**labels.csv** 将包含所有可能的标签：\n\n```\ntoxic\nsevere_toxic\nobscene\nthreat\ninsult\nidentity_hate\n```\n\n随后，**train.csv** 文件将为每个标签设置一列，每列的值为 0 或 1。进行多标签分类时，请别忘了将 `BertDataBunch` 中的 `multi_label` 参数设置为 `True`。\n\n| id  | 文本                                                                       | toxic | severe_toxic | obscene | threat | insult | identity_hate |\n| --- | -------------------------------------------------------------------------- | ----- | ------------ | ------- | ------ | ------ | ------------- |\n| 0   | 为什么以我的用户名“硬核金属乐迷”所做的编辑被回退了？                       | 0     | 0            | 0       | 0      | 0      | 0             |\n| 0   | 我会让你吃不了兜着走                                                     | 1     | 0            | 0       | 1      | 0      | 0             |\n\nlabel_col 将是一个标签列名的列表。在本例中，它将是：\n\n```python\n['toxic','severe_toxic','obscene','threat','insult','identity_hate']\n```\n\n#### 分词器\n\n您可以创建一个分词器对象并将其传递给 DataBunch，也可以直接将模型名称作为分词器传入，DataBunch 会自动下载并实例化相应的分词器对象。\n\n例如，若要使用 XLNet base cased 模型，只需将 tokenizer 参数设置为 'xlnet-base-cased'。DataBunch 会自动下载并实例化适用于 xlnet-base-cased 模型词汇表的 XLNetTokenizer。\n\n#### 模型类型\n\nFast-Bert 支持基于 XLNet、RoBERTa 和 BERT 的分类模型。请将 model_type 参数的值设置为 **'bert'**、**roberta** 或 **'xlnet'**，以初始化相应的 DataBunch 对象。\n\n### 2. 创建 Learner 对象\n\nBertLearner 是负责整合所有组件的“学习器”对象。它封装了模型生命周期中的关键逻辑，例如训练、验证和推理。\n\n该学习器对象会将先前创建的数据批次（databunch）作为输入，并接受其他参数，如预训练模型的路径、是否使用 FP16 精度训练、多 GPU 支持以及多标签分类选项等。\n\nLearner 类包含了训练循环、验证循环、优化器策略以及关键指标计算的逻辑。这使得开发者可以专注于自定义用例，而无需担心这些重复性的操作。\n\n同时，Learner 对象也足够灵活，既可以通过调整参数进行定制，也可以通过继承 BertLearner 并重写相关方法来实现更高级的自定义功能。\n\n```python\nfrom fast_bert.learner_cls import BertLearner\nfrom fast_bert.metrics import accuracy\nimport logging\n\nlogger = logging.getLogger()\ndevice_cuda = torch.device(\"cuda\")\nmetrics = [{'name': 'accuracy', 'function': accuracy}]\n\nlearner = BertLearner.from_pretrained_model(\n    databunch,\n    pretrained_path='bert-base-uncased',\n    metrics=metrics,\n    device=device_cuda,\n    logger=logger,\n    output_dir=OUTPUT_DIR,\n    finetuned_wgts_path=None,\n    warmup_steps=500,\n    multi_gpu=True,\n    is_fp16=True,\n    multi_label=False,\n    logging_steps=50)\n```\n\n| 参数           | 描述                                                                                                                                                                                                                    |\n| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |\n| databunch           | 先前创建的数据批次对象                                                                                                                                                                                               |\n| pretrained_path     | 预训练模型文件的存储目录，或预训练模型的名称，例如 bert-base-uncased、xlnet-large-cased 等                                                                            |\n| metrics             | 您希望模型在验证集上计算的指标函数列表，例如准确率、F1 分数等                                                                                                                 |\n| device              | `torch.device` 类型，可为 `_cuda_` 或 `_cpu_`                                                                                                                                                                                           |\n| logger              | 日志记录对象                                                                                                                                                                                                                  |\n| output_dir          | 用于保存训练好的模型权重、分词器词汇表以及 TensorBoard 文件的目录                                                                                                                                      |\n| finetuned_wgts_path | 提供微调后的语言模型权重路径（实验性功能）                                                                                                                                                      |\n| warmup_steps        | 用于调度器的预热步数                                                                                                                                                                               |\n| multi_gpu           | 是否使用多 GPU 进行训练，例如在 AWS p3.8xlarge 实例上运行时                                                                                                                                             |\n| is_fp16             | 是否使用 FP16 精度训练                                                                                                                                                                                                                  |\n| multi_label         | 是否进行多标签分类                                                                                                                                                                                                      |\n| logging_steps       | 每次计算 TensorBoard 指标之间的步数。将其设置为 0 可禁用 TensorBoard 日志记录。如果该值设置得太小，每次记录指标时都会对模型进行评估，从而降低训练速度 |\n\n### 3. 寻找最优学习率\n\n学习率是模型训练中最重要的超参数之一。我们引入了由 Leslie Smith 提出并集成到 fastai 库中的学习率查找工具。\n\n```python\nlearner.lr_find(start_lr=1e-5, optimizer_type='lamb')\n```\n\n这段代码大量借鉴了 David Silva 的 [pytorch-lr-finder 库](https:\u002F\u002Fgithub.com\u002Fdavidtvs\u002Fpytorch-lr-finder)。\n\n![学习率范围测试](https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Fappvision-ai_fast-bert_readme_8dd886374272.png)\n\n### 4. 训练模型\n\n```python\nlearner.fit(epochs=6,\n            lr=6e-5,\n            validate=True,  # 每个 epoch 结束后评估模型\n            schedule_type=\"warmup_cosine\",\n            optimizer_type=\"lamb\")\n```\n\nFast-Bert 现在支持 LAMB 优化器。由于其训练速度较快，我们已将 LAMB 设置为默认优化器。您可以通过将 `optimizer_type` 设置为 `'adamw'` 来切换回 AdamW 优化器。\n\n### 5. 保存训练好的模型文件\n\n```python\nlearner.save_model()\n```\n\n模型文件将被持久化到传入学习器对象的 `output_dir\u002F'model_out'` 路径下。具体包含以下文件：\n\n| 文件名               | 描述                                      |\n| ----------------------- | ------------------------------------------------ |\n| pytorch_model.bin       | 训练好的模型权重                            |\n| spiece.model            | 句子分词器词汇表（适用于 XLNet 模型） |\n| vocab.txt               | 工作分词器词汇表（适用于 BERT 模型） |\n| special_tokens_map.json | 特殊标记映射                              |\n| config.json             | 模型配置                                  |\n| added_tokens.json       | 新增标记列表                               |\n\n由于所有模型文件都存储在同一目录下，您可以直接将 `pretrained_path` 指向该目录，即可实例化学习器对象进行推理。\n\n### 6. 模型推理\n\n如果你已经有一个实例化的、训练好的 Learner 对象，只需在该对象上调用 predict_batch 方法，并传入文本数据列表即可：\n\n```python\ntexts = ['我真的很喜欢 Netflix 的原创电影',\n\t\t '这部电影不值得看']\npredictions = learner.predict_batch(texts)\n```\n\n如果你有一个持久化的已训练模型，并且只想对该模型运行推理逻辑，则可以使用第二种方法，即通过 Predictor 对象。\n\n```python\nfrom fast_bert.prediction import BertClassificationPredictor\n\nMODEL_PATH = OUTPUT_DIR\u002F'model_out'\n\npredictor = BertClassificationPredictor(\n\t\t\t\tmodel_path=MODEL_PATH,\n\t\t\t\tlabel_path=LABEL_PATH, # labels.csv 文件的路径\n\t\t\t\tmulti_label=False,\n\t\t\t\tmodel_type='xlnet',\n\t\t\t\tdo_lower_case=False,\n\t\t\t\tdevice=None) # 可自定义 torch.device，若可用则默认为 cuda\n\n# 单条预测\nsingle_prediction = predictor.predict(\"请给我这条文本的结果\")\n\n# 批量预测\ntexts = [\n\t\"这是第一条文本\",\n\t\"这是第二条文本\"\n\t]\n\nmultiple_predictions = predictor.predict_batch(texts)\n```\n\n## 语言模型微调\n\n将基于 BERT 的模型应用于自定义数据集的一种有效方法是先针对该数据集对语言模型任务进行微调，这正是 fast.ai 的 ULMFit 所采用的方法。其核心思想是从预训练模型开始，然后在自定义数据集的原始文本上进一步训练模型。我们将使用掩码语言模型任务来微调语言模型。\n\n本节将介绍如何使用 FastBert 对语言模型进行微调。\n\n### 1. 导入必要的库\n\n所需的对象存储在带有 `_lm` 后缀的文件中。\n\n```python\n# 语言模型数据批次\nfrom fast_bert.data_lm import BertLMDataBunch\n# 语言模型学习器\nfrom fast_bert.learner_lm import BertLMLearner\n\nfrom pathlib import Path\nfrom box import Box\n```\n\n### 2. 定义参数并设置数据路径\n\n```python\n# Box 是一个方便的工具，可从 JSON 字典创建对象\nargs = Box({\n    \"seed\": 42,\n    \"task_name\": 'imdb_reviews_lm',\n    \"model_name\": 'roberta-base',\n    \"model_type\": 'roberta',\n    \"train_batch_size\": 16,\n    \"learning_rate\": 4e-5,\n    \"num_train_epochs\": 20,\n    \"fp16\": True,\n    \"fp16_opt_level\": \"O2\",\n    \"warmup_steps\": 1000,\n    \"logging_steps\": 0,\n    \"max_seq_length\": 512,\n    \"multi_gpu\": True if torch.cuda.device_count() > 1 else False\n})\n\nDATA_PATH = Path('..\u002Flm_data\u002F')\nLOG_PATH = Path('..\u002Flogs')\nMODEL_PATH = Path('..\u002Flm_model_{}\u002F'.format(args.model_type))\n\nDATA_PATH.mkdir(exist_ok=True)\nMODEL_PATH.mkdir(exist_ok=True)\nLOG_PATH.mkdir(exist_ok=True)\n```\n\n### 3. 创建 DataBunch 对象\n\nBertLMDataBunch 类包含一个静态方法 `from_raw_corpus`，该方法会接收原始文本列表，并为语言模型学习器创建 DataBunch。此方法首先会对文本列表进行预处理，去除 HTML 标签、多余空格等，然后生成 `lm_train.txt` 和 `lm_val.txt` 文件。这些文件将用于训练和评估语言模型微调任务。\n\n下一步是对文本进行特征提取：文本将被分词、转换为数字表示，并按 512 个标记（包括特殊标记）划分为多个块。\n\n```python\ndatabunch_lm = BertLMDataBunch.from_raw_corpus(\n\t\t\t\t\tdata_dir=DATA_PATH,\n\t\t\t\t\ttext_list=texts,\n\t\t\t\t\ttokenizer=args.model_name,\n\t\t\t\t\tbatch_size_per_gpu=args.train_batch_size,\n\t\t\t\t\tmax_seq_length=args.max_seq_length,\n                    multi_gpu=args.multi_gpu,\n                    model_type=args.model_type,\n                    logger=logger)\n```\n\n由于此步骤耗时可能较长，具体取决于自定义数据集文本的大小，因此特征化后的数据会被缓存到 data_dir\u002Flm_cache 文件夹中的 pickle 文件中。\n\n下次可以直接实例化 DataBunch 对象，而无需再次调用 from_raw_corpus 方法，如下所示：\n\n```python\ndatabunch_lm = BertLMDataBunch(\n\t\t\t\t\t\tdata_dir=DATA_PATH,\n\t\t\t\t\t\ttokenizer=args.model_name,\n                        batch_size_per_gpu=args.train_batch_size,\n                        max_seq_length=args.max_seq_length,\n                        multi_gpu=args.multi_gpu,\n                        model_type=args.model_type,\n                        logger=logger)\n```\n\n### 4. 创建 LM Learner 对象\n\nBertLearner 是一个“学习器”对象，它将所有内容整合在一起。它封装了模型生命周期中的关键逻辑，例如训练、验证和推理。Learner 对象会以先前创建的 DataBunch 作为输入，并接受其他参数，如预训练模型的位置、FP16 训练、多 GPU 支持以及多标签选项等。\n\nLearner 类包含了训练循环、验证循环和优化策略的逻辑，这有助于开发者专注于自己的用例，而无需担心这些重复性的工作。同时，Learner 对象也足够灵活，可以通过调整参数或通过创建 BertLearner 的子类并重新定义相关方法来进行定制。\n\n```python\nlearner = BertLMLearner.from_pretrained_model(\n\t\t\t\t\t\t\tdataBunch=databunch_lm,\n\t\t\t\t\t\t\tpretrained_path=args.model_name,\n\t\t\t\t\t\t\toutput_dir=MODEL_PATH,\n\t\t\t\t\t\t\tmetrics=[],\n\t\t\t\t\t\t\tdevice=device,\n\t\t\t\t\t\t\tlogger=logger,\n\t\t\t\t\t\t\tmulti_gpu=args.multi_gpu,\n\t\t\t\t\t\t\tlogging_steps=args.logging_steps,\n\t\t\t\t\t\t\tfp16_opt_level=args.fp16_opt_level)\n```\n\n### 5. 训练模型\n\n```python\nlearner.fit(epochs=6,\n\t\t\tlr=6e-5,\n\t\t\tvalidate=True, \t# 每个 epoch 结束后评估模型\n\t\t\tschedule_type=\"warmup_cosine\",\n\t\t\toptimizer_type=\"lamb\")\n```\n\nFast-Bert 现在支持 LAMB 优化器。由于其训练速度较快，我们已将其设为默认优化器。你可以通过将 optimizer_type 设置为 'adamw' 来切换回 AdamW 优化器。\n\n### 6. 保存训练好的模型文件\n\n```python\nlearner.save_model()\n```\n\n模型文件将被保存到 Learner 对象提供的 output_dir\u002F'model_out' 路径下。保存的文件包括：\n\n| 文件名               | 描述                                      |\n| ----------------------- | ------------------------------------------------ |\n| pytorch_model.bin       | 训练好的模型权重                            |\n| spiece.model            | 句子分词器词汇表（适用于 xlnet 模型）         |\n| vocab.txt               | 工作片段分词器词汇表（适用于 bert 模型）      |\n| special_tokens_map.json | 特殊标记映射                              |\n| config.json             | 模型配置                                  |\n| added_tokens.json       | 新增标记列表                              |\n\n其中，pytorch_model.bin 包含微调后的权重，你可以在分类任务的 Learner 对象中通过 `finetuned_wgts_path` 参数指向该文件。\n\n## Amazon SageMaker 支持\n\n本库旨在帮助您训练和部署生产级别的模型。由于 Transformer 模型的训练需要昂贵的 GPU，因此我添加了在 AWS SageMaker 上训练和部署模型的支持。\n\n该仓库包含用于在 Amazon SageMaker 中构建基于 BERT 的分类模型的 Docker 镜像和代码。\n\n请参阅我的博客 [使用 FastBert 和 Amazon SageMaker 训练并部署强大的 BERT 基础 NLP 模型](https:\u002F\u002Ftowardsdatascience.com\u002Ftrain-and-deploy-mighty-transformer-nlp-models-using-fastbert-and-aws-sagemaker-cc4303c51cf3)，其中详细介绍了如何将 SageMaker 与 FastBert 结合使用。\n\n## 引用\n\n如果您在已发表或开源项目中使用本工作，请务必提及 [此库](https:\u002F\u002Fgithub.com\u002Fkaushaltrivedi\u002Ffast-bert) 和 Hugging Face 的 [pytorch-transformers](https:\u002F\u002Fgithub.com\u002Fhuggingface\u002Fpytorch-transformers) 库，并附上本仓库的链接。\n\n同时，请引用我关于该主题的两篇博客：\n\n- [介绍 FastBert — 一个用于 BERT 模型的简单深度学习库](https:\u002F\u002Fmedium.com\u002Fhuggingface\u002Fintroducing-fastbert-a-simple-deep-learning-library-for-bert-models-89ff763ad384)\n- [使用 BERT 进行多标签文本分类——强大的 Transformer](https:\u002F\u002Fmedium.com\u002Fhuggingface\u002Fmulti-label-text-classification-using-bert-the-mighty-transformer-69714fa3fb3d)\n\n- [使用 FastBert 和 Amazon SageMaker 训练并部署强大的 BERT 基础 NLP 模型](https:\u002F\u002Ftowardsdatascience.com\u002Ftrain-and-deploy-mighty-transformer-nlp-models-using-fastbert-and-aws-sagemaker-cc4303c51cf3)","# Fast-Bert 快速上手指南\n\nFast-Bert 是一个基于 PyTorch 的深度学习库，旨在让开发者能够轻松训练和部署 BERT、RoBERTa、XLNet 及 DistilBERT 模型，主要用于文本分类任务（支持多分类和多标签分类）。它借鉴了 fast.ai 的设计理念，简化了微调流程。\n\n## 环境准备\n\n在开始之前，请确保您的开发环境满足以下要求：\n\n*   **操作系统**: Linux (推荐) 或 macOS。\n*   **Python 版本**: 3.6 或 3.7+。\n*   **GPU 支持**: 强烈建议使用 NVIDIA GPU 以加速训练。\n*   **前置依赖**:\n    *   PyTorch\n    *   **NVIDIA Apex**: Fast-Bert 依赖 Apex 进行混合精度训练（FP16）和使用 LAMB 优化器。必须预先安装。\n\n## 安装步骤\n\n### 1. 安装 NVIDIA Apex\nFast-Bert 需要编译版的 Apex 支持。请执行以下命令克隆并安装：\n\n```bash\ngit clone https:\u002F\u002Fgithub.com\u002FNVIDIA\u002Fapex\ncd apex\npip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" .\u002F\n```\n*(注：如果国内网络下载缓慢，可尝试配置 pip 国内镜像源，但 Apex 通常需要从源码编译)*\n\n### 2. 安装 Fast-Bert\n您可以选择通过 pip 直接安装或从源码安装。\n\n**方式一：使用 pip 安装（推荐）**\n```bash\npip install fast-bert\n```\n*国内用户建议使用清华或阿里镜像加速：*\n```bash\npip install fast-bert -i https:\u002F\u002Fpypi.tuna.tsinghua.edu.cn\u002Fsimple\n```\n\n**方式二：从源码安装**\n```bash\npip install git+https:\u002F\u002Fgithub.com\u002Fkaushaltrivedi\u002Ffast-bert.git\n```\n\n## 基本使用\n\n以下是使用 Fast-Bert 进行文本分类的最小化工作流程，包含数据准备、模型构建两个核心步骤。\n\n### 第一步：创建 DataBunch 对象\n`BertDataBunch` 负责加载 CSV 数据、处理分词（Tokenizer）并生成 DataLoader。\n\n假设您有以下文件：\n*   `train.csv`: 包含 `text` 和 `label` 列的训练数据。\n*   `val.csv`: 验证数据。\n*   `labels.csv`: 包含所有唯一标签列表（例如：`pos`, `neg`）。\n\n```python\nfrom fast_bert.data_cls import BertDataBunch\n\n# 定义路径\nDATA_PATH = '\u002Fpath\u002Fto\u002Fdata'\nLABEL_PATH = '\u002Fpath\u002Fto\u002Flabels'\nOUTPUT_DIR = '\u002Fpath\u002Fto\u002Foutput'\n\n# 初始化 DataBunch\ndatabunch = BertDataBunch(DATA_PATH, LABEL_PATH,\n                          tokenizer='bert-base-uncased', # 自动下载并实例化分词器\n                          train_file='train.csv',\n                          val_file='val.csv',\n                          label_file='labels.csv',\n                          text_col='text',               # 文本列名\n                          label_col='label',             # 标签列名\n                          batch_size_per_gpu=16,\n                          max_seq_length=512,\n                          multi_gpu=True,                # 是否启用多卡\n                          multi_label=False,             # 是否为多标签分类\n                          model_type='bert')             # 支持 'bert', 'roberta', 'xlnet'\n```\n\n**数据格式说明：**\n*   **单标签分类**: `labels.csv` 每行一个标签；CSV 数据文件中只需一列标签。\n*   **多标签分类**: `labels.csv` 列出所有可能标签；CSV 数据文件中需为每个标签设一列（值为 0 或 1），且需设置 `multi_label=True`，`label_col` 传入列名列表。\n\n### 第二步：创建 Learner 对象并开始训练\n`BertLearner` 封装了训练循环、验证逻辑和优化策略。\n\n```python\nimport torch\nimport logging\nfrom fast_bert.learner_cls import BertLearner\nfrom fast_bert.metrics import accuracy\n\n# 配置日志和设备\nlogger = logging.getLogger()\ndevice_cuda = torch.device(\"cuda\")\nmetrics = [{'name': 'accuracy', 'function': accuracy}]\n\n# 初始化 Learner\nlearner = BertLearner.from_pretrained_model(\n\t\t\t\t\t\tdatabunch,\n\t\t\t\t\t\tpretrained_path='bert-base-uncased', # 预训练模型名称或本地路径\n\t\t\t\t\t\tmetrics=metrics,\n\t\t\t\t\t\tdevice=device_cuda,\n\t\t\t\t\t\tlogger=logger,\n\t\t\t\t\t\toutput_dir=OUTPUT_DIR,\n\t\t\t\t\t\tfinetuned_wgts_path=None,\n\t\t\t\t\t\twarmup_steps=500,\n\t\t\t\t\t\tmulti_gpu=True,\n\t\t\t\t\t\tis_fp16=True,                        # 启用混合精度训练 (需 Apex)\n\t\t\t\t\t\tmulti_label=False,\n\t\t\t\t\t\tlogging_steps=50)\n\n# 开始训练\n# lr: 学习率, epochs: 轮数, validate_after_step: 每隔多少步验证一次\nlearner.fit(lr=2e-5, epochs=4, validate_after_step=200)\n```\n\n训练完成后，模型权重、分词器词汇表及 TensorBoard 日志将自动保存至 `output_dir` 指定目录，可直接用于推理或部署（如 AWS Sagemaker）。","某电商公司的数据科学团队需要快速构建一个能自动识别用户评论情感（正面\u002F负面）及具体投诉类别的多标签分类系统，以实时监控产品口碑。\n\n### 没有 fast-bert 时\n- **代码门槛高**：工程师需手动编写大量底层 PyTorch 代码来处理 BERT 的分词、DataLoader 构建及注意力掩码，开发周期长达数周。\n- **调优难度大**：缺乏内置的学习率查找器和先进的优化器支持，模型收敛慢且容易陷入局部最优，难以发挥预训练模型的全部潜力。\n- **部署流程繁琐**：从训练完成到导出模型并部署至 AWS SageMaker 等生产环境，需要额外编写复杂的转换脚本和接口代码。\n- **多模型适配难**：若想在 BERT、RoBERTa 和 XLNet 之间切换对比效果，需重构大量数据预处理逻辑，试错成本极高。\n\n### 使用 fast-bert 后\n- **开箱即用**：仅需几行代码即可通过 `BertDataBunch` 对象自动完成数据加载、分词及批次处理，将原型开发时间缩短至几天。\n- **智能调优**：直接调用内置的学习率查找器和 LAMB 优化器，显著加速训练过程并提升模型准确率，无需手动摸索超参数。\n- **一键部署**：训练好的模型可直接保存并无缝部署到 AWS SageMaker，大幅简化了从实验到生产的落地路径。\n- **灵活切换**：统一接口支持随时在 BERT、DistilBERT 或 XLNet 等不同架构间切换，轻松进行多模型性能比对与迭代。\n\nfast-bert 通过封装复杂的底层细节，让团队能以极简的代码高效完成从数据准备到模型部署的全流程，真正实现了前沿 NLP 技术的平民化应用。","https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Fappvision-ai_fast-bert_1708e0b4.png","appvision-ai","appvision.ai","https:\u002F\u002Foss.gittoolsai.com\u002Favatars\u002Fappvision-ai_f1ce9998.png","",null,"https:\u002F\u002Fgithub.com\u002Fappvision-ai",[83,87,91,95],{"name":84,"color":85,"percentage":86},"Python","#3572A5",71.1,{"name":88,"color":89,"percentage":90},"Jupyter Notebook","#DA5B0B",25.9,{"name":92,"color":93,"percentage":94},"Shell","#89e051",1.6,{"name":96,"color":97,"percentage":98},"Dockerfile","#384d54",1.4,1920,340,"2026-03-31T12:38:00","Apache-2.0",4,"未说明","需要 NVIDIA GPU（需安装 NVIDIA Apex 以支持 CUDA 扩展），具体显存大小和 CUDA 版本未说明",{"notes":107,"python":108,"dependencies":109},"该工具基于 PyTorch，必须安装 NVIDIA Apex 库以支持混合精度训练（FP16）和更快的训练速度（需从源码编译安装并启用 cpp_ext 和 cuda_ext 选项）。支持 BERT、RoBERTa、XLNet 和 DistilBERT 模型。虽然代码中展示了 multi_gpu 参数，但 README 未明确列出多卡的具体配置要求。","3.6, 3.7",[110,111,112,113],"pytorch-transformers (Hugging Face)","NVIDIA Apex","torch","cuda",[15],[116,67,117,118],"bert","fastai","transformers","2026-03-27T02:49:30.150509","2026-04-06T06:45:55.078315",[122,127,132,137,142,146],{"id":123,"question_zh":124,"answer_zh":125,"source_url":126},14979,"运行 Notebook 时遇到 CUDA 显存不足（Out of Memory）错误怎么办？","可以通过调整训练参数来解决显存不足的问题。具体方案包括：\n1. 重启 Jupyter Kernel 以释放缓存。\n2. 设置梯度累积步数 `gradient_accumulation = 8` 并将 `batch_size` 设为 8，这样等效批量大小为 64，但能适配显存。\n3. 保持序列长度 `sequence length` 为 256 或适当减小 `max_seq_length`。\n这些设置可以在不降低有效批处理大小的情况下减少单次显存占用。","https:\u002F\u002Fgithub.com\u002Fappvision-ai\u002Ffast-bert\u002Fissues\u002F8",{"id":128,"question_zh":129,"answer_zh":130,"source_url":131},14980,"调用 learner.save_model() 时出现 KeyError 或保存 tokenizer\u002Fvocab 文件失败如何解决？","该问题通常是因为 PyPI 上的 fast-bert 版本尚未包含修复代码。解决方案有两种：\n1. 使用包含修复代码的 Fork 版本进行安装：`git:\u002F\u002Fgithub.com\u002Faaronbriel\u002Ffast-bert@master#egg=fast_bert`。\n2. 或者直接从 GitHub 下载该 Fork 的源代码并在本地安装使用，直到官方库更新。","https:\u002F\u002Fgithub.com\u002Fappvision-ai\u002Ffast-bert\u002Fissues\u002F200",{"id":133,"question_zh":134,"answer_zh":135,"source_url":136},14981,"遇到 'BertDataBunch' object has no attribute 'model_type' 属性错误怎么办？","这通常是因为导入了错误的类导致的。请检查导入语句，确保从正确的模块导入 `BertDataBunch`：\n错误写法：`from fast_bert.data import BertDataBunch`\n正确写法：`from fast_bert.data_cls import BertDataBunch`\n修改导入路径后，对象将包含所需的 `model_type` 属性，错误即可解决。","https:\u002F\u002Fgithub.com\u002Fappvision-ai\u002Ffast-bert\u002Fissues\u002F220",{"id":138,"question_zh":139,"answer_zh":140,"source_url":141},14982,"训练验证阶段出现 TypeError: argmax() got an unexpected keyword argument 'axis' 或无法将 CUDA tensor 转换为 Numpy 的错误？","这是由于 PyTorch Tensor 直接在 GPU 上被传入 numpy.argmax 导致的兼容性问题。根据社区反馈，该问题在 fast-bert v1.5.0 版本中已得到修复。如果您使用的是 v1.6.0 或其他版本仍遇到此问题，建议尝试回退到 v1.5.0 版本，或者等待后续版本的修复。该错误通常发生在 validate() 阶段，可能不影响模型的训练和预测功能，但会中断验证流程。","https:\u002F\u002Fgithub.com\u002Fappvision-ai\u002Ffast-bert\u002Fissues\u002F152",{"id":143,"question_zh":144,"answer_zh":145,"source_url":126},14983,"如何正确加载预训练的 BERT 模型以避免 FileNotFoundError？","不要手动指定本地不存在的具体模型文件路径（如 `uncased_L-12_H-768_A-12`），这会导致文件找不到错误。建议在 tokenizer 和 `from_pretrained_model` 步骤中直接使用 Hugging Face 的模型标识符，例如使用 `'bert-base-uncased'` 代替具体的本地路径作为模型规格（model spec），系统会自动下载或加载对应的预训练权重。",{"id":147,"question_zh":148,"answer_zh":149,"source_url":126},14984,"在 Windows 上安装或使用 fast-bert 时遇到 apex 相关错误怎么办？","Windows 用户在使用 apex 混合精度训练时可能会遇到特定错误。可以参考 NVIDIA\u002Fapex 仓库的第 429 号 Issue (https:\u002F\u002Fgithub.com\u002FNVIDIA\u002Fapex\u002Fissues\u002F429) 中的解决方案。通常涉及重新编译 apex 或调整安装参数以适配 Windows 环境。",[151,156,161,166,171],{"id":152,"version":153,"summary_zh":154,"released_at":155},81830,"v1.8.0","版本 1.8.0 - 新增学习率查找器，已集成到学习器对象中。","2020-07-09T12:05:40",{"id":157,"version":158,"summary_zh":159,"released_at":160},81831,"v1.7.0","我们已切换到自动模型用于多分类任务。这使您可以针对文本分类任务训练任何预训练模型架构。","2020-04-14T21:58:36",{"id":162,"version":163,"summary_zh":164,"released_at":165},81832,"v1.6.0","现已支持抽取式摘要推理的初始版本，采用 fast-bert 风格。\n\n在不久后的某个版本中，您将能够使用基于自定义语料库微调的自定义语言模型作为编码器模型。","2019-12-22T14:27:08",{"id":167,"version":168,"summary_zh":169,"released_at":170},81833,"v1.5.1","修复了与 fastai 依赖相关的一些 bug。","2019-12-14T13:10:22",{"id":172,"version":173,"summary_zh":174,"released_at":175},81834,"v1.5.0","\u003Ch2>v1.5.0 版本新增了三种模型\u003C\u002Fh2>\n\n\u003Cli>\n\u003Cul>\u003Cb>ALBERT\u003C\u002Fb>（PyTorch）（由 Google Research 和芝加哥丰田技术研究所共同推出），随论文《ALBERT：用于语言表示自监督学习的轻量级 BERT》一同发布，作者为 Zhenzhong Lan、Mingda Chen、Sebastian Goodman、Kevin Gimpel、Piyush Sharma 和 Radu Soricut。\u003C\u002Ful>\n\u003Cul>\n\u003Cb>CamemBERT\u003C\u002Fb>（PyTorch）（由 Facebook AI Research、INRIA 和巴黎索邦大学联合开发），作为首个大规模 Transformer 语言模型。它与论文《CamemBERT：一款美味的法语语言模型》同时发布，作者为 Louis Martin、Benjamin Muller、Pedro Javier Ortiz Suarez、Yoann Dupont、Laurent Romary、Eric Villemonte de la Clergerie、Djame Seddah 和 Benoît Sagot。该模型由 @louismartin 在 @julien-c 的协助下添加。\n\u003C\u002Ful>\n\u003Cul>\n\u003Cb>DistilRoberta\u003C\u002Fb>（PyTorch），由 @VictorSanh 开发，是继 DistilBERT 和 DistilGPT-2 之后的第三款蒸馏模型。\n\u003C\u002Ful>\n\u003C\u002Fli>\n","2019-11-28T23:57:21"]