[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"similar-activeloopai--deeplake":3,"tool-activeloopai--deeplake":64},[4,17,27,35,43,56],{"id":5,"name":6,"github_repo":7,"description_zh":8,"stars":9,"difficulty_score":10,"last_commit_at":11,"category_tags":12,"status":16},3808,"stable-diffusion-webui","AUTOMATIC1111\u002Fstable-diffusion-webui","stable-diffusion-webui 是一个基于 Gradio 构建的网页版操作界面，旨在让用户能够轻松地在本地运行和使用强大的 Stable Diffusion 图像生成模型。它解决了原始模型依赖命令行、操作门槛高且功能分散的痛点，将复杂的 AI 绘图流程整合进一个直观易用的图形化平台。\n\n无论是希望快速上手的普通创作者、需要精细控制画面细节的设计师，还是想要深入探索模型潜力的开发者与研究人员，都能从中获益。其核心亮点在于极高的功能丰富度：不仅支持文生图、图生图、局部重绘（Inpainting）和外绘（Outpainting）等基础模式，还独创了注意力机制调整、提示词矩阵、负向提示词以及“高清修复”等高级功能。此外，它内置了 GFPGAN 和 CodeFormer 等人脸修复工具，支持多种神经网络放大算法，并允许用户通过插件系统无限扩展能力。即使是显存有限的设备，stable-diffusion-webui 也提供了相应的优化选项，让高质量的 AI 艺术创作变得触手可及。",162132,3,"2026-04-05T11:01:52",[13,14,15],"开发框架","图像","Agent","ready",{"id":18,"name":19,"github_repo":20,"description_zh":21,"stars":22,"difficulty_score":23,"last_commit_at":24,"category_tags":25,"status":16},1381,"everything-claude-code","affaan-m\u002Feverything-claude-code","everything-claude-code 是一套专为 AI 编程助手（如 Claude Code、Codex、Cursor 等）打造的高性能优化系统。它不仅仅是一组配置文件，而是一个经过长期实战打磨的完整框架，旨在解决 AI 代理在实际开发中面临的效率低下、记忆丢失、安全隐患及缺乏持续学习能力等核心痛点。\n\n通过引入技能模块化、直觉增强、记忆持久化机制以及内置的安全扫描功能，everything-claude-code 能显著提升 AI 在复杂任务中的表现，帮助开发者构建更稳定、更智能的生产级 AI 代理。其独特的“研究优先”开发理念和针对 Token 消耗的优化策略，使得模型响应更快、成本更低，同时有效防御潜在的攻击向量。\n\n这套工具特别适合软件开发者、AI 研究人员以及希望深度定制 AI 工作流的技术团队使用。无论您是在构建大型代码库，还是需要 AI 协助进行安全审计与自动化测试，everything-claude-code 都能提供强大的底层支持。作为一个曾荣获 Anthropic 黑客大奖的开源项目，它融合了多语言支持与丰富的实战钩子（hooks），让 AI 真正成长为懂上",138956,2,"2026-04-05T11:33:21",[13,15,26],"语言模型",{"id":28,"name":29,"github_repo":30,"description_zh":31,"stars":32,"difficulty_score":23,"last_commit_at":33,"category_tags":34,"status":16},2271,"ComfyUI","Comfy-Org\u002FComfyUI","ComfyUI 是一款功能强大且高度模块化的视觉 AI 引擎，专为设计和执行复杂的 Stable Diffusion 图像生成流程而打造。它摒弃了传统的代码编写模式，采用直观的节点式流程图界面，让用户通过连接不同的功能模块即可构建个性化的生成管线。\n\n这一设计巧妙解决了高级 AI 绘图工作流配置复杂、灵活性不足的痛点。用户无需具备编程背景，也能自由组合模型、调整参数并实时预览效果，轻松实现从基础文生图到多步骤高清修复等各类复杂任务。ComfyUI 拥有极佳的兼容性，不仅支持 Windows、macOS 和 Linux 全平台，还广泛适配 NVIDIA、AMD、Intel 及苹果 Silicon 等多种硬件架构，并率先支持 SDXL、Flux、SD3 等前沿模型。\n\n无论是希望深入探索算法潜力的研究人员和开发者，还是追求极致创作自由度的设计师与资深 AI 绘画爱好者，ComfyUI 都能提供强大的支持。其独特的模块化架构允许社区不断扩展新功能，使其成为当前最灵活、生态最丰富的开源扩散模型工具之一，帮助用户将创意高效转化为现实。",107662,"2026-04-03T11:11:01",[13,14,15],{"id":36,"name":37,"github_repo":38,"description_zh":39,"stars":40,"difficulty_score":23,"last_commit_at":41,"category_tags":42,"status":16},3704,"NextChat","ChatGPTNextWeb\u002FNextChat","NextChat 是一款轻量且极速的 AI 助手，旨在为用户提供流畅、跨平台的大模型交互体验。它完美解决了用户在多设备间切换时难以保持对话连续性，以及面对众多 AI 模型不知如何统一管理的痛点。无论是日常办公、学习辅助还是创意激发，NextChat 都能让用户随时随地通过网页、iOS、Android、Windows、MacOS 或 Linux 端无缝接入智能服务。\n\n这款工具非常适合普通用户、学生、职场人士以及需要私有化部署的企业团队使用。对于开发者而言，它也提供了便捷的自托管方案，支持一键部署到 Vercel 或 Zeabur 等平台。\n\nNextChat 的核心亮点在于其广泛的模型兼容性，原生支持 Claude、DeepSeek、GPT-4 及 Gemini Pro 等主流大模型，让用户在一个界面即可自由切换不同 AI 能力。此外，它还率先支持 MCP（Model Context Protocol）协议，增强了上下文处理能力。针对企业用户，NextChat 提供专业版解决方案，具备品牌定制、细粒度权限控制、内部知识库整合及安全审计等功能，满足公司对数据隐私和个性化管理的高标准要求。",87618,"2026-04-05T07:20:52",[13,26],{"id":44,"name":45,"github_repo":46,"description_zh":47,"stars":48,"difficulty_score":23,"last_commit_at":49,"category_tags":50,"status":16},2268,"ML-For-Beginners","microsoft\u002FML-For-Beginners","ML-For-Beginners 是由微软推出的一套系统化机器学习入门课程，旨在帮助零基础用户轻松掌握经典机器学习知识。这套课程将学习路径规划为 12 周，包含 26 节精炼课程和 52 道配套测验，内容涵盖从基础概念到实际应用的完整流程，有效解决了初学者面对庞大知识体系时无从下手、缺乏结构化指导的痛点。\n\n无论是希望转型的开发者、需要补充算法背景的研究人员，还是对人工智能充满好奇的普通爱好者，都能从中受益。课程不仅提供了清晰的理论讲解，还强调动手实践，让用户在循序渐进中建立扎实的技能基础。其独特的亮点在于强大的多语言支持，通过自动化机制提供了包括简体中文在内的 50 多种语言版本，极大地降低了全球不同背景用户的学习门槛。此外，项目采用开源协作模式，社区活跃且内容持续更新，确保学习者能获取前沿且准确的技术资讯。如果你正寻找一条清晰、友好且专业的机器学习入门之路，ML-For-Beginners 将是理想的起点。",84991,"2026-04-05T10:45:23",[14,51,52,53,15,54,26,13,55],"数据工具","视频","插件","其他","音频",{"id":57,"name":58,"github_repo":59,"description_zh":60,"stars":61,"difficulty_score":10,"last_commit_at":62,"category_tags":63,"status":16},3128,"ragflow","infiniflow\u002Fragflow","RAGFlow 是一款领先的开源检索增强生成（RAG）引擎，旨在为大语言模型构建更精准、可靠的上下文层。它巧妙地将前沿的 RAG 技术与智能体（Agent）能力相结合，不仅支持从各类文档中高效提取知识，还能让模型基于这些知识进行逻辑推理和任务执行。\n\n在大模型应用中，幻觉问题和知识滞后是常见痛点。RAGFlow 通过深度解析复杂文档结构（如表格、图表及混合排版），显著提升了信息检索的准确度，从而有效减少模型“胡编乱造”的现象，确保回答既有据可依又具备时效性。其内置的智能体机制更进一步，使系统不仅能回答问题，还能自主规划步骤解决复杂问题。\n\n这款工具特别适合开发者、企业技术团队以及 AI 研究人员使用。无论是希望快速搭建私有知识库问答系统，还是致力于探索大模型在垂直领域落地的创新者，都能从中受益。RAGFlow 提供了可视化的工作流编排界面和灵活的 API 接口，既降低了非算法背景用户的上手门槛，也满足了专业开发者对系统深度定制的需求。作为基于 Apache 2.0 协议开源的项目，它正成为连接通用大模型与行业专有知识之间的重要桥梁。",77062,"2026-04-04T04:44:48",[15,14,13,26,54],{"id":65,"github_repo":66,"name":67,"description_en":68,"description_zh":69,"ai_summary_zh":69,"readme_en":70,"readme_zh":71,"quickstart_zh":72,"use_case_zh":73,"hero_image_url":74,"owner_login":75,"owner_name":76,"owner_avatar_url":77,"owner_bio":78,"owner_company":79,"owner_location":79,"owner_email":79,"owner_twitter":75,"owner_website":80,"owner_url":81,"languages":82,"stars":114,"forks":115,"last_commit_at":116,"license":117,"difficulty_score":118,"env_os":119,"env_gpu":119,"env_ram":119,"env_deps":120,"category_tags":124,"github_topics":125,"view_count":23,"oss_zip_url":79,"oss_zip_packed_at":79,"status":16,"created_at":145,"updated_at":146,"faqs":147,"releases":175},3652,"activeloopai\u002Fdeeplake","deeplake","Deeplake is AI Data Runtime for Agents. It provides serverless postgres with a multimodal datalake, enabling scalable retrieval and training.","Deep Lake 是专为人工智能打造的“数据运行时”，旨在解决大模型应用与深度学习训练中多模态数据管理难、检索慢及扩展性差的痛点。无论是构建基于 LLM 的智能体，还是训练复杂的视觉模型，Deep Lake 都能将文本、图像、音频、视频乃至医疗影像等海量数据统一存储与管理。\n\n它特别适合 AI 开发者、数据科学家及研究人员使用。Deep Lake 的核心优势在于其独特的技术架构：支持无服务器（Serverless）部署，允许用户将数据直接存放在自己的云端（如 AWS S3、Google GCP 或 Azure），既保障了数据安全又降低了成本。其内置的懒加载机制配合原生压缩技术，让开发者能像操作本地 NumPy 数组一样高效处理云端海量数据，无需担心内存溢出。此外，Deep Lake 原生集成了 PyTorch 和 TensorFlow 的数据加载器，并无缝对接 LangChain、LlamaIndex 等主流框架，大幅简化了从数据版本控制、向量检索到模型训练的全流程，帮助用户轻松构建企业级 AI 应用。","\u003Cimg src=\"https:\u002F\u002Fstatic.scarf.sh\u002Fa.png?x-pxid=bc3c57b0-9a65-49fe-b8ea-f711c4d35b82\" \u002F>\u003Cp align=\"center\">\n     \u003Cimg src=\"https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Factiveloopai_deeplake_readme_c4b00ddc21ee.png\" width=\"400\"\u002F>\n\u003C\u002Fh1>\n\n\u003C\u002Fbr>\n\n\u003Ch1 align=\"center\">Deep Lake: Database for AI\u003C\u002Fh1>\n\n\u003Cp align=\"center\">\n    \u003Ca href=\"https:\u002F\u002Fpypi.org\u002Fproject\u002Fdeeplake\u002F\">\u003Cimg src=\"https:\u002F\u002Fbadge.fury.io\u002Fpy\u002Fdeeplake.svg\" alt=\"PyPI version\" height=\"18\">\u003C\u002Fa>\n    \u003Ca href=\"https:\u002F\u002Fpepy.tech\u002Fproject\u002Fdeeplake\">\u003Cimg src=\"https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Factiveloopai_deeplake_readme_150dfe653354.png\" alt=\"PyPI version\" height=\"18\">\u003C\u002Fa>\n  \u003Ch3 align=\"center\">\n   \u003Ca href=\"https:\u002F\u002Fdocs.deeplake.ai\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">\u003Cb>Docs\u003C\u002Fb>\u003C\u002Fa> •\n   \u003Ca href=\"https:\u002F\u002Fdocs.deeplake.ai\u002Flatest\u002Fgetting-started\u002Fquickstart\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">\u003Cb>Get Started\u003C\u002Fb>\u003C\u002Fa> •\n   \u003Ca href=\"https:\u002F\u002Fdocs.deeplake.ai\u002Flatest\u002Fapi\u002Fdataset\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">\u003Cb>API Reference\u003C\u002Fb>\u003C\u002Fa> •  \n   \u003Ca href=\"http:\u002F\u002Flearn.activeloop.ai\">\u003Cb>LangChain & VectorDBs Course\u003C\u002Fb>\u003C\u002Fa> •\n   \u003Ca href=\"https:\u002F\u002Fwww.activeloop.ai\u002Fresources\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">\u003Cb>Blog\u003C\u002Fb>\u003C\u002Fa> •\n   \u003Ca href=\"https:\u002F\u002Fwww.deeplake.ai\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">\u003Cb>Whitepaper\u003C\u002Fb>\u003C\u002Fa> •  \n  \u003Ca href=\"http:\u002F\u002Fslack.activeloop.ai\">\u003Cb>Slack\u003C\u002Fb>\u003C\u002Fa> •\n  \u003Ca href=\"https:\u002F\u002Ftwitter.com\u002Fintent\u002Ftweet?url=https%3A%2F%2Factiveloop.ai%2F&via=activeloopai&text=Deep%20Lake%20is%20the%20Database%20for%20all%20AI%20data.%20Check%20it%20out%21&hashtags=DeepLake%2Cactiveloop%2Copensource\">\u003Cb>Twitter\u003C\u002Fb>\u003C\u002Fa>\n \u003C\u002Fh3>\n\n## What is Deep Lake?\n\nDeep Lake is a Database for AI powered by a storage format optimized for deep-learning applications. Deep Lake can be used for:\n\n1. Storing and searching data plus vectors while building LLM applications\n2. Managing datasets while training deep learning models\n   \nDeep Lake simplifies the deployment of enterprise-grade LLM-based products by offering storage for all data types (embeddings, audio, text, videos, images, dicom, pdfs, annotations, [and more](https:\u002F\u002Fdocs.deeplake.ai\u002Flatest\u002Fapi\u002Ftypes\u002F)), querying and vector search, data streaming while training models at scale, data versioning and lineage, and integrations with popular tools such as LangChain, LlamaIndex, Weights & Biases, and many more. Deep Lake works with data of any size, it is serverless, and it enables you to store all of your data in your own cloud and in one place. Deep Lake is used by Intel, Bayer Radiology, Matterport, ZERO Systems, Red Cross, Yale, & Oxford. \n\n### Deep Lake includes the following features:\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Multi-Cloud Support (S3, GCP, Azure)\u003C\u002Fb>\u003C\u002Fsummary>\nUse one API to upload, download, and stream datasets to\u002Ffrom S3, Azure, GCP, Activeloop cloud, local storage, or in-memory storage. Compatible with any S3-compatible storage such as MinIO. \n\u003C\u002Fdetails>\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Native Compression with Lazy NumPy-like Indexing\u003C\u002Fb>\u003C\u002Fsummary>\nStore images, audio, and videos in their native compression. Slice, index, iterate, and interact with your data like a collection of NumPy arrays in your system's memory. Deep Lake lazily loads data only when needed, e.g., when training a model or running queries.\n\u003C\u002Fdetails>\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Dataloaders for Popular Deep Learning Frameworks\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake comes with built-in dataloaders for Pytorch and TensorFlow. Train your model with a few lines of code - we even take care of dataset shuffling. :)\n\u003C\u002Fdetails>\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Integrations with Powerful Tools\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake has integrations with \u003Ca href=\"https:\u002F\u002Fgithub.com\u002Fhwchase17\u002Flangchain\">Langchain\u003C\u002Fa> and \u003Ca href=\"https:\u002F\u002Fgithub.com\u002Fjerryjliu\u002Fllama_index\">LLamaIndex\u003C\u002Fa> as a vector store for LLM apps, \u003Ca href=\"https:\u002F\u002Fwandb.ai\u002F\">Weights & Biases\u003C\u002Fa> for data lineage during model training, \u003Ca href=\"https:\u002F\u002Fgithub.com\u002Fopen-mmlab\u002Fmmdetection\">MMDetection\u003C\u002Fa> for training object detection models, and \u003Ca href=\"https:\u002F\u002Fgithub.com\u002Fopen-mmlab\u002Fmmsegmentation\">MMSegmentation\u003C\u002Fa> for training semantic segmentation models.\n\u003C\u002Fdetails>\n\u003Cdetails>\n  \u003Csummary>\u003Cb>100+ most-popular image, video, and audio datasets available in seconds\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake community has uploaded \u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002Fdatasets\u002Factiveloop?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">100+ image, video and audio datasets\u003C\u002Fa> like \u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002Factiveloop\u002Fmnist-train?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">MNIST\u003C\u002Fa>, \u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002Factiveloop\u002Fcoco-train?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">COCO\u003C\u002Fa>,  \u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002Factiveloop\u002Fimagenet-train?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">ImageNet\u003C\u002Fa>,  \u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002Factiveloop\u002Fcifar100-test?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">CIFAR\u003C\u002Fa>,  \u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002Factiveloop\u002Fgtzan-genre?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">GTZAN\u003C\u002Fa> and others.\n\u003C\u002Fdetails>\n\u003C\u002Fdetails>\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Instant Visualization Support in the \u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">Deep Lake App\u003C\u002Fa>\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake datasets are instantly visualized with bounding boxes, masks, annotations, etc. in \u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">Deep Lake Visualizer\u003C\u002Fa> (see below).\n\u003C\u002Fdetails>\n\n[![Visualizer](https:\u002F\u002Fwww.linkpicture.com\u002Fq\u002FReadMe.gif \"Visualizer\")](https:\u002F\u002Fwww.youtube.com\u002Fwatch?v=SxsofpSIw3k)\n\n## 🚀 How to install Deep Lake\nDeep Lake can be installed using pip:\n```sh\npip install deeplake\n```\n\n### To access all of Deep Lake's features, please register in the [Deep Lake App](https:\u002F\u002Fapp.activeloop.ai\u002Fregister\u002F).\n\n## 🧠 Deep Lake Code Examples by Application\n\n### Vector Store Applications\nUsing Deep Lake as a Vector Store for building LLM applications:\n### - [Vector Store Quickstart](https:\u002F\u002Fdocs.deeplake.ai\u002Flatest\u002Fguides\u002Frag\u002F)\n### - [Vector Store Tutorials](https:\u002F\u002Fdocs-v3.activeloop.ai\u002Fexamples\u002Frag\u002Ftutorials)\n### - [LangChain Integration](https:\u002F\u002Fdocs-v3.activeloop.ai\u002Fexamples\u002Frag\u002Flangchain-integration)\n### - [LlamaIndex Integration](https:\u002F\u002Fdocs-v3.activeloop.ai\u002Fexamples\u002Frag\u002Fllamaindex-integration)\n### - [Image Similarity Search with Deep Lake](https:\u002F\u002Fdocs.deeplake.ai\u002Flatest\u002Fguides\u002Frag\u002F#5-integrating-image-embeddings-for-multi-modal-search)\n\n\n### Deep Learning Applications\nUsing Deep Lake for managing data while training Deep Learning models:\n### - [Deep Learning Quickstart](https:\u002F\u002Fdocs.deeplake.ai\u002Flatest\u002Fguides\u002Fdeep-learning\u002Fdeep-learning\u002F)\n### - [Tutorials for Training Models](https:\u002F\u002Fdocs-v3.activeloop.ai\u002Fexamples\u002Fdl\u002Ftutorials\u002Ftraining-models)\n\n## ⚙️ Integrations\n\nDeep Lake offers integrations with other tools in order to streamline your deep learning workflows. Current integrations include:\n\n* **LLM Apps**\n  * Use [Deep Lake as a vector store for LLM apps](https:\u002F\u002Fwww.activeloop.ai\u002Fresources\u002Fultimate-guide-to-lang-chain-deep-lake-build-chat-gpt-to-answer-questions-on-your-financial-data\u002F). Our integration combines the [Langchain](https:\u002F\u002Fgithub.com\u002Fhwchase17\u002Flangchain) [VectorStores API](https:\u002F\u002Fpython.langchain.com\u002Fen\u002Flatest\u002Freference\u002Fmodules\u002Fvectorstore.html?highlight=pinecone#langchain.vectorstores.DeepLake) with Deep Lake datasets as the underlying data storage. The integration is a serverless vector store that can be deployed locally or in a cloud of your choice.\n\n## 📚 Documentation\n\nGetting started guides, examples, tutorials, API reference, and other useful information can be found on our [documentation page](http:\u002F\u002Fdocs.deeplake.ai\u002F?utm_source=github&utm_medium=repo&utm_campaign=readme).\n\n## 🎓 For Students and Educators\nDeep Lake users can access and visualize a variety of popular datasets through a free integration with Deep Lake's App. Universities can get up to 1TB of data storage and 100,000 monthly queries on the Tensor Database for free per month. Chat in on [our website](https:\u002F\u002Factiveloop.ai): to claim the access!\n\n## 👩‍💻 Comparisons to Familiar Tools\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake vs Chroma \u003C\u002Fb>\u003C\u002Fsummary>\n  \nBoth Deep Lake & ChromaDB enable users to store and search vectors (embeddings) and offer integrations with LangChain and LlamaIndex. However, they are architecturally very different. ChromaDB is a Vector Database that can be deployed locally or on a server using Docker and will offer a hosted solution shortly. Deep Lake is a serverless Vector Store deployed on the user’s own cloud, locally, or in-memory. All computations run client-side, which enables users to support lightweight production apps in seconds. Unlike ChromaDB, Deep Lake’s data format can store raw data such as images, videos, and text, in addition to embeddings. ChromaDB is limited to light metadata on top of the embeddings and has no visualization. Deep Lake datasets can be visualized and version controlled. Deep Lake also has a performant dataloader for fine-tuning your Large Language Models.\n\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake vs Pinecone\u003C\u002Fb>\u003C\u002Fsummary>\n  \nBoth Deep Lake and Pinecone enable users to store and search vectors (embeddings) and offer integrations with LangChain and LlamaIndex. However, they are  architecturally very different. Pinecone is a fully-managed Vector Database that is optimized for highly demanding applications requiring a search for billions of vectors. Deep Lake is serverless. All computations run client-side, which enables users to get started in seconds. Unlike Pinecone, Deep Lake’s data format can store raw data such as images, videos, and text, in addition to embeddings. Deep Lake datasets can be visualized and version controlled. Pinecone is limited to light metadata on top of the embeddings and has no visualization. Deep Lake also has a performant dataloader for fine-tuning your Large Language Models.\n\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake vs Weaviate\u003C\u002Fb>\u003C\u002Fsummary>\n  \nBoth Deep Lake and Weaviate enable users to store and search vectors (embeddings) and offer integrations with LangChain and LlamaIndex. However, they are  architecturally very different. Weaviate is a Vector Database that can be deployed in a managed service or by the user via Kubernetes or Docker. Deep Lake is serverless. All computations run client-side, which enables users to support lightweight production apps in seconds. Unlike Weaviate, Deep Lake’s data format can store raw data such as images, videos, and text, in addition to embeddings. Deep Lake datasets can be visualized and version controlled. Weaviate is limited to light metadata on top of the embeddings and has no visualization. Deep Lake also has a performant dataloader for fine-tuning your Large Language Models.\n\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake vs DVC\u003C\u002Fb>\u003C\u002Fsummary>\n  \nDeep Lake and DVC offer dataset version control similar to git for data, but their methods for storing data differ significantly. Deep Lake converts and stores data as chunked compressed arrays, which enables rapid streaming to ML models, whereas DVC operates on top of data stored in less efficient traditional file structures. The Deep Lake format makes dataset versioning significantly easier compared to traditional file structures by DVC when datasets are composed of many files (i.e., many images). An additional distinction is that DVC primarily uses a command-line interface, whereas Deep Lake is a Python package. Lastly, Deep Lake offers an API to easily connect datasets to ML frameworks and other common ML tools and enables instant dataset visualization through [Activeloop's visualization tool](http:\u002F\u002Fapp.activeloop.ai\u002F?utm_source=github&utm_medium=repo&utm_campaign=readme).\n\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake vs MosaicML MDS format \u003C\u002Fb>\u003C\u002Fsummary>\n  \n* **Data Storage Format:** Deep Lake operates on a columnar storage format, whereas MDS utilizes a row-wise storage approach. This fundamentally impacts how data is read, written, and organized in each system.\n* **Compression:** Deep Lake offers a more flexible compression scheme, allowing control over both chunk-level and sample-level compression for each column or tensor. This feature eliminates the need for additional compressions like zstd, which would otherwise demand more CPU cycles for decompressing on top of formats like jpeg.\n* **Shuffling:** MDS currently offers more advanced shuffling strategies.\n* **Version Control & Visualization Support:** A notable feature of Deep Lake is its native version control and in-browser data visualization, a feature not present for MosaicML data format. This can provide significant advantages in managing, understanding, and tracking different versions of the data.\n\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake vs TensorFlow Datasets (TFDS)\u003C\u002Fb>\u003C\u002Fsummary>\n  \nDeep Lake and TFDS seamlessly connect popular datasets to ML frameworks. Deep Lake datasets are compatible with both PyTorch and TensorFlow, whereas TFDS are only compatible with TensorFlow. A key difference between Deep Lake and TFDS is that Deep Lake datasets are designed for streaming from the cloud, whereas TFDS must be downloaded locally prior to use. As a result, with Deep Lake, one can import datasets directly from TensorFlow Datasets and stream them either to PyTorch or TensorFlow. In addition to providing access to popular publicly available datasets, Deep Lake also offers powerful tools for creating custom datasets, storing them on a variety of cloud storage providers, and collaborating with others via simple API. TFDS is primarily focused on giving the public easy access to commonly available datasets, and management of custom datasets is not the primary focus. A full comparison article can be found [here](https:\u002F\u002Fwww.activeloop.ai\u002Fresources\u002Ftensor-flow-tf-data-activeloop-hub-how-to-implement-your-tensor-flow-data-pipelines-with-hub\u002F).\n\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake vs HuggingFace\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake and HuggingFace offer access to popular datasets, but Deep Lake primarily focuses on computer vision, whereas HuggingFace focuses on natural language processing. HuggingFace Transforms and other computational tools for NLP are not analogous to features offered by Deep Lake.\n\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake vs WebDatasets\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake and WebDatasets both offer rapid data streaming across networks. They have nearly identical steaming speeds because the underlying network requests and data structures are very similar. However, Deep Lake offers superior random access and shuffling, its simple API is in python instead of command-line, and Deep Lake enables simple indexing and modification of the dataset without having to recreate it.\n\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake vs Zarr\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake and Zarr both offer storage of data as chunked arrays. However, Deep Lake is primarily designed for returning data as arrays using a simple API, rather than actually storing raw arrays (even though that's also possible). Deep Lake stores data in use-case-optimized formats, such as jpeg or png for images, or mp4 for video, and the user treats the data as if it's an array, because Deep Lake handles all the data processing in between. Deep Lake offers more flexibility for storing arrays with dynamic shape (ragged tensors), and it provides several features that are not naively available in Zarr such as version control, data streaming, and connecting data to ML Frameworks.\n\n\u003C\u002Fdetails>\n\n## Community\n\nJoin our [**Slack community**](https:\u002F\u002Fslack.activeloop.ai) to learn more about unstructured dataset management using Deep Lake and to get help from the Activeloop team and other users.\n\nWe'd love your feedback by completing our 3-minute [**survey**](https:\u002F\u002Fforms.gle\u002FrLi4w33dow6CSMcm9).\n\nAs always, thanks to our amazing contributors!\n\n\u003Ca href=\"https:\u002F\u002Fgithub.com\u002Factiveloopai\u002Fdeeplake\u002Fgraphs\u002Fcontributors\">\n  \u003Cimg src=\"https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Factiveloopai_deeplake_readme_dedf6850e342.png\" \u002F>\n\u003C\u002Fa>\n\nMade with [contributors-img](https:\u002F\u002Fcontrib.rocks).\n\nPlease read [CONTRIBUTING.md](CONTRIBUTING.md) to get started with making contributions to Deep Lake.\n\n## README Badge\n\nUsing Deep Lake? Add a README badge to let everyone know:\n\n[![deeplake](https:\u002F\u002Fimg.shields.io\u002Fbadge\u002Fpowered%20by-Deep%20Lake%20-ff5a1f.svg)](https:\u002F\u002Fgithub.com\u002Factiveloopai\u002Fdeeplake)\n\n```markdown\n[![deeplake](https:\u002F\u002Fimg.shields.io\u002Fbadge\u002Fpowered%20by-Deep%20Lake%20-ff5a1f.svg)](https:\u002F\u002Fgithub.com\u002Factiveloopai\u002Fdeeplake)\n```\n\n## Disclaimers\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb> Dataset Licenses\u003C\u002Fb>\u003C\u002Fsummary>\n  \nDeep Lake users may have access to a variety of publicly available datasets. We do not host or distribute these datasets, vouch for their quality or fairness, or claim that you have a license to use the datasets. It is your responsibility to determine whether you have permission to use the datasets under their license.\n\nIf you're a dataset owner and do not want your dataset to be included in this library, please get in touch through a [GitHub issue](https:\u002F\u002Fgithub.com\u002Factiveloopai\u002Fdeeplake\u002Fissues\u002Fnew). Thank you for your contribution to the ML community!\n\n\u003C\u002Fdetails>\n\n## Citation\n\nIf you use Deep Lake in your research, please cite Activeloop using:\n\n```markdown\n@article{deeplake,\n  title = {Deep Lake: a Lakehouse for Deep Learning},\n  author = {Hambardzumyan, Sasun and Tuli, Abhinav and Ghukasyan, Levon and Rahman, Fariz and Topchyan, Hrant and Isayan, David and Harutyunyan, Mikayel and Hakobyan, Tatevik and Stranic, Ivo and Buniatyan, Davit},\n  url = {https:\u002F\u002Fwww.cidrdb.org\u002Fcidr2023\u002Fpapers\u002Fp69-buniatyan.pdf},\n  booktitle={Proceedings of CIDR},\n  year = {2023},\n}\n```\n\n\n## Acknowledgment\n\nThis technology was inspired by our research work at Princeton University. We would like to thank William Silversmith @SeungLab for his awesome [cloud-volume](https:\u002F\u002Fgithub.com\u002Fseung-lab\u002Fcloud-volume) tool.\n","\u003Cimg src=\"https:\u002F\u002Fstatic.scarf.sh\u002Fa.png?x-pxid=bc3c57b0-9a65-49fe-b8ea-f711c4d35b82\" \u002F>\u003Cp align=\"center\">\n     \u003Cimg src=\"https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Factiveloopai_deeplake_readme_c4b00ddc21ee.png\" width=\"400\"\u002F>\n\u003C\u002Fh1>\n\n\u003C\u002Fbr>\n\n\u003Ch1 align=\"center\">Deep Lake：面向人工智能的数据库\u003C\u002Fh1>\n\n\u003Cp align=\"center\">\n    \u003Ca href=\"https:\u002F\u002Fpypi.org\u002Fproject\u002Fdeeplake\u002F\">\u003Cimg src=\"https:\u002F\u002Fbadge.fury.io\u002Fpy\u002Fdeeplake.svg\" alt=\"PyPI版本\" height=\"18\">\u003C\u002Fa>\n    \u003Ca href=\"https:\u002F\u002Fpepy.tech\u002Fproject\u002Fdeeplake\">\u003Cimg src=\"https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Factiveloopai_deeplake_readme_150dfe653354.png\" alt=\"PyPI版本\" height=\"18\">\u003C\u002Fa>\n  \u003Ch3 align=\"center\">\n   \u003Ca href=\"https:\u002F\u002Fdocs.deeplake.ai\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">\u003Cb>文档\u003C\u002Fb>\u003C\u002Fa> •\n   \u003Ca href=\"https:\u002F\u002Fdocs.deeplake.ai\u002Flatest\u002Fgetting-started\u002Fquickstart\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">\u003Cb>快速入门\u003C\u002Fb>\u003C\u002Fa> •\n   \u003Ca href=\"https:\u002F\u002Fdocs.deeplake.ai\u002Flatest\u002Fapi\u002Fdataset\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">\u003Cb>API参考\u003C\u002Fb>\u003C\u002Fa> •  \n   \u003Ca href=\"http:\u002F\u002Flearn.activeloop.ai\">\u003Cb>LangChain与向量数据库课程\u003C\u002Fb>\u003C\u002Fa> •\n   \u003Ca href=\"https:\u002F\u002Fwww.activeloop.ai\u002Fresources\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">\u003Cb>博客\u003C\u002Fb>\u003C\u002Fa> •\n   \u003Ca href=\"https:\u002F\u002Fwww.deeplake.ai\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">\u003Cb>白皮书\u003C\u002Fb>\u003C\u002Fa> •  \n  \u003Ca href=\"http:\u002F\u002Fslack.activeloop.ai\">\u003Cb>Slack\u003C\u002Fb>\u003C\u002Fa> •\n  \u003Ca href=\"https:\u002F\u002Ftwitter.com\u002Fintent\u002Ftweet?url=https%3A%2F%2Factiveloop.ai%2F&via=activeloopai&text=Deep%20Lake%20is%20the%20Database%20for%20all%20AI%20data.%20Check%20it%20out%21&hashtags=DeepLake%2Cactiveloop%2Copensource\">\u003Cb>Twitter\u003C\u002Fb>\u003C\u002Fa>\n \u003C\u002Fh3>\n\n## 什么是 Deep Lake？\n\nDeep Lake 是一款由专为深度学习应用优化的存储格式驱动的人工智能数据库。Deep Lake 可用于：\n\n1. 构建 LLM 应用时存储和搜索数据及向量\n2. 训练深度学习模型时管理数据集\n\nDeep Lake 通过提供对所有数据类型（嵌入、音频、文本、视频、图像、DICOM、PDF、标注等）的存储、查询和向量搜索功能，以及大规模模型训练时的数据流式传输、数据版本控制和血缘追踪，并与 LangChain、LlamaIndex、Weights & Biases 等流行工具集成，简化了企业级基于 LLM 的产品的部署。Deep Lake 可处理任意规模的数据，采用无服务器架构，使您能够将所有数据存储在自己的云环境中并集中管理。Intel、Bayer Radiology、Matterport、ZERO Systems、红十字会、耶鲁大学和牛津大学等机构都在使用 Deep Lake。\n\n### Deep Lake 包含以下特性：\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>多云支持（S3、GCP、Azure）\u003C\u002Fb>\u003C\u002Fsummary>\n使用一个 API 即可将数据集上传、下载和流式传输到或从 S3、Azure、GCP、Activeloop 云、本地存储或内存存储中进行操作。兼容任何 S3 兼容的存储系统，例如 MinIO。\n\u003C\u002Fdetails>\n\u003Cdetails>\n  \u003Csummary>\u003Cb>原生压缩与惰性 NumPy 式索引\u003C\u002Fb>\u003C\u002Fsummary>\n以原生压缩格式存储图像、音频和视频。像操作系统内存中的 NumPy 数组集合一样对数据进行切片、索引、迭代和交互。Deep Lake 只会在需要时才懒加载数据，例如在训练模型或执行查询时。\n\u003C\u002Fdetails>\n\u003Cdetails>\n  \u003Csummary>\u003Cb>适用于主流深度学习框架的数据加载器\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake 自带 PyTorch 和 TensorFlow 的内置数据加载器。只需几行代码即可开始训练模型——我们甚至会自动完成数据集的打乱工作 :)。\n\u003C\u002Fdetails>\n\u003Cdetails>\n  \u003Csummary>\u003Cb>与强大工具的集成\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake 与 \u003Ca href=\"https:\u002F\u002Fgithub.com\u002Fhwchase17\u002Flangchain\">Langchain\u003C\u002Fa> 和 \u003Ca href=\"https:\u002F\u002Fgithub.com\u002Fjerryjliu\u002Fllama_index\">LLamaIndex\u003C\u002Fa> 集成，可用作 LLM 应用的向量存储；与 \u003Ca href=\"https:\u002F\u002Fwandb.ai\u002F\">Weights & Biases\u003C\u002Fa> 集成，用于模型训练过程中的数据血缘追踪；与 \u003Ca href=\"https:\u002F\u002Fgithub.com\u002Fopen-mmlab\u002Fmmdetection\">MMDetection\u003C\u002Fa> 集成，用于训练目标检测模型；与 \u003Ca href=\"https:\u002F\u002Fgithub.com\u002Fopen-mmlab\u002Fmmsegmentation\">MMSegmentation\u003C\u002Fa> 集成，用于训练语义分割模型。\n\u003C\u002Fdetails>\n\u003Cdetails>\n  \u003Csummary>\u003Cb>100 多个最受欢迎的图像、视频和音频数据集可在几秒钟内获取\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake 社区已上传了 \u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002Fdatasets\u002Factiveloop?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">100 多个图像、视频和音频数据集\u003C\u002Fa>,例如 \u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002Factiveloop\u002Fmnist-train?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">MNIST\u003C\u002Fa>、\u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002Factiveloop\u002Fcoco-train?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">COCO\u003C\u002Fa>、\u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002Factiveloop\u002Fimagenet-train?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">ImageNet\u003C\u002Fa>、\u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002Factiveloop\u002Fcifar100-test?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">CIFAR\u003C\u002Fa>、\u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002Factiveloop\u002Fgtzan-genre?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">GTZAN\u003C\u002Fa> 等。\n\u003C\u002Fdetails>\n\u003C\u002Fdetails>\n\u003Cdetails>\n  \u003Csummary>\u003Cb>在\u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">Deep Lake 应用程序\u003C\u002Fa>中即时可视化支持\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake 数据集可在 \u003Ca href=\"https:\u002F\u002Fapp.activeloop.ai\u002F?utm_source=github&utm_medium=github&utm_campaign=github_readme&utm_id=readme\">Deep Lake 可视化工具\u003C\u002Fa> 中立即以边界框、掩码、标注等形式进行可视化（见下文）。\n\u003C\u002Fdetails>\n\n[![可视化工具](https:\u002F\u002Fwww.linkpicture.com\u002Fq\u002FReadMe.gif \"可视化工具\")](https:\u002F\u002Fwww.youtube.com\u002Fwatch?v=SxsofpSIw3k)\n\n## 🚀 如何安装 Deep Lake\n可以通过 pip 安装 Deep Lake：\n```sh\npip install deeplake\n```\n\n### 若要访问 Deep Lake 的全部功能，请在 [Deep Lake 应用程序](https:\u002F\u002Fapp.activeloop.ai\u002Fregister\u002F) 中注册。\n\n## 🧠 按应用场景划分的 Deep Lake 代码示例\n\n### 向量存储应用\n将 Deep Lake 用作向量存储来构建 LLM 应用：\n### - [向量存储快速入门](https:\u002F\u002Fdocs.deeplake.ai\u002Flatest\u002Fguides\u002Frag\u002F)\n### - [向量存储教程](https:\u002F\u002Fdocs-v3.activeloop.ai\u002Fexamples\u002Frag\u002Ftutorials)\n### - [LangChain 集成](https:\u002F\u002Fdocs-v3.activeloop.ai\u002Fexamples\u002Frag\u002Flangchain-integration)\n### - [LlamaIndex 集成](https:\u002F\u002Fdocs-v3.activeloop.ai\u002Fexamples\u002Frag\u002Fllamaindex-integration)\n### - [使用 Deep Lake 进行图像相似度搜索](https:\u002F\u002Fdocs.deeplake.ai\u002Flatest\u002Fguides\u002Frag\u002F#5-integrating-image-embeddings-for-multi-modal-search)\n\n\n### 深度学习应用\n将 Deep Lake 用于深度学习模型训练时的数据管理：\n\n### - [深度学习快速入门](https:\u002F\u002Fdocs.deeplake.ai\u002Flatest\u002Fguides\u002Fdeep-learning\u002Fdeep-learning\u002F)\n### - [训练模型教程](https:\u002F\u002Fdocs-v3.activeloop.ai\u002Fexamples\u002Fdl\u002Ftutorials\u002Ftraining-models)\n\n## ⚙️ 集成\n\nDeep Lake 提供与其他工具的集成，以简化您的深度学习工作流。当前的集成包括：\n\n* **LLM 应用**\n  * 使用 [Deep Lake 作为 LLM 应用的向量存储](https:\u002F\u002Fwww.activeloop.ai\u002Fresources\u002Fultimate-guide-to-lang-chain-deep-lake-build-chat-gpt-to-answer-questions-on-your-financial-data\u002F)。我们的集成将 [Langchain](https:\u002F\u002Fgithub.com\u002Fhwchase17\u002Flangchain) 的 [VectorStores API](https:\u002F\u002Fpython.langchain.com\u002Fen\u002Flatest\u002Freference\u002Fmodules\u002Fvectorstore.html?highlight=pinecone#langchain.vectorstores.DeepLake) 与 Deep Lake 数据集相结合，作为底层数据存储。该集成是一个无服务器向量存储，可以部署在本地或您选择的云环境中。\n\n## 📚 文档\n\n入门指南、示例、教程、API 参考以及其他有用的信息，都可以在我们的 [文档页面](http:\u002F\u002Fdocs.deeplake.ai\u002F?utm_source=github&utm_medium=repo&utm_campaign=readme) 上找到。\n\n## 🎓 针对学生和教育工作者\n\nDeep Lake 用户可以通过与 Deep Lake App 的免费集成，访问并可视化各种流行的数据集。高校每月可免费获得高达 1TB 的数据存储空间以及 Tensor Database 上的 10 万次查询权限。请访问 [我们的网站](https:\u002F\u002Factiveloop.ai)，申请访问权限！\n\n## 👩‍💻 与常用工具的比较\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake 与 Chroma\u003C\u002Fb>\u003C\u002Fsummary>\n  \nDeep Lake 和 ChromaDB 都允许用户存储和搜索向量（嵌入），并提供与 LangChain 和 LlamaIndex 的集成。然而，它们的架构截然不同。ChromaDB 是一个向量数据库，可以通过 Docker 在本地或服务器上部署，并将在不久的将来推出托管解决方案。而 Deep Lake 是一种无服务器向量存储，可部署在用户自己的云环境、本地或内存中。所有计算都在客户端进行，这使得用户能够在几秒钟内支持轻量级的生产应用。与 ChromaDB 不同，Deep Lake 的数据格式不仅可以存储嵌入，还可以存储原始数据，如图像、视频和文本。ChromaDB 仅限于在嵌入之上添加少量元数据，且不支持可视化。Deep Lake 数据集则可以进行可视化和版本控制。此外，Deep Lake 还提供高效的 DataLoader，用于微调大型语言模型。\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake 与 Pinecone\u003C\u002Fb>\u003C\u002Fsummary>\n  \nDeep Lake 和 Pinecone 都允许用户存储和搜索向量（嵌入），并提供与 LangChain 和 LlamaIndex 的集成。然而，它们的架构截然不同。Pinecone 是一个完全托管的向量数据库，专为需要搜索数十亿个向量的高要求应用而优化。Deep Lake 则是无服务器的。所有计算都在客户端进行，使用户能够在几秒钟内开始使用。与 Pinecone 不同，Deep Lake 的数据格式不仅可以存储嵌入，还可以存储原始数据，如图像、视频和文本。Deep Lake 数据集可以进行可视化和版本控制。Pinecone 仅限于在嵌入之上添加少量元数据，且不支持可视化。Deep Lake 还拥有高效的 DataLoader，可用于微调大型语言模型。\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake 与 Weaviate\u003C\u002Fb>\u003C\u002Fsummary>\n  \nDeep Lake 和 Weaviate 都允许用户存储和搜索向量（嵌入），并提供与 LangChain 和 LlamaIndex 的集成。然而，它们的架构截然不同。Weaviate 是一个向量数据库，可以通过托管服务或由用户通过 Kubernetes 或 Docker 部署。而 Deep Lake 是无服务器的。所有计算都在客户端进行，使用户能够在几秒钟内支持轻量级的生产应用。与 Weaviate 不同，Deep Lake 的数据格式不仅可以存储嵌入，还可以存储原始数据，如图像、视频和文本。Deep Lake 数据集可以进行可视化和版本控制。Weaviate 仅限于在嵌入之上添加少量元数据，且不支持可视化。Deep Lake 还拥有高效的 DataLoader，用于微调大型语言模型。\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake 与 DVC\u003C\u002Fb>\u003C\u002Fsummary>\n  \nDeep Lake 和 DVC 都提供了类似于 Git 的数据集版本控制功能，但它们存储数据的方式却大相径庭。Deep Lake 将数据转换并存储为分块压缩数组，从而能够快速流式传输至机器学习模型；而 DVC 则基于效率较低的传统文件结构来操作数据。当数据集由大量文件组成时（例如许多图像），Deep Lake 的格式相比 DVC 的传统文件结构，显著简化了数据集版本控制。另一个区别在于，DVC 主要使用命令行界面，而 Deep Lake 是一个 Python 包。最后，Deep Lake 提供 API，可轻松将数据集连接到机器学习框架及其他常用工具，并通过 [Activeloop 的可视化工具](http:\u002F\u002Fapp.activeloop.ai\u002F?utm_source=github&utm_medium=repo&utm_campaign=readme) 实现数据的即时可视化。\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake 与 MosaicML MDS 格式\u003C\u002Fb>\u003C\u002Fsummary>\n  \n* **数据存储格式：** Deep Lake 采用列式存储格式，而 MDS 则采用行式存储方式。这一点从根本上影响了每个系统中数据的读取、写入和组织方式。\n* **压缩：** Deep Lake 提供更灵活的压缩方案，允许对每一列或张量进行分块级别和样本级别的压缩控制。这一特性无需额外的压缩算法（如 zstd），否则在 jpeg 等格式的基础上解压缩会消耗更多 CPU 资源。\n* **打乱顺序：** 目前，MDS 提供更为先进的打乱顺序策略。\n* **版本控制与可视化支持：** Deep Lake 的一个显著特点是其原生的版本控制和浏览器内数据可视化功能，而 MosaicML 数据格式并不具备这一特性。这在管理、理解和跟踪不同版本的数据方面具有明显优势。\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake 与 TensorFlow Datasets (TFDS)\u003C\u002Fb>\u003C\u002Fsummary>\n  \nDeep Lake 和 TFDS 都能无缝地将流行的数据集连接到机器学习框架中。Deep Lake 数据集同时兼容 PyTorch 和 TensorFlow，而 TFDS 只兼容 TensorFlow。Deep Lake 和 TFDS 的一个关键区别在于，Deep Lake 数据集专为从云端流式传输而设计，而 TFDS 则必须在使用前先下载到本地。因此，借助 Deep Lake，可以直接从 TensorFlow Datasets 导入数据集，并将其流式传输到 PyTorch 或 TensorFlow 中。除了提供对流行的公开数据集的访问外，Deep Lake 还提供了强大的工具来创建自定义数据集、将其存储在多种云存储提供商上，并通过简单的 API 与他人协作。TFDS 主要专注于为公众提供易于访问的常用数据集，而自定义数据集的管理并不是其主要关注点。完整的对比文章可以在这里找到 [链接](https:\u002F\u002Fwww.activeloop.ai\u002Fresources\u002Ftensor-flow-tf-data-activeloop-hub-how-to-implement-your-tensor-flow-data-pipelines-with-hub\u002F)。\n\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake 与 HuggingFace\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake 和 HuggingFace 都提供对流行数据集的访问，但 Deep Lake 主要专注于计算机视觉领域，而 HuggingFace 则专注于自然语言处理。HuggingFace Transforms 及其他用于 NLP 的计算工具，并不等同于 Deep Lake 所提供的功能。\n\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake 与 WebDatasets\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake 和 WebDatasets 都支持跨网络的快速数据流式传输。由于底层的网络请求和数据结构非常相似，两者的流式传输速度几乎相同。然而，Deep Lake 提供了更优越的随机访问和打乱功能，其简单 API 使用 Python 而非命令行，并且 Deep Lake 允许对数据集进行简单的索引和修改，而无需重新创建数据集。\n\n\u003C\u002Fdetails>\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb>Deep Lake 与 Zarr\u003C\u002Fb>\u003C\u002Fsummary>\nDeep Lake 和 Zarr 都支持将数据存储为分块数组。不过，Deep Lake 主要设计用于通过简单 API 将数据以数组形式返回，而不是真正存储原始数组（尽管这也可行）。Deep Lake 会将数据以针对特定用例优化的格式存储，例如图像使用 jpeg 或 png，视频使用 mp4，用户则可以像操作数组一样使用这些数据，因为 Deep Lake 会在背后完成所有的数据处理工作。Deep Lake 在存储具有动态形状的数组（稀疏张量）方面更加灵活，并且提供了 Zarr 中并不直接具备的多项功能，如版本控制、数据流式传输以及将数据与机器学习框架连接等。\n\n\u003C\u002Fdetails>\n\n\n\n## 社区\n\n加入我们的 [**Slack 社区**](https:\u002F\u002Fslack.activeloop.ai)，了解更多关于使用 Deep Lake 进行非结构化数据集管理的信息，并从 Activeloop 团队和其他用户那里获得帮助。\n\n我们非常期待您的反馈，请花 3 分钟时间完成我们的 [**调查问卷**](https:\u002F\u002Fforms.gle\u002FrLi4w33dow6CSMcm9)。\n\n一如既往，感谢我们出色的贡献者们！\n\n\u003Ca href=\"https:\u002F\u002Fgithub.com\u002Factiveloopai\u002Fdeeplake\u002Fgraphs\u002Fcontributors\">\n  \u003Cimg src=\"https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Factiveloopai_deeplake_readme_dedf6850e342.png\" \u002F>\n\u003C\u002Fa>\n\n由 [contributors-img](https:\u002F\u002Fcontrib.rocks) 制作。\n\n请阅读 [CONTRIBUTING.md](CONTRIBUTING.md)，开始为 Deep Lake 做出贡献。\n\n## README 徽章\n\n正在使用 Deep Lake 吗？添加一个 README 徽章，让大家知道：\n\n[![deeplake](https:\u002F\u002Fimg.shields.io\u002Fbadge\u002Fpowered%20by-Deep%20Lake%20-ff5a1f.svg)](https:\u002F\u002Fgithub.com\u002Factiveloopai\u002Fdeeplake)\n\n```markdown\n[![deeplake](https:\u002F\u002Fimg.shields.io\u002Fbadge\u002Fpowered%20by-Deep%20Lake%20-ff5a1f.svg)](https:\u002F\u002Fgithub.com\u002Factiveloopai\u002Fdeeplake)\n```\n\n## 免责声明\n\n\u003Cdetails>\n  \u003Csummary>\u003Cb> 数据集许可\u003C\u002Fb>\u003C\u002Fsummary>\n  \nDeep Lake 用户可以访问各种公开可用的数据集。但我们并不托管或分发这些数据集，也不对其质量或公平性背书，更不声称您拥有使用这些数据集的许可。您有责任根据其许可协议确定自己是否有权使用这些数据集。\n如果您是数据集的所有者，并且不希望您的数据集被包含在本库中，请通过 [GitHub 问题](https:\u002F\u002Fgithub.com\u002Factiveloopai\u002Fdeeplake\u002Fissues\u002Fnew) 与我们联系。感谢您为机器学习社区所做的贡献！\n\n\u003C\u002Fdetails>\n\n## 引用\n\n如果您在研究中使用了 Deep Lake，请引用 Activeloop 如下：\n\n```markdown\n@article{deeplake,\n  title = {Deep Lake: a Lakehouse for Deep Learning},\n  author = {Hambardzumyan, Sasun and Tuli, Abhinav and Ghukasyan, Levon and Rahman, Fariz and Topchyan, Hrant and Isayan, David and Harutyunyan, Mikayel and Hakobyan, Tatevik and Stranic, Ivo and Buniatyan, Davit},\n  url = {https:\u002F\u002Fwww.cidrdb.org\u002Fcidr2023\u002Fpapers\u002Fp69-buniatyan.pdf},\n  booktitle={Proceedings of CIDR},\n  year = {2023},\n}\n```\n\n\n## 致谢\n\n这项技术灵感来源于我们在普林斯顿大学的研究工作。我们要特别感谢 SeungLab 的 William Silversmith 及其出色的 [cloud-volume](https:\u002F\u002Fgithub.com\u002Fseung-lab\u002Fcloud-volume) 工具。","# Deep Lake 快速上手指南\n\nDeep Lake 是一款专为 AI 打造的数据库，支持存储和搜索向量及多模态数据（图像、视频、音频、文本等）。它适用于构建大语言模型（LLM）应用以及管理深度学习训练数据集，具备无服务器（Serverless）、原生压缩、懒加载及与 LangChain\u002FLlamaIndex 深度集成等特性。\n\n## 环境准备\n\n*   **操作系统**：支持 Linux、macOS 和 Windows。\n*   **Python 版本**：建议 Python 3.8 及以上版本。\n*   **前置依赖**：\n    *   已安装 `pip` 包管理工具。\n    *   （可选）若需使用云存储功能，请确保已配置对应云服务（AWS S3, Google GCP, Azure）的访问凭证。\n    *   （可选）若用于深度学习训练，建议预先安装 `pytorch` 或 `tensorflow`。\n\n## 安装步骤\n\n使用 pip 直接安装 Deep Lake：\n\n```bash\npip install deeplake\n```\n\n> **提示**：国内开发者若遇到下载速度慢的问题，可使用清华或阿里镜像源加速安装：\n> ```bash\n> pip install deeplake -i https:\u002F\u002Fpypi.tuna.tsinghua.edu.cn\u002Fsimple\n> ```\n\n若要解锁全部功能（如高级可视化、团队协作等），建议在 [Deep Lake App](https:\u002F\u002Fapp.activeloop.ai\u002Fregister\u002F) 注册账号并获取 API Token。\n\n## 基本使用\n\n以下示例展示如何创建一个本地数据集、添加图像与标签数据，并进行简单的读取操作。\n\n### 1. 创建数据集并写入数据\n\n```python\nimport deeplake\nimport numpy as np\n\n# 在本地创建一个名为 \"my_dataset\" 的新数据集\nds = deeplake.empty(\"hub:\u002F\u002Fmy_dataset\", overwrite=True)\n\n# 定义数据结构（张量）\nwith ds:\n    ds.create_tensor(\"images\", htype=\"image\", sample_compression=\"jpg\")\n    ds.create_tensor(\"labels\", htype=\"class_label\", class_names=[\"cat\", \"dog\"])\n\n    # 模拟添加数据 (此处使用随机生成的数组代替真实图片)\n    # 实际使用中可传入图片路径列表或直接加载图片\n    dummy_images = np.random.randint(0, 255, (10, 224, 224, 3), dtype=np.uint8)\n    dummy_labels = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1] # 0: cat, 1: dog\n\n    ds.images.append(dummy_images)\n    ds.labels.append(dummy_labels)\n\nprint(f\"数据集包含 {len(ds)} 条样本\")\n```\n\n### 2. 读取与索引数据\n\nDeep Lake 支持类似 NumPy 的切片和索引操作，数据仅在需要时懒加载。\n\n```python\n# 重新加载数据集（模拟在不同会话中）\nds = deeplake.load(\"hub:\u002F\u002Fmy_dataset\")\n\n# 访问第一条数据的图片和标签\nfirst_image = ds.images[0].numpy()\nfirst_label = ds.labels[0].numpy()\n\nprint(f\"第一张标签类别 ID: {first_label}\")\n\n# 批量迭代数据\nfor i in range(len(ds)):\n    img = ds.images[i].numpy()\n    label = ds.labels[i].numpy()\n    # 在此处进行预处理或推理\n```\n\n### 3. 集成 PyTorch 进行训练\n\nDeep Lake 内置了高性能 DataLoader，可直接对接 PyTorch：\n\n```python\nimport torch\nfrom deeplake import enterprise\n\n# 创建 PyTorch DataLoader\n# transform 函数可用于数据增强\ndef transform(sample):\n    return {\n        \"image\": sample[\"images\"].numpy().transpose(2, 0, 1) \u002F 255.0, # CHW format\n        \"label\": int(sample[\"labels\"].numpy())\n    }\n\ndl = ds.pytorch(num_workers=4, transform=transform, batch_size=4)\n\n# 在训练循环中使用\nfor batch in dl:\n    images, labels = batch[\"image\"], batch[\"label\"]\n    # 开始模型训练...\n```","某医疗 AI 团队正在构建一个基于多模态数据（CT 影像、病理报告文本及医生标注）的智能诊断助手，需同时支持模型训练与检索增强生成（RAG）。\n\n### 没有 deeplake 时\n- **数据存储割裂**：影像文件存于对象存储，向量和文本分散在不同数据库，维护多套系统导致架构复杂且成本高昂。\n- **训练效率低下**：加载大规模高清医学影像需先解压并全部读入内存，常因显存溢出（OOM）导致训练中断，预处理耗时极长。\n- **检索开发繁琐**：为构建 RAG 应用，需手动编写代码同步向量库与原始文件，难以保证数据版本一致性，调试困难。\n- **缺乏统一版本管理**：数据集更新后无法追溯历史版本，模型复现时经常因数据变动而导致结果不一致。\n\n### 使用 deeplake 后\n- **多模态统一存储**：利用 deeplake 将影像、文本、向量及标注信息存储在单一逻辑数据集中，直接对接 S3 等云存储，大幅简化架构。\n- **流式训练加速**：借助原生压缩与懒加载（Lazy Loading）特性，像操作 NumPy 数组一样直接流式读取云端数据，无需本地解压，训练启动速度提升数倍且杜绝 OOM。\n- **无缝集成 LLM 应用**：通过内置的 LangChain 和 LlamaIndex 接口，一键将多模态数据转化为向量存储，快速搭建高精度的医疗问答机器人。\n- **完整数据血缘**：自动记录数据版本与变更 lineage，确保每次模型训练或推理都能精确回溯到特定版本的数据集，满足医疗合规要求。\n\ndeeplake 通过统一的流式数据运行时，彻底解决了多模态 AI 应用在数据存储、高效训练及检索落地之间的断层难题。","https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Factiveloopai_deeplake_c4b00ddc.png","activeloopai","Activeloop","https:\u002F\u002Foss.gittoolsai.com\u002Favatars\u002Factiveloopai_1f15b27d.png","Activeloop team created Deep Lake, the database for AI. Stream, visualize, query, version all AI data - embeddings, images, video, text, etc. & use it with LLMs",null,"activeloop.ai","https:\u002F\u002Fgithub.com\u002Factiveloopai",[83,87,91,95,99,103,107,111],{"name":84,"color":85,"percentage":86},"C++","#f34b7d",52.4,{"name":88,"color":89,"percentage":90},"Python","#3572A5",38.8,{"name":92,"color":93,"percentage":94},"PLpgSQL","#336790",5.5,{"name":96,"color":97,"percentage":98},"CMake","#DA3434",2.2,{"name":100,"color":101,"percentage":102},"Shell","#89e051",0.9,{"name":104,"color":105,"percentage":106},"Makefile","#427819",0.1,{"name":108,"color":109,"percentage":110},"C","#555555",0,{"name":112,"color":113,"percentage":110},"Dockerfile","#384d54",9060,708,"2026-04-04T19:02:03","Apache-2.0",1,"未说明",{"notes":121,"python":119,"dependencies":122},"Deep Lake 是一个无服务器（serverless）工具，所有计算在客户端运行。支持将数据存储在本机、内存或用户自己的云端（S3, GCP, Azure, MinIO 等）。提供 PyTorch 和 TensorFlow 的内置数据加载器。建议注册 Deep Lake App 以访问完整功能。",[123],"numpy",[51,15,54,14,13,26],[126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144],"deep-learning","pytorch","ai","mlops","computer-vision","datalake","llm","large-language-models","vector-database","postgres","agent","agentic-rag","clawbot","filesystem","memory","multimodal","openclaw","rag","skill","2026-03-27T02:49:30.150509","2026-04-06T05:15:11.383232",[148,153,157,162,166,171],{"id":149,"question_zh":150,"answer_zh":151,"source_url":152},16735,"Deep Lake 数据加载器的性能为何比 PyTorch 原生加载器慢？是否有优化方案？","Deep Lake 并未针对本地数据进行显式优化，其大部分优化旨在提升流式应用（streaming applications）的性能，因此在处理本地数据集时，性能可能与 PyTorch 原生加载器相当或略慢。如果您需要更高的性能，建议尝试企业版 API（Enterprise API），该版本在本地数据集上的表现预计与 PyTorch 加载器相似。要使用企业版功能，您需要注册 Growth Plan 或更高版本的计划（含 14 天免费试用），并通过 `pip install deeplake[enterprise]` 安装。","https:\u002F\u002Fgithub.com\u002Factiveloopai\u002Fdeeplake\u002Fissues\u002F1931",{"id":154,"question_zh":155,"answer_zh":156,"source_url":152},16736,"如何访问 Deep Lake 的企业版 API 以进行性能评估？","企业版 API 仅对 Growth Plan 及以上计划的用户开放。最简单的试用方法是：1. 在 Deep Lake 官网 APP (https:\u002F\u002Fapp.activeloop.ai\u002F) 注册账号；2. 通过命令 `pip install deeplake[enterprise]` 安装包含企业功能的版本。注册后通常包含 14 天的免费试用期，可用于本地环境的性能基准测试。",{"id":158,"question_zh":159,"answer_zh":160,"source_url":161},16737,"如何向 htype 为 'json' 的张量中追加数据？支持哪些数据结构？","对于 `htype='json'` 的张量，您可以直接追加 Python 的字典（dict）或列表（list）结构。在内部，这些数据会被序列化为 JSON 字符串（使用 `json.dumps(...)`），然后以 `utf-8` 编码转换为字节。Deep Lake 使用了自定义的 JSON 编码器\u002F解码器，因此也支持嵌套在字典或列表中的 NumPy 数组。","https:\u002F\u002Fgithub.com\u002Factiveloopai\u002Fdeeplake\u002Fissues\u002F1735",{"id":163,"question_zh":164,"answer_zh":165,"source_url":161},16738,"如何在创建数据集时保留原始文件的元数据（如 URL、MD5 哈希值等）？","您可以在创建数据集时定义专门的文本张量（`htype=\"text\"`）或数值张量来存储元数据。例如，可以创建 `metadata\u002Fauthor`、`metadata\u002Fphoto_url`、`images\u002Ffile_md5`、`images\u002Fpixel_md5` 等张量。在上传数据时，将这些信息作为对应张量的样本一并追加。这样可以确保数据集包含完整的来源信息和完整性校验哈希，且重新计算像素 MD5 时能与原始文件匹配。",{"id":167,"question_zh":168,"answer_zh":169,"source_url":170},16739,"Deep Lake 是否支持嵌套的组（nested groups）结构来组织数据集？","是的，Deep Lake 支持嵌套的组结构。您可以通过在创建张量时使用斜杠分隔的路径名称（例如 `ds.create_tensor(\"images\u002Fimage\")` 或 `ds.create_tensor(\"metadata\u002Fauthor\")`）来自动创建层级结构。这使得您可以像文件系统一样组织数据，将图像、缩略图、元数据等分类存储在不同的组下。","https:\u002F\u002Fgithub.com\u002Factiveloopai\u002Fdeeplake\u002Fissues\u002F1439",{"id":172,"question_zh":173,"answer_zh":174,"source_url":170},16740,"如何在 Jupyter Notebook 中以更友好的格式查看数据集和张量的详细信息？","Deep Lake 已增强了对象的打印显示功能。直接输入数据集变量名（如 `ds`）可显示包含路径、样本数量及各张量详情（类型、dtype、形状、压缩方式）的表格。输入具体的张量（如 `ds.height`）会显示该张量的详细属性。在 Jupyter Notebook 环境中，这些输出会自动渲染为类似 Pandas DataFrame 的表格格式，并且对于图像类型的张量，切片操作（如 `ds[0:5]`）还可以直接可视化显示图像内容。",[176,181,186,191,195,200,205,210,215,220,225,230,235,240,245,250,255,260,265,270],{"id":177,"version":178,"summary_zh":179,"released_at":180},99007,"v4.5.2","## Deeplake v4.5.2 发行说明\n\n- 支持数据集和列元数据中的二进制数据。\n- 移除对 `libatomic` 的依赖。","2026-02-11T02:06:12",{"id":182,"version":183,"summary_zh":184,"released_at":185},99018,"v4.3.4","🎉 New Features\r\nPostgreSQL 18 Support\r\n\r\nPostgreSQL 18 compatibility - pg_deeplake now supports PostgreSQL 18\r\n\r\nPerformance Optimizations\r\n\r\nAdaptive backoff for worker threads - Implemented an intelligent backoff strategy for idle worker threads to reduce CPU usage\r\nCgroup-aware concurrency - Hardware concurrency now respects cgroup limits for better container performance\r\n\r\nAzure Integration\r\n\r\nAzure container name handling - Improved Azure getter with proper container name handling\r\n\r\n🐛 Bug Fixes\r\n\r\nBuild errors resolved - Fixed compilation errors in pg_deeplake\r\nVisualizer fixes - Resolved issues in the visualizer component\r\n\r\n🧹 Code Quality\r\n\r\nCode cleanup - Improved code organization with proper renames and refactoring\r\nModular architecture - Continued separation of concerns for better maintainability\r\n\r\n🧪 Testing\r\n\r\nEnhanced test coverage - Added comprehensive tests for pg_deeplake","2025-10-10T15:16:47",{"id":187,"version":188,"summary_zh":189,"released_at":190},99008,"v4.5.1","## Deeplake v4.5.1 发行说明                                                                                                      \r\n                                                                                                                                        \r\n  ### 性能                                                                                                                       \r\n                                                                                                                                        \r\n  - **mimalloc 内存分配器** — 将核心内存分配切换至 mimalloc，以降低开销                                                   \r\n  - **simdjson 日志解析** — 通过 simdjson 实现更快的 deeplog 反序列化\r\n  - **LZ4 压缩的 deeplog 日志** — 减小了 deeplog 的磁盘占用空间\r\n  - **字符串优化** 和 SIMD 加速的代码路径\r\n  - **减小 `nd::array` 的大小** — 每个数组的内存占用更小\r\n  - **原地下载** — 减少了聚类索引路径中的内存拷贝\r\n  - **异步预取器优先级修复**\r\n\r\n  ### 功能特性\r\n\r\n  - **空值支持** — 在数组、序列化以及日志回放中实现完整的空值处理\r\n  - **列统计信息** — 用于数据集的统计信息收集\r\n  - **链接内容直通** — 直接将内容传递给链接类型\r\n  - **二进制目录数据集** — 支持表目录类型的数据集创建\r\n\r\n  ### 错误修复\r\n\r\n  - **BM25 和倒排索引** — 修复了未索引的搜索及 `IN` 操作符查询问题\r\n  - **`exact_text` 搜索** — 修正了精确文本搜索\r\n  - **fork 安全性** — 修复了 `fork()` 后异步队列中的死锁问题\r\n  - **凭据刷新时机** — 修复了刷新时机相关的 bug\r\n  - **区域检测延迟** — 修复了区域自动检测过慢的问题\r\n  - **Link[bytes] 更新** — 修正了更新路径\r\n  - **Sanitizer 修复** — 解决了多个 UBSan\u002FTSan\u002FASan 问题","2026-02-07T05:15:26",{"id":192,"version":193,"summary_zh":79,"released_at":194},99009,"v4.5.0","2026-01-22T17:07:43",{"id":196,"version":197,"summary_zh":198,"released_at":199},99010,"v4.4.5","- 改进了对 NULL 值的 Deeplake 支持\n- 修复了同时创建行和列时出现的问题","2026-01-13T18:47:57",{"id":201,"version":202,"summary_zh":203,"released_at":204},99011,"v4.4.4","- Deeplake API 现已提供 CMake 和 pkg-config 文件，以实现更好的集成\n- 提升了存储访问性能\n- 重新优化了 PostgreSQL 数据入库流程，改进了批量入库功能","2025-12-14T07:20:07",{"id":206,"version":207,"summary_zh":208,"released_at":209},99012,"v4.4.3","第一个 C++ 库发布。","2025-12-03T02:02:54",{"id":211,"version":212,"summary_zh":213,"released_at":214},99013,"v4.4.1","# 发布说明\n\n## 🎉 新功能与改进\n\n**存储与 API 改进**\n- Storage list_dirs API - 由 @khustup2 新增的用于列出目录的 API 端点\n- 增加 mesh 类型支持 - 由 @khustup2 扩展的数据类型能力\n- PLY 可视化与本地文件列表 - 由 @azat-manukyan 增强的节点服务器可视化工具\n- 简单可视化工具 - 由 @khustup2 新增的用于数据检查的可视化组件\n- 凭证密钥属性 - 由 @activesoull 添加的用于只读数据集暴露凭证密钥的属性\n\n**数据管理**\n- 高效的版本锁定只读视图 - 由 @ashotshakhkyan 实现的优化只读数据集视图\n- 库预加载 - 由 @vahan-activeloop 提供的支持预加载库以提升性能\n\n## ⚡ 性能改进\n\n- LRU 缓存插入运行时提升 30% - 由 @ashotshakhkyan 实现的缓存操作显著提速\n- 数据集删除时清除缓存 - 由 @ashotshakhkyan 进行的小幅清理性能优化\n- 异步预取器重构 - 由 @ashotshakhkyan 简化批处理预取逻辑以提升性能\n\n## 🐛 错误修复\n\n**核心功能**\n- 修复 rebuild_branch 的进度报告 - 由 @khustup2 更正分支重建过程中的进度跟踪\n- 修复 deeplake 行为 - 由 @activesoull 解决的关键 deeplake 模块问题\n- 修复 JS 构建 - 由 @ashotshakhkyan 解决的 JavaScript 编译错误\n\n**数据处理**\n- 修正摘要生成 - 由 @activesoull 更正摘要生成逻辑\n- MP3 头部魔数 - 由 @azat-manukyan 修复音频文件格式检测问题\n\n**索引与存储**\n- 修正 compactor 的索引模式 - 由 @ashotshakhkyan 修复 compactor 的索引行为\n- 重复索引数据处理 - 由 @ashotshakhkyan 解决并发索引构建中的竞态条件\n- 修复警告 - 由 @activesoull 处理编译器警告\n\n## 🏗️ 架构与重构\n\n**代码组织**\n- 简化 rebuild_branch 逻辑 - 由 @khustup2 去除临时分支创建，使代码更整洁\n- 减少公开头文件 - 由 @ashotshakhkyan 将实现细节从公共 API 中隐藏\n- Deeplog include 清理 - 由 @ashotshakhkyan 整理包含语句，以提高可维护性\n\n**API 改进**\n- 添加合并提交信息 - 由 @activesoull 增强提交元数据","2025-11-21T10:52:52",{"id":216,"version":217,"summary_zh":218,"released_at":219},99014,"v4.3.3","🎉 新功能\npg_deeplake 功能增强\n\nCHAR\u002FBPCHAR 数据类型支持 - pg_deeplake 现在支持 CHAR 和 BPCHAR 数据类型\n库预加载 - 通过在 pg_deeplake 中预加载库，提升了性能\n表自动重新加载 - 如果在存储中未找到表，现在会自动重新加载表\n\n架构改进\n\n查询核心模块 - 将查询引擎与执行模块分离，以提高模块化程度\nGROUP BY 执行重构 - 将 GROUP BY 执行逻辑分离，以便后续功能增强\n\n🐛 错误修复\n\n修复了链接音频类型数据的处理问题 - 解决了链接音频类型数据的相关问题\n修复了编译错误 - 解决了 pg_deeplake 中的编译错误\n\n💬 用户体验\n\n更友好的错误提示 - 添加了针对缺少凭证密钥的清晰错误信息\n\n🧹 代码质量\n\n代码清理与重构 - 通过规范的重命名和清理，改善了代码组织\n\n🧪 测试\n\n测试覆盖率提升 - 为 pg_deeplake 增加了全面的测试用例\n","2025-09-29T07:14:30",{"id":221,"version":222,"summary_zh":223,"released_at":224},99015,"v3.9.52","# 🧭 变更内容\n* 允许使用 NumPy 2.x 版本。（#3075）@khustup2\n# ⚙️ 贡献者\n@activesoull 和 @khustup2\n","2025-09-05T13:50:53",{"id":226,"version":227,"summary_zh":228,"released_at":229},99016,"v4.4.0","# 发行说明\n\n## 新特性\n\n### 索引优化\n- **支持 JSON 数据索引** - 新增对 JSON 数据类型的索引支持，实现正确的数据处理和查询优化\n- **泛型列索引规范** - 新增在泛型列上指定索引类型的功能，提供更细粒度的控制\n- **索引创建进度条** - 在索引创建过程中添加可视化进度指示器，提升用户体验\n- **索引处理统一化** - 统一并简化了代码库中各处的索引处理逻辑，提高一致性和可维护性\n\n### 文档更新\n- **补充缺失的文档章节** - 扩充了此前未记录的功能和 API 的文档覆盖范围\n\n## 改进\n\n### 性能与可靠性\n- **修复主分支构建错误** - 解决了影响主开发分支的编译问题\n- **Windows 测试修复** - 通过针对性修复提升了 Windows 平台上的测试可靠性\n- **线程安全的取消机制** - 改进了取消机制，确保线程安全性\n- **重构分支内部 API** - 重新组织内部 API 结构，以提升架构清晰度和可维护性\n- **修复压缩期间重建索引的问题** - 更正了数据库压缩操作中索引重建的行为\n\n### 平台与构建支持\n- **Windows 平台 Python 绑定支持** - 新增对 Windows 平台上 Python 绑定构建的全面支持\n- **更新 libtiff 依赖** - 将 libtiff 库升级至最新版本，以提升兼容性和安全性\n\n## Bug 修复\n\n- **修复浏览器崩溃问题** - 解决了特定场景下导致浏览器崩溃的严重问题\n- **修复无效迭代器问题** - 修正了可能导致运行时错误的无效迭代器处理问题\n- **其他通用修复** - 包括多项稳定性改进及边缘情况的处理\n\n## 贡献者\n\n感谢 @activesoull、@vahan-activeloop、@khustup2 和 @ashotshakhkyan 对本次发布的贡献。","2025-10-24T19:03:18",{"id":231,"version":232,"summary_zh":233,"released_at":234},99017,"v4.3.5","数据类型支持\n\n链接到字节支持 - 新增对链接到字节数据类型的支持（DEE-299）\n灵活的文本列处理 - 文本列现在允许追加未知 dtype 的数据，以提高兼容性\n\n🐛 错误修复\n\ndeeplake.exists 错误处理 - 修复了 deeplake.exists 中的异常处理逻辑，使其能够优雅地处理错误，而非抛出异常（DEE-300）\nPython PNG 链接问题 - 解决了 Python PNG 链接相关的问题\n\n📚 文档\n\n文档更新 - 改进并更新了文档（DEE-36）\n\n🧹 代码质量\n\n通用改进 - 进行了多项小修复和代码优化","2025-10-10T17:50:47",{"id":236,"version":237,"summary_zh":238,"released_at":239},99019,"v4.3.0","# Deeplake 4.3.0\r\n\r\nDeeplake 4.3.0 is a major update bringing many new features to the Deeplake ecosystem.\r\n\r\n## New Data and Index Types\r\n- Complete revisit of `Sequence` types to support visual and structured data\r\n- Video type support is now available in Deeplake, supporting MP4 and MKV videos with H264 codec and providing fast random access to video frames\r\n- Indexing for numeric types, enabling fast queries for numeric comparisons in TQL, including `IN` and `BETWEEN` operations\r\n- Significant improvements to textual index types, providing faster search without requiring index regeneration\r\n\r\n## Data Import\u002FExport\r\n- Fully rewritten `from_csv` function with support for large CSV files\r\n- New `to_csv` API to export Deeplake datasets\u002Fviews to CSV format\r\n\r\n## Python Typing\r\n- Support for specifying Python builtin types when defining dataset schemas\r\n- Support for using Pydantic Models as dataset schemas\r\n- Enriched async operations typing, to support better integration with linters and IDEs.\r\n\r\n## Improvements and Bug Fixes\r\n- Improved TQL data fetching and linear scan performance for non-indexed columns\r\n- Better memory usage tracking to prevent out-of-memory errors\r\n- Various stability improvements and bug fixes\r\n\r\n## Compatibility Notice\r\nDeeplake 4.3.0 is backward compatible with datasets created in v4.2.x. However, datasets created or modified with v4.3.0 cannot be opened with v4.2.x versions due to internal format enhancements. We recommend upgrading all environments to v4.3.0 when working with shared datasets.","2025-08-29T08:50:37",{"id":241,"version":242,"summary_zh":243,"released_at":244},99020,"v3.9.51","# 🧭 What's Changed\r\n* Update version for 3.9.51 release (#3073) @activesoull\r\n* [DEE-180] fix dependency resolver issue (#3072) @activesoull\r\n# ⚙️ Who Contributes\r\n@activesoull \r\n","2025-08-07T07:23:51",{"id":246,"version":247,"summary_zh":248,"released_at":249},99021,"v3.9.50","# 🧭 What's Changed\r\n* Updated versions for 3.9.50 release (#3069) @activesoull\r\n* Refactor frame extraction logic and improve frame rate retrieval in L… (#3071) @azat-manukyan\r\n# ⚙️ Who Contributes\r\n@activesoull and @azat-manukyan\r\n","2025-08-01T11:57:05",{"id":251,"version":252,"summary_zh":253,"released_at":254},99022,"v3.9.46","# 🧭 What's Changed\r\n* Base mmsegmentation dataset (#3060) @activesoull\r\nadded a base mmsegmentation dataset class where in case of using mmseg 1.x version you need to inherit from to make integration workflow smoother\r\n\r\n* 3.9.46 (#3058) @activesoull\r\n# ⚙️ Who Contributes\r\n@activesoull\r\n","2025-07-09T14:33:09",{"id":256,"version":257,"summary_zh":258,"released_at":259},99023,"v4.2.14","- Added `Audio` type\r\n- Fixed conversion from V3 for `Polygon` type","2025-07-08T23:03:45",{"id":261,"version":262,"summary_zh":263,"released_at":264},99024,"v4.2.12","- Added autocommit.\r\n- Improved parallel ingestion and compaction functionality\r\n- Added zlib compression support for `SegmentMask` type\r\n- Added embeddings indexing support for `float16` and `bfloat16`\r\n- Added AVG function to TQL\r\n- Improved SAMPLE BY in TQL\r\n- Added images support for labelbox integration","2025-07-02T11:03:04",{"id":266,"version":267,"summary_zh":268,"released_at":269},99025,"v4.2.8","- Added `from_csv`, extended `from_parquet` to support content bytes\r\n- Added zlib and lz4 sample compression for `SegmentMask` type","2025-06-10T14:39:31",{"id":271,"version":272,"summary_zh":273,"released_at":274},99026,"v3.9.45","# 🧭 What's Changed\r\n* added sampler setter function (#3057) @activesoull\r\n* Hierarchical namespace (#3056) @activesoull\r\n* Updated versions for 3.9.45 release (#3045) @activesoull\r\n# ⚙️ Who Contributes\r\n@activesoull\r\n","2025-06-10T10:56:59"]