[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"similar-clemlesne--scrape-it-now":3,"tool-clemlesne--scrape-it-now":64},[4,17,27,35,43,56],{"id":5,"name":6,"github_repo":7,"description_zh":8,"stars":9,"difficulty_score":10,"last_commit_at":11,"category_tags":12,"status":16},3808,"stable-diffusion-webui","AUTOMATIC1111\u002Fstable-diffusion-webui","stable-diffusion-webui 是一个基于 Gradio 构建的网页版操作界面，旨在让用户能够轻松地在本地运行和使用强大的 Stable Diffusion 图像生成模型。它解决了原始模型依赖命令行、操作门槛高且功能分散的痛点，将复杂的 AI 绘图流程整合进一个直观易用的图形化平台。\n\n无论是希望快速上手的普通创作者、需要精细控制画面细节的设计师，还是想要深入探索模型潜力的开发者与研究人员，都能从中获益。其核心亮点在于极高的功能丰富度：不仅支持文生图、图生图、局部重绘（Inpainting）和外绘（Outpainting）等基础模式，还独创了注意力机制调整、提示词矩阵、负向提示词以及“高清修复”等高级功能。此外，它内置了 GFPGAN 和 CodeFormer 等人脸修复工具，支持多种神经网络放大算法，并允许用户通过插件系统无限扩展能力。即使是显存有限的设备，stable-diffusion-webui 也提供了相应的优化选项，让高质量的 AI 艺术创作变得触手可及。",162132,3,"2026-04-05T11:01:52",[13,14,15],"开发框架","图像","Agent","ready",{"id":18,"name":19,"github_repo":20,"description_zh":21,"stars":22,"difficulty_score":23,"last_commit_at":24,"category_tags":25,"status":16},1381,"everything-claude-code","affaan-m\u002Feverything-claude-code","everything-claude-code 是一套专为 AI 编程助手（如 Claude Code、Codex、Cursor 等）打造的高性能优化系统。它不仅仅是一组配置文件，而是一个经过长期实战打磨的完整框架，旨在解决 AI 代理在实际开发中面临的效率低下、记忆丢失、安全隐患及缺乏持续学习能力等核心痛点。\n\n通过引入技能模块化、直觉增强、记忆持久化机制以及内置的安全扫描功能，everything-claude-code 能显著提升 AI 在复杂任务中的表现，帮助开发者构建更稳定、更智能的生产级 AI 代理。其独特的“研究优先”开发理念和针对 Token 消耗的优化策略，使得模型响应更快、成本更低，同时有效防御潜在的攻击向量。\n\n这套工具特别适合软件开发者、AI 研究人员以及希望深度定制 AI 工作流的技术团队使用。无论您是在构建大型代码库，还是需要 AI 协助进行安全审计与自动化测试，everything-claude-code 都能提供强大的底层支持。作为一个曾荣获 Anthropic 黑客大奖的开源项目，它融合了多语言支持与丰富的实战钩子（hooks），让 AI 真正成长为懂上",138956,2,"2026-04-05T11:33:21",[13,15,26],"语言模型",{"id":28,"name":29,"github_repo":30,"description_zh":31,"stars":32,"difficulty_score":23,"last_commit_at":33,"category_tags":34,"status":16},2271,"ComfyUI","Comfy-Org\u002FComfyUI","ComfyUI 是一款功能强大且高度模块化的视觉 AI 引擎，专为设计和执行复杂的 Stable Diffusion 图像生成流程而打造。它摒弃了传统的代码编写模式，采用直观的节点式流程图界面，让用户通过连接不同的功能模块即可构建个性化的生成管线。\n\n这一设计巧妙解决了高级 AI 绘图工作流配置复杂、灵活性不足的痛点。用户无需具备编程背景，也能自由组合模型、调整参数并实时预览效果，轻松实现从基础文生图到多步骤高清修复等各类复杂任务。ComfyUI 拥有极佳的兼容性，不仅支持 Windows、macOS 和 Linux 全平台，还广泛适配 NVIDIA、AMD、Intel 及苹果 Silicon 等多种硬件架构，并率先支持 SDXL、Flux、SD3 等前沿模型。\n\n无论是希望深入探索算法潜力的研究人员和开发者，还是追求极致创作自由度的设计师与资深 AI 绘画爱好者，ComfyUI 都能提供强大的支持。其独特的模块化架构允许社区不断扩展新功能，使其成为当前最灵活、生态最丰富的开源扩散模型工具之一，帮助用户将创意高效转化为现实。",107662,"2026-04-03T11:11:01",[13,14,15],{"id":36,"name":37,"github_repo":38,"description_zh":39,"stars":40,"difficulty_score":23,"last_commit_at":41,"category_tags":42,"status":16},3704,"NextChat","ChatGPTNextWeb\u002FNextChat","NextChat 是一款轻量且极速的 AI 助手，旨在为用户提供流畅、跨平台的大模型交互体验。它完美解决了用户在多设备间切换时难以保持对话连续性，以及面对众多 AI 模型不知如何统一管理的痛点。无论是日常办公、学习辅助还是创意激发，NextChat 都能让用户随时随地通过网页、iOS、Android、Windows、MacOS 或 Linux 端无缝接入智能服务。\n\n这款工具非常适合普通用户、学生、职场人士以及需要私有化部署的企业团队使用。对于开发者而言，它也提供了便捷的自托管方案，支持一键部署到 Vercel 或 Zeabur 等平台。\n\nNextChat 的核心亮点在于其广泛的模型兼容性，原生支持 Claude、DeepSeek、GPT-4 及 Gemini Pro 等主流大模型，让用户在一个界面即可自由切换不同 AI 能力。此外，它还率先支持 MCP（Model Context Protocol）协议，增强了上下文处理能力。针对企业用户，NextChat 提供专业版解决方案，具备品牌定制、细粒度权限控制、内部知识库整合及安全审计等功能，满足公司对数据隐私和个性化管理的高标准要求。",87618,"2026-04-05T07:20:52",[13,26],{"id":44,"name":45,"github_repo":46,"description_zh":47,"stars":48,"difficulty_score":23,"last_commit_at":49,"category_tags":50,"status":16},2268,"ML-For-Beginners","microsoft\u002FML-For-Beginners","ML-For-Beginners 是由微软推出的一套系统化机器学习入门课程，旨在帮助零基础用户轻松掌握经典机器学习知识。这套课程将学习路径规划为 12 周，包含 26 节精炼课程和 52 道配套测验，内容涵盖从基础概念到实际应用的完整流程，有效解决了初学者面对庞大知识体系时无从下手、缺乏结构化指导的痛点。\n\n无论是希望转型的开发者、需要补充算法背景的研究人员，还是对人工智能充满好奇的普通爱好者，都能从中受益。课程不仅提供了清晰的理论讲解，还强调动手实践，让用户在循序渐进中建立扎实的技能基础。其独特的亮点在于强大的多语言支持，通过自动化机制提供了包括简体中文在内的 50 多种语言版本，极大地降低了全球不同背景用户的学习门槛。此外，项目采用开源协作模式，社区活跃且内容持续更新，确保学习者能获取前沿且准确的技术资讯。如果你正寻找一条清晰、友好且专业的机器学习入门之路，ML-For-Beginners 将是理想的起点。",84991,"2026-04-05T10:45:23",[14,51,52,53,15,54,26,13,55],"数据工具","视频","插件","其他","音频",{"id":57,"name":58,"github_repo":59,"description_zh":60,"stars":61,"difficulty_score":10,"last_commit_at":62,"category_tags":63,"status":16},3128,"ragflow","infiniflow\u002Fragflow","RAGFlow 是一款领先的开源检索增强生成（RAG）引擎，旨在为大语言模型构建更精准、可靠的上下文层。它巧妙地将前沿的 RAG 技术与智能体（Agent）能力相结合，不仅支持从各类文档中高效提取知识，还能让模型基于这些知识进行逻辑推理和任务执行。\n\n在大模型应用中，幻觉问题和知识滞后是常见痛点。RAGFlow 通过深度解析复杂文档结构（如表格、图表及混合排版），显著提升了信息检索的准确度，从而有效减少模型“胡编乱造”的现象，确保回答既有据可依又具备时效性。其内置的智能体机制更进一步，使系统不仅能回答问题，还能自主规划步骤解决复杂问题。\n\n这款工具特别适合开发者、企业技术团队以及 AI 研究人员使用。无论是希望快速搭建私有知识库问答系统，还是致力于探索大模型在垂直领域落地的创新者，都能从中受益。RAGFlow 提供了可视化的工作流编排界面和灵活的 API 接口，既降低了非算法背景用户的上手门槛，也满足了专业开发者对系统深度定制的需求。作为基于 Apache 2.0 协议开源的项目，它正成为连接通用大模型与行业专有知识之间的重要桥梁。",77062,"2026-04-04T04:44:48",[15,14,13,26,54],{"id":65,"github_repo":66,"name":67,"description_en":68,"description_zh":69,"ai_summary_zh":69,"readme_en":70,"readme_zh":71,"quickstart_zh":72,"use_case_zh":73,"hero_image_url":74,"owner_login":75,"owner_name":76,"owner_avatar_url":77,"owner_bio":78,"owner_company":79,"owner_location":80,"owner_email":81,"owner_twitter":82,"owner_website":83,"owner_url":84,"languages":85,"stars":102,"forks":103,"last_commit_at":104,"license":105,"difficulty_score":10,"env_os":106,"env_gpu":107,"env_ram":107,"env_deps":108,"category_tags":120,"github_topics":121,"view_count":23,"oss_zip_url":82,"oss_zip_packed_at":82,"status":16,"created_at":127,"updated_at":128,"faqs":129,"releases":159},2329,"clemlesne\u002Fscrape-it-now","scrape-it-now","Web scraper made for AI and simplicity in mind. It runs as a CLI that can be parallelized and outputs high-quality markdown content.","scrape-it-now 是一款专为 AI 应用打造的命令行网页抓取工具，旨在以极简的方式获取高质量数据。它主要解决了传统爬虫在应对动态网页、广告干扰及数据清洗时的痛点，能够直接将复杂的网页内容转换为结构清晰、语义连贯的 Markdown 格式，并自动完成去重与分块处理，为后续的大模型训练或语义搜索提供“开箱即用”的优质语料。\n\n这款工具特别适合开发者、AI 研究人员及数据工程师使用，尤其是那些需要构建本地知识库、进行网络数据采集或搭建 RAG（检索增强生成）系统的技术团队。其独特亮点在于强大的工程化设计：支持基于 Azure 或本地磁盘的并行任务队列，大幅提升抓取效率；内置 Playwright 引擎可完美渲染 JavaScript 动态内容；智能拦截广告以降低带宽成本；还能在保护用户匿名性的同时，自动提取元数据、截图及图片。更值得一提的是，它集成了 OpenAI 嵌入模型与 Azure AI Search，能自动建立可语义检索的智能索引，让数据从采集到应用的全流程更加顺畅高效。","# 🛰️ Scrape It Now!\n\nWeb scraper made for AI and simplicity in mind. It runs as a CLI that can be parallelized and outputs high-quality markdown content.\n\n[![GitHub last release date](https:\u002F\u002Fimg.shields.io\u002Fgithub\u002Frelease-date\u002Fclemlesne\u002Fscrape-it-now)](https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Freleases)\n[![GitHub project license](https:\u002F\u002Fimg.shields.io\u002Fgithub\u002Flicense\u002Fclemlesne\u002Fscrape-it-now)](https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fblob\u002Fmain\u002FLICENSE)\n[![PyPI package version](https:\u002F\u002Fimg.shields.io\u002Fpypi\u002Fv\u002Fscrape-it-now)](https:\u002F\u002Fpypi.org\u002Fproject\u002Fscrape-it-now)\n[![PyPI supported Python versions](https:\u002F\u002Fimg.shields.io\u002Fpypi\u002Fpyversions\u002Fscrape-it-now)](https:\u002F\u002Fpypi.org\u002Fproject\u002Fscrape-it-now)\n\n## Features\n\nShared:\n\n- 🏗️ Decoupled architecture with [Azure Queue Storage](https:\u002F\u002Flearn.microsoft.com\u002Fen-us\u002Fazure\u002Fstorage\u002Fqueues) or local [sqlite](https:\u002F\u002Fsqlite.org)\n- ⚙️ Idempotent operations that can be run in parallel\n- 💾 Scraped content is stored in [Azure Blob Storage](https:\u002F\u002Flearn.microsoft.com\u002Fen-us\u002Fazure\u002Fstorage\u002Fblobs) or local disk\n\nScraper:\n\n- 🛑 Avoid re-scraping a page if it hasn't changed\n- 🚫 Block ads to lower network costs with [The Block List Project](https:\u002F\u002Fgithub.com\u002Fblocklistproject\u002FLists)\n- 🔗 Explore pages in depth by detecting links and de-duplicating them\n- ✍️ Extract markdown content from a page with [Pandoc](https:\u002F\u002Fgithub.com\u002Fjgm\u002Fpandoc)\n- 🏷️ Extract [metadata elements](https:\u002F\u002Fdeveloper.mozilla.org\u002Fen-US\u002Fdocs\u002FWeb\u002FHTML\u002FElement\u002Fmeta) from the page\n- 🖥️ Load dynamic JavaScript content with [Playwright](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002Fplaywright-python) and [Chromium](https:\u002F\u002Fwww.chromium.org\u002FHome)\n- 🕵️‍♂️ Preserve anonymity with a random user agent, random viewport size, and no client hints headers\n- 📊 Show progress with a status command\n- 🖼️ Store images collected on the page\n- 📸 Store screenshot of the page\n- 📡 Track progress of total network usage\n\nIndexer:\n\n- 🧠 AI Search index is created automatically\n- ✂️ Chunk markdown while keeping the content coherent\n- 📈 Embed chunks with OpenAI embeddings\n- 🔍 Indexed content is semantically searchable with [Azure AI Search](https:\u002F\u002Flearn.microsoft.com\u002Fen-us\u002Fazure\u002Fsearch)\n\n## Installation\n\n### From PyPI\n\n```bash\n# Install the package\npython3 -m pip install scrape-it-now\n# Run the CLI\nscrape-it-now --help\n```\n\nTo configure the CLI (including authentication to the backend services), use environment variables, a `.env` file or command line options.\n\n### From sources\n\nApplication must be run with Python 3.13 or later. If this version is not installed, an easy way to install it is [pyenv](https:\u002F\u002Fgithub.com\u002Fpyenv\u002Fpyenv).\n\n```bash\n# Download the source code\ngit clone https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now.git\n# Move to the directory\ncd scrape-it-now\n# Run install scripts\nmake install dev\n# Run the CLI\nscrape-it-now --help\n```\n\n## How to use\n\n### Scrape a website\n\n#### Run a job\n\nUsage with Azure Blob Storage and Azure Queue Storage:\n\n```bash\n# Azure Storage configuration\nexport AZURE_STORAGE_ACCESS_KEY=xxx\nexport AZURE_STORAGE_ACCOUNT_NAME=xxx\n# Run the job\nscrape-it-now scrape run https:\u002F\u002Fnytimes.com\n```\n\nUsage with Local Disk Blob and Local Disk Queue:\n\n```bash\n# Local disk configuration\nexport BLOB_PROVIDER=local_disk\nexport QUEUE_PROVIDER=local_disk\n# Run the job\nscrape-it-now scrape run https:\u002F\u002Fnytimes.com\n```\n\nExample:\n\n```bash\n❯ scrape-it-now scrape run https:\u002F\u002Fnytimes.com\n2024-11-08T13:18:49.169320Z [info     ] Start scraping job lydmtyz\n2024-11-08T13:18:49.169392Z [info     ] Installing dependencies if needed, this may take a few minutes\n2024-11-08T13:18:52.542422Z [info     ] Queued 1\u002F1 URLs\n2024-11-08T13:18:58.509221Z [info     ] Start processing https:\u002F\u002Fnytimes.com depth=1 process=scrape-lydmtyz-4 task=63dce50\n2024-11-08T13:19:04.173198Z [info     ] Loaded 154554 ads and trackers process=scrape-lydmtyz-4\n2024-11-08T13:19:16.393045Z [info     ] Queued 310\u002F311 URLs            depth=1 process=scrape-lydmtyz-4 task=63dce50\n2024-11-08T13:19:16.393323Z [info     ] Scraped                        depth=1 process=scrape-lydmtyz-4 task=63dce50\n...\n```\n\nMost frequent options are:\n\n| `Options` | Description | `Environment variable` |\n|-|-|-|\n| `--azure-storage-access-key`\u003C\u002Fbr>`-asak` | Azure Storage access key | `AZURE_STORAGE_ACCESS_KEY` |\n| `--azure-storage-account-name`\u003C\u002Fbr>`-asan` | Azure Storage account name | `AZURE_STORAGE_ACCOUNT_NAME` |\n| `--blob-provider`\u003C\u002Fbr>`-bp` | Blob provider | `BLOB_PROVIDER` |\n| `--job-name`\u003C\u002Fbr>`-jn` | Job name | `JOB_NAME` |\n| `--max-depth`\u003C\u002Fbr>`-md` | Maximum depth | `MAX_DEPTH` |\n| `--queue-provider`\u003C\u002Fbr>`-qp` | Queue provider | `QUEUE_PROVIDER` |\n| `--save-images`\u003C\u002Fbr>`-si` | Save images | `SAVE_IMAGES` |\n| `--save-screenshot`\u003C\u002Fbr>`-ss` | Save screenshot | `SAVE_SCREENSHOT` |\n| `--whitelist`\u003C\u002Fbr>`-w` | Whitelist | `WHITELIST` |\n\nFor documentation on all available options, run:\n\n```bash\nscrape-it-now scrape run --help\n```\n\n#### Show job status\n\nUsage with Azure Blob Storage:\n\n```bash\n# Azure Storage configuration\nexport AZURE_STORAGE_CONNECTION_STRING=xxx\n# Show the job status\nscrape-it-now scrape status [job_name]\n```\n\nUsage with Local Disk Blob:\n\n```bash\n# Local disk configuration\nexport BLOB_PROVIDER=local_disk\n# Show the job status\nscrape-it-now scrape status [job_name]\n```\n\nExample:\n\n```bash\n❯ scrape-it-now scrape status lydmtyz\n{\"created_at\":\"2024-11-08T13:18:52.839060Z\",\"last_updated\":\"2024-11-08T13:19:16.528370Z\",\"network_used_mb\":2.6666793823242188,\"processed\":1,\"queued\":311}\n```\n\nMost frequent options are:\n\n| `Options` | Description | `Environment variable` |\n|-|-|-|\n| `--azure-storage-access-key`\u003C\u002Fbr>`-asak` | Azure Storage access key | `AZURE_STORAGE_ACCESS_KEY` |\n| `--azure-storage-account-name`\u003C\u002Fbr>`-asan` | Azure Storage account name | `AZURE_STORAGE_ACCOUNT_NAME` |\n| `--blob-provider`\u003C\u002Fbr>`-bp` | Blob provider | `BLOB_PROVIDER` |\n\nFor documentation on all available options, run:\n\n```bash\nscrape-it-now scrape status --help\n```\n\n### Index a scraped website\n\n#### Run a job\n\nUsage with Azure Blob Storage, Azure Queue Storage and Azure AI Search:\n\n```bash\n# Azure OpenAI configuration\nexport AZURE_OPENAI_API_KEY=xxx\nexport AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME=xxx\nexport AZURE_OPENAI_EMBEDDING_DIMENSIONS=xxx\nexport AZURE_OPENAI_EMBEDDING_MODEL_NAME=xxx\nexport AZURE_OPENAI_ENDPOINT=xxx\n\n# Azure Search configuration\nexport AZURE_SEARCH_API_KEY=xxx\nexport AZURE_SEARCH_ENDPOINT=xxx\n\n# Azure Storage configuration\nexport AZURE_STORAGE_ACCESS_KEY=xxx\nexport AZURE_STORAGE_ACCOUNT_NAME=xxx\n\n# Run the job\nscrape-it-now index run [job_name]\n```\n\nUsage with Local Disk Blob, Local Disk Queue and Azure AI Search:\n\n```bash\n# Azure OpenAI configuration\nexport AZURE_OPENAI_API_KEY=xxx\nexport AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME=xxx\nexport AZURE_OPENAI_EMBEDDING_DIMENSIONS=xxx\nexport AZURE_OPENAI_EMBEDDING_MODEL_NAME=xxx\nexport AZURE_OPENAI_ENDPOINT=xxx\n# Azure Search configuration\nexport AZURE_SEARCH_API_KEY=xxx\nexport AZURE_SEARCH_ENDPOINT=xxx\n# Local disk configuration\nexport BLOB_PROVIDER=local_disk\nexport QUEUE_PROVIDER=local_disk\n# Run the job\nscrape-it-now index run [job_name]\n```\n\nExample:\n\n```bash\n❯ scrape-it-now index run lydmtyz\n2024-11-08T13:20:37.129411Z [info     ] Start indexing job lydmtyz\n2024-11-08T13:20:38.945954Z [info     ] Start processing https:\u002F\u002Fnytimes.com process=index-lydmtyz-4 task=63dce50\n2024-11-08T13:20:39.162692Z [info     ] Chunked into 7 parts           process=index-lydmtyz-4 task=63dce50\n2024-11-08T13:20:42.407391Z [info     ] Indexed 7 chunks               process=index-lydmtyz-4 task=63dce50\n...\n```\n\nMost frequent options are:\n\n| `Options` | Description | `Environment variable` |\n|-|-|-|\n| `--azure-openai-api-key`\u003C\u002Fbr>`-aoak` | Azure OpenAI API key | `AZURE_OPENAI_API_KEY` |\n| `--azure-openai-embedding-deployment-name`\u003C\u002Fbr>`-aoedn` | Azure OpenAI embedding deployment name | `AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME` |\n| `--azure-openai-embedding-dimensions`\u003C\u002Fbr>`-aoed` | Azure OpenAI embedding dimensions | `AZURE_OPENAI_EMBEDDING_DIMENSIONS` |\n| `--azure-openai-embedding-model-name`\u003C\u002Fbr>`-aoemn` | Azure OpenAI embedding model name | `AZURE_OPENAI_EMBEDDING_MODEL_NAME` |\n| `--azure-openai-endpoint`\u003C\u002Fbr>`-aoe` | Azure OpenAI endpoint | `AZURE_OPENAI_ENDPOINT` |\n| `--azure-search-api-key`\u003C\u002Fbr>`-asak` | Azure Search API key | `AZURE_SEARCH_API_KEY` |\n| `--azure-search-endpoint`\u003C\u002Fbr>`-ase` | Azure Search endpoint | `AZURE_SEARCH_ENDPOINT` |\n| `--azure-storage-access-key`\u003C\u002Fbr>`-asak` | Azure Storage access key | `AZURE_STORAGE_ACCESS_KEY` |\n| `--azure-storage-account-name`\u003C\u002Fbr>`-asan` | Azure Storage account name | `AZURE_STORAGE_ACCOUNT_NAME` |\n| `--blob-provider`\u003C\u002Fbr>`-bp` | Blob provider | `BLOB_PROVIDER` |\n| `--queue-provider`\u003C\u002Fbr>`-qp` | Queue provider | `QUEUE_PROVIDER` |\n\nFor documentation on all available options, run:\n\n```bash\nscrape-it-now index run --help\n```\n\n## Architecture\n\n### Scrape\n\n```mermaid\n---\ntitle: Scrape process with Azure Storage\n---\ngraph LR\n  cli[\"CLI\"]\n  web[\"Website\"]\n\n  subgraph \"Azure Queue Storage\"\n    to_chunk[\"To chunk\"]\n    to_scrape[\"To scrape\"]\n  end\n\n  subgraph \"Azure Blob Storage\"\n    subgraph \"Container\"\n      job[\"job\"]\n      scraped[\"scraped\"]\n      state[\"state\"]\n    end\n  end\n\n  cli -- (1) Pull message --> to_scrape\n  cli -- (2) Get cache --> scraped\n  cli -- (3) Browse --> web\n  cli -- (4) Update cache --> scraped\n  cli -- (5) Push state --> state\n  cli -- (6) Add message --> to_scrape\n  cli -- (7) Add message --> to_chunk\n  cli -- (8) Update state --> job\n```\n\n### Index\n\n```mermaid\n---\ntitle: Scrape process with Azure Storage and Azure AI Search\n---\ngraph LR\n  search[\"Azure AI Search\"]\n  cli[\"CLI\"]\n  embeddings[\"Azure OpenAI Embeddings\"]\n\n  subgraph \"Azure Queue Storage\"\n    to_chunk[\"To chunk\"]\n  end\n\n  subgraph \"Azure Blob Storage\"\n    subgraph \"Container\"\n      scraped[\"scraped\"]\n    end\n  end\n\n  cli -- (1) Pull message --> to_chunk\n  cli -- (2) Get cache --> scraped\n  cli -- (3) Chunk --> cli\n  cli -- (4) Embed --> embeddings\n  cli -- (5) Push to search --> search\n```\n\n## Design\n\nBlob storage is organized in folders:\n\n```txt\n[job_name]-scraping\u002F            # Job name (either defined by the user or generated)\n    scraped\u002F                    # All the data from the pages\n        [page_id]\u002F              # Assets from a page\n            screenshot.jpeg     # Screenshot (if enabled)\n            [image_id].[ext]    # Image binary (if enabled)\n            [image_id].json     # Image metadata (if enabled)\n        [page_id].json          # Data from a page\n    state\u002F                      # Job states (cache & parallelization)\n        [page_id]               # Page state\n    job.json                    # Job state (aggregated stats)\n```\n\nPage data is considered as an API (won't break until the next major version) and is stored in JSON format:\n\n```json\n{\n  \"created_at\": \"2024-09-11T14:06:43.566187Z\",\n  \"redirect\": \"https:\u002F\u002Fwww.nytimes.com\u002Finteractive\u002F2024\u002Fpodcasts\u002Fserial-season-four-guantanamo.html\",\n  \"status\": 200,\n  \"url\": \"https:\u002F\u002Fwww.nytimes.com\u002Finteractive\u002F2024\u002Fpodcasts\u002Fserial-season-four-guantanamo.html\",\n  \"content\": \"## Listen to the trailer for Serial Season 4...\",\n  \"etag\": null,\n  \"links\": [\n    \"https:\u002F\u002Fpodcasts.apple.com\u002Fus\u002Fpodcast\u002Fserial\u002Fid917918570\",\n    \"https:\u002F\u002Fmusic.amazon.com\u002Fpodcasts\u002Fd1022069-8863-42f3-823e-857fd8a7b616\u002Fserial?ref=dm_sh_OVBHkKYvW1poSzCOsBqHFXuLc\",\n    ...\n  ],\n  \"metas\": {\n    \"description\": \"“Serial” returns with a history of Guantánamo told by people who lived through key moments in Guantánamo’s evolution, who know things the rest of us don’t about what it’s like to be caught inside an improvised justice system.\",\n    \"articleid\": \"100000009373583\",\n    \"twitter:site\": \"@nytimes\",\n    ...\n  },\n  \"network_used_mb\": 1.041460037231445,\n  \"raw\": \"\u003Chead>...\u003C\u002Fhead>\u003Cbody>...\u003C\u002Fbody>\",\n  \"valid_until\": \"2024-09-11T14:11:37.790570Z\"\n}\n```\n\nThen, indexed data is stored in Azure AI Search:\n\n| Field | Type | Description |\n|-|-|-|\n| `chunck_number` | `Edm.Int32` | Chunk number, from `0` to *`x`* |\n| `content` | `Edm.String` | Chunck content |\n| `created_at` | `Edm.DateTimeOffset` | Source scrape date |\n| `id` | `Edm.String` | Chunck ID |\n| `title` | `Edm.String` | Source page title |\n| `url` | `Edm.String` | Source page URL |\n\n## Advanced usage\n\n### Whitelist\n\nWhitelist option allows to restrict to a domain and ignore sub paths. It is a list of regular expressions:\n\n```txt\ndomain1,regexp1,regexp2 domain2,regexp3\n```\n\nFor examples:\n\nTo whitelist `learn.microsoft.com`:\n\n```txt\nlearn\\.microsoft\\.com\n```\n\nTo whitelist `learn.microsoft.com` and `go.microsoft.com`, but ignore all sub paths except `\u002Fen-us`:\n\n```txt\nlearn\\.microsoft\\.com,^\u002F(?!en-us).* go\\.microsoft\\.com\n```\n\n### Source environment variables\n\nTo configure easily the CLI, source environment variables from a `.env` file. For example, for the `--azure-storage-access-key` option:\n\n```bash\nAZURE_STORAGE_ACCESS_KEY=xxx\n```\n\nFor arguments that accept multiple values, use a space-separated list. For example, for the `--whitelist` option:\n\n```bash\nWHITELIST=learn\\.microsoft\\.com go\\.microsoft\\.com\n```\n\n### Application cache directory\n\nThe cache directoty depends on the operating system:\n\n- `~\u002F.config\u002Fscrape-it-now` (Unix)\n- `~\u002FLibrary\u002FApplication Support\u002Fscrape-it-now` (macOS)\n- `C:\\Users\\\u003Cuser>\\AppData\\Roaming\\scrape-it-now` (Windows)\n\n### Broswer binary installation\n\nBrowser binaries are automatically downloaded or updated at each run. Browser is Chromium and it is not configurable (feel free to open an issue if you need another browser), it weights around 450MB. Cache is stored in the cache directory.\n\n### How Local Disk storage works\n\nLocal Disk storage is used for both blob and queue. It is not recommended for production use, as it is not easily scalable, and not fault-tolerant. It is useful for testing and development or when you cannot use Azure services.\n\nImplementation:\n\n- Local Disk Blob uses a directory structure to store blobs. Each blob is stored in a file with the blob name as the file name. Lease is implemented with lock files. By default, files are stored in a directory relative to the command execution directory.\n- Local Disk Queue uses a SQLite database to store messages. Database is stored in the cache directory. SQL databases implement visibility timeout and deletion tokens to ensure consistency to the stateless queue services like Azure Queue Storage.\n\n### Use proxies for anonymity\n\nProxies are not implemented in the application. Network security cannot be achieved from the application level. Use a VPN (e.g. your, third-party) or a proxy service (e.g. residential procies, Tor) to ensure anonymity and configure the system firewall to limit the application network access to it.\n\n### Bundle with a container\n\nAs the application is packaged to PyPi, it can easily be bundled with a container. At every start, the application will download the dependencies (browser, etc.) and cache them. You can pre-download them by running the command `scrape-it-now scrape install`.\n\nA good technique for performance would also to parallelize the scraping and indexing jobs by running multiple containers of each. This can be achieved with [KEDA](https:\u002F\u002Fkeda.sh), by configuring a [queue scaler](https:\u002F\u002Fkeda.sh\u002Fdocs\u002F2.16\u002Fscalers\u002Fazure-storage-queue).\n","# 🛰️ 立即抓取！\n\n专为 AI 和简洁性设计的网页抓取工具。它以 CLI 形式运行，支持并行化，并输出高质量的 Markdown 内容。\n\n[![GitHub 最新发布日期](https:\u002F\u002Fimg.shields.io\u002Fgithub\u002Frelease-date\u002Fclemlesne\u002Fscrape-it-now)](https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Freleases)\n[![GitHub 项目许可证](https:\u002F\u002Fimg.shields.io\u002Fgithub\u002Flicense\u002Fclemlesne\u002Fscrape-it-now)](https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fblob\u002Fmain\u002FLICENSE)\n[![PyPI 包版本](https:\u002F\u002Fimg.shields.io\u002Fpypi\u002Fv\u002Fscrape-it-now)](https:\u002F\u002Fpypi.org\u002Fproject\u002Fscrape-it-now)\n[![PyPI 支持的 Python 版本](https:\u002F\u002Fimg.shields.io\u002Fpypi\u002Fpyversions\u002Fscrape-it-now)](https:\u002F\u002Fpypi.org\u002Fproject\u002Fscrape-it-now)\n\n## 功能\n\n共享：\n\n- 🏗️ 解耦架构，使用 [Azure 队列存储](https:\u002F\u002Flearn.microsoft.com\u002Fen-us\u002Fazure\u002Fstorage\u002Fqueues) 或本地 [sqlite](https:\u002F\u002Fsqlite.org)\n- ⚙️ 幂等操作，可并行执行\n- 💾 抓取内容存储在 [Azure Blob 存储](https:\u002F\u002Flearn.microsoft.com\u002Fen-us\u002Fazure\u002Fstorage\u002Fblobs) 或本地磁盘\n\n抓取器：\n\n- 🛑 如果页面未更改，则避免重复抓取\n- 🚫 使用 [The Block List Project](https:\u002F\u002Fgithub.com\u002Fblocklistproject\u002FLists) 屏蔽广告以降低网络成本\n- 🔗 通过检测和去重链接深入探索页面\n- ✍️ 使用 [Pandoc](https:\u002F\u002Fgithub.com\u002Fjgm\u002Fpandoc) 从页面中提取 Markdown 内容\n- 🏷️ 提取页面中的 [元数据元素](https:\u002F\u002Fdeveloper.mozilla.org\u002Fen-US\u002Fdocs\u002FWeb\u002FHTML\u002FElement\u002Fmeta)\n- 🖥️ 使用 [Playwright](https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002Fplaywright-python) 和 [Chromium](https:\u002F\u002Fwww.chromium.org\u002FHome) 加载动态 JavaScript 内容\n- 🕵️‍♂️ 通过随机用户代理、随机视口大小以及不发送客户端提示头信息来保护匿名性\n- 📊 使用状态命令显示进度\n- 🖼️ 存储页面上收集的图片\n- 📸 存储页面截图\n- 📡 跟踪总网络用量的进度\n\n索引器：\n\n- 🧠 自动创建 AI 搜索索引\n- ✂️ 在保持内容连贯性的前提下对 Markdown 进行分块\n- 📈 使用 OpenAI 嵌入技术对分块内容进行嵌入\n- 🔍 索引后的内容可通过 [Azure AI 搜索](https:\u002F\u002Flearn.microsoft.com\u002Fen-us\u002Fazure\u002Fsearch) 进行语义搜索\n\n## 安装\n\n### 从 PyPI\n\n```bash\n# 安装包\npython3 -m pip install scrape-it-now\n# 运行 CLI\nscrape-it-now --help\n```\n\n要配置 CLI（包括后端服务的身份验证），可以使用环境变量、`.env` 文件或命令行选项。\n\n### 从源代码\n\n应用程序必须使用 Python 3.13 或更高版本运行。如果尚未安装此版本，可以通过 [pyenv](https:\u002F\u002Fgithub.com\u002Fpyenv\u002Fpyenv) 轻松安装。\n\n```bash\n# 下载源代码\ngit clone https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now.git\n# 进入目录\ncd scrape-it-now\n# 运行安装脚本\nmake install dev\n# 运行 CLI\nscrape-it-now --help\n```\n\n## 使用方法\n\n### 抓取一个网站\n\n#### 运行任务\n\n使用 Azure Blob 存储和 Azure 队列存储：\n\n```bash\n# Azure 存储配置\nexport AZURE_STORAGE_ACCESS_KEY=xxx\nexport AZURE_STORAGE_ACCOUNT_NAME=xxx\n# 运行任务\nscrape-it-now scrape run https:\u002F\u002Fnytimes.com\n```\n\n使用本地磁盘 Blob 和本地磁盘队列：\n\n```bash\n# 本地磁盘配置\nexport BLOB_PROVIDER=local_disk\nexport QUEUE_PROVIDER=local_disk\n# 运行任务\nscrape-it-now scrape run https:\u002F\u002Fnytimes.com\n```\n\n示例：\n\n```bash\n❯ scrape-it-now scrape run https:\u002F\u002Fnytimes.com\n2024-11-08T13:18:49.169320Z [info     ] 开始抓取任务 lydmtyz\n2024-11-08T13:18:49.169392Z [info     ] 如果需要，正在安装依赖项，这可能需要几分钟\n2024-11-08T13:18:52.542422Z [info     ] 已将 1\u002F1 个 URL 排入队列\n2024-11-08T13:18:58.509221Z [info     ] 开始处理 https:\u002F\u002Fnytimes.com，深度=1，进程=scrape-lydmtyz-4，任务=63dce50\n2024-11-08T13:19:04.173198Z [info     ] 已加载 154554 个广告和跟踪器，进程=scrape-lydmtyz-4\n2024-11-08T13:19:16.393045Z [info     ] 已将 310\u002F311 个 URL 排入队列，深度=1，进程=scrape-lydmtyz-4，任务=63dce50\n2024-11-08T13:19:16.393323Z [info     ] 已抓取，深度=1，进程=scrape-lydmtyz-4，任务=63dce50\n...\n```\n\n常用选项如下：\n\n| `选项` | 描述 | `环境变量` |\n|-|-|-|\n| `--azure-storage-access-key`\u003C\u002Fbr>`-asak` | Azure 存储访问密钥 | `AZURE_STORAGE_ACCESS_KEY` |\n| `--azure-storage-account-name`\u003C\u002Fbr>`-asan` | Azure 存储账户名称 | `AZURE_STORAGE_ACCOUNT_NAME` |\n| `--blob-provider`\u003C\u002Fbr>`-bp` | Blob 提供者 | `BLOB_PROVIDER` |\n| `--job-name`\u003C\u002Fbr>`-jn` | 任务名称 | `JOB_NAME` |\n| `--max-depth`\u003C\u002Fbr>`-md` | 最大深度 | `MAX_DEPTH` |\n| `--queue-provider`\u003C\u002Fbr>`-qp` | 队列提供者 | `QUEUE_PROVIDER` |\n| `--save-images`\u003C\u002Fbr>`-si` | 保存图片 | `SAVE_IMAGES` |\n| `--save-screenshot`\u003C\u002Fbr>`-ss` | 保存截图 | `SAVE_SCREENSHOT` |\n| `--whitelist`\u003C\u002Fbr>`-w` | 白名单 | `WHITELIST` |\n\n有关所有可用选项的文档，请运行：\n\n```bash\nscrape-it-now scrape run --help\n```\n\n#### 显示任务状态\n\n使用 Azure Blob 存储：\n\n```bash\n# Azure 存储配置\nexport AZURE_STORAGE_CONNECTION_STRING=xxx\n# 显示任务状态\nscrape-it-now scrape status [job_name]\n```\n\n使用本地磁盘 Blob：\n\n```bash\n# 本地磁盘配置\nexport BLOB_PROVIDER=local_disk\n# 显示任务状态\nscrape-it-now scrape status [job_name]\n```\n\n示例：\n\n```bash\n❯ scrape-it-now scrape status lydmtyz\n{\"created_at\":\"2024-11-08T13:18:52.839060Z\",\"last_updated\":\"2024-11-08T13:19:16.528370Z\",\"network_used_mb\":2.6666793823242188,\"processed\":1,\"queued\":311}\n```\n\n常用选项如下：\n\n| `选项` | 描述 | `环境变量` |\n|-|-|-|\n| `--azure-storage-access-key`\u003C\u002Fbr>`-asak` | Azure 存储访问密钥 | `AZURE_STORAGE_ACCESS_KEY` |\n| `--azure-storage-account-name`\u003C\u002Fbr>`-asan` | Azure 存储账户名称 | `AZURE_STORAGE_ACCOUNT_NAME` |\n| `--blob-provider`\u003C\u002Fbr>`-bp` | Blob 提供者 | `BLOB_PROVIDER` |\n\n有关所有可用选项的文档，请运行：\n\n```bash\nscrape-it-now scrape status --help\n```\n\n### 对抓取的网站进行索引\n\n#### 运行任务\n\n使用 Azure Blob 存储、Azure 队列存储和 Azure AI 搜索：\n\n```bash\n# Azure OpenAI 配置\nexport AZURE_OPENAI_API_KEY=xxx\nexport AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME=xxx\nexport AZURE_OPENAI_EMBEDDING_DIMENSIONS=xxx\nexport AZURE_OPENAI_EMBEDDING_MODEL_NAME=xxx\nexport AZURE_OPENAI_ENDPOINT=xxx\n\n# Azure 搜索配置\nexport AZURE_SEARCH_API_KEY=xxx\nexport AZURE_SEARCH_ENDPOINT=xxx\n\n# Azure 存储配置\nexport AZURE_STORAGE_ACCESS_KEY=xxx\nexport AZURE_STORAGE_ACCOUNT_NAME=xxx\n\n# 运行任务\nscrape-it-now index run [job_name]\n```\n\n使用本地磁盘 Blob、本地磁盘队列和 Azure AI 搜索：\n\n```bash\n# Azure OpenAI 配置\nexport AZURE_OPENAI_API_KEY=xxx\nexport AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME=xxx\nexport AZURE_OPENAI_EMBEDDING_DIMENSIONS=xxx\nexport AZURE_OPENAI_EMBEDDING_MODEL_NAME=xxx\nexport AZURE_OPENAI_ENDPOINT=xxx\n# Azure 搜索配置\nexport AZURE_SEARCH_API_KEY=xxx\nexport AZURE_SEARCH_ENDPOINT=xxx\n\n# 本地磁盘配置\nexport BLOB_PROVIDER=local_disk\nexport QUEUE_PROVIDER=local_disk\n# 运行作业\nscrape-it-now index run [job_name]\n```\n\n示例：\n\n```bash\n❯ scrape-it-now index run lydmtyz\n2024-11-08T13:20:37.129411Z [info     ] 开始索引作业 lydmtyz\n2024-11-08T13:20:38.945954Z [info     ] 开始处理 https:\u002F\u002Fnytimes.com process=index-lydmtyz-4 task=63dce50\n2024-11-08T13:20:39.162692Z [info     ] 已分块为 7 个部分           process=index-lydmtyz-4 task=63dce50\n2024-11-08T13:20:42.407391Z [info     ] 已索引 7 个块               process=index-lydmtyz-4 task=63dce50\n...\n```\n\n最常见的选项如下：\n\n| `选项` | 描述 | `环境变量` |\n|-|-|-|\n| `--azure-openai-api-key`\u003C\u002Fbr>`-aoak` | Azure OpenAI API 密钥 | `AZURE_OPENAI_API_KEY` |\n| `--azure-openai-embedding-deployment-name`\u003C\u002Fbr>`-aoedn` | Azure OpenAI 嵌入模型部署名称 | `AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME` |\n| `--azure-openai-embedding-dimensions`\u003C\u002Fbr>`-aoed` | Azure OpenAI 嵌入维度 | `AZURE_OPENAI_EMBEDDING_DIMENSIONS` |\n| `--azure-openai-embedding-model-name`\u003C\u002Fbr>`-aoemn` | Azure OpenAI 嵌入模型名称 | `AZURE_OPENAI_EMBEDDING_MODEL_NAME` |\n| `--azure-openai-endpoint`\u003C\u002Fbr>`-aoe` | Azure OpenAI 终端节点 | `AZURE_OPENAI_ENDPOINT` |\n| `--azure-search-api-key`\u003C\u002Fbr>`-asak` | Azure Search API 密钥 | `AZURE_SEARCH_API_KEY` |\n| `--azure-search-endpoint`\u003C\u002Fbr>`-ase` | Azure Search 终端节点 | `AZURE_SEARCH_ENDPOINT` |\n| `--azure-storage-access-key`\u003C\u002Fbr>`-asak` | Azure 存储访问密钥 | `AZURE_STORAGE_ACCESS_KEY` |\n| `--azure-storage-account-name`\u003C\u002Fbr>`-asan` | Azure 存储账户名称 | `AZURE_STORAGE_ACCOUNT_NAME` |\n| `--blob-provider`\u003C\u002Fbr>`-bp` | Blob 提供者 | `BLOB_PROVIDER` |\n| `--queue-provider`\u003C\u002Fbr>`-qp` | 队列提供者 | `QUEUE_PROVIDER` |\n\n如需查看所有可用选项的文档，请运行以下命令：\n\n```bash\nscrape-it-now index run --help\n```\n\n## 架构\n\n### 抓取\n\n```mermaid\n---\ntitle: 使用 Azure 存储的抓取流程\n---\ngraph LR\n  cli[\"CLI\"]\n  web[\"网站\"]\n\n  subgraph \"Azure 队列存储\"\n    to_chunk[\"待分块\"]\n    to_scrape[\"待抓取\"]\n  end\n\n  subgraph \"Azure Blob 存储\"\n    subgraph \"容器\"\n      job[\"作业\"]\n      scraped[\"已抓取\"]\n      state[\"状态\"]\n    end\n  end\n\n  cli -- (1) 拉取消息 --> to_scrape\n  cli -- (2) 获取缓存 --> scraped\n  cli -- (3) 浏览 --> web\n  cli -- (4) 更新缓存 --> scraped\n  cli -- (5) 推送状态 --> state\n  cli -- (6) 添加消息 --> to_scrape\n  cli -- (7) 添加消息 --> to_chunk\n  cli -- (8) 更新状态 --> job\n```\n\n### 索引\n\n```mermaid\n---\ntitle: 使用 Azure 存储和 Azure AI 搜索的索引流程\n---\ngraph LR\n  search[\"Azure AI 搜索\"]\n  cli[\"CLI\"]\n  embeddings[\"Azure OpenAI 嵌入\"]\n\n  subgraph \"Azure 队列存储\"\n    to_chunk[\"待分块\"]\n  end\n\n  subgraph \"Azure Blob 存储\"\n    subgraph \"容器\"\n      scraped[\"已抓取\"]\n    end\n  end\n\n  cli -- (1) 拉取消息 --> to_chunk\n  cli -- (2) 获取缓存 --> scraped\n  cli -- (3) 分块 --> cli\n  cli -- (4) 嵌入 --> embeddings\n  cli -- (5) 推送到搜索 --> search\n```\n\n## 设计\n\nBlob 存储按文件夹组织：\n\n```txt\n[job_name]-scraping\u002F            # 作业名称（由用户定义或自动生成）\n    scraped\u002F                    # 所有页面的数据\n        [page_id]\u002F              # 页面资源\n            screenshot.jpeg     # 截图（若启用）\n            [image_id].[ext]    # 图片二进制文件（若启用）\n            [image_id].json     # 图片元数据（若启用）\n        [page_id].json          # 页面数据\n    state\u002F                      # 作业状态（缓存与并行化）\n        [page_id]               # 页面状态\n    job.json                    # 作业状态（汇总统计）\n```\n\n页面数据被视为一个 API（在下一个主要版本之前不会破坏），以 JSON 格式存储：\n\n```json\n{\n  \"created_at\": \"2024-09-11T14:06:43.566187Z\",\n  \"redirect\": \"https:\u002F\u002Fwww.nytimes.com\u002Finteractive\u002F2024\u002Fpodcasts\u002Fserial-season-four-guantanamo.html\",\n  \"status\": 200,\n  \"url\": \"https:\u002F\u002Fwww.nytimes.com\u002Finteractive\u002F2024\u002Fpodcasts\u002Fserial-season-four-guantanamo.html\",\n  \"content\": \"## 听听《Serial》第四季的预告片...\",\n  \"etag\": null,\n  \"links\": [\n    \"https:\u002F\u002Fpodcasts.apple.com\u002Fus\u002Fpodcast\u002Fserial\u002Fid917918570\",\n    \"https:\u002F\u002Fmusic.amazon.com\u002Fpodcasts\u002Fd1022069-8863-42f3-823e-857fd8a7b616\u002Fserial?ref=dm_sh_OVBHkKYvW1poSzCOsBqHFXuLc\",\n    ...\n  ],\n  \"metas\": {\n    \"description\": \"《Serial》回归，讲述关塔那摩的历史，由那些经历过关塔那摩关键时刻的人们讲述。他们了解我们其他人所不了解的，在临时司法体系中被卷入其中的感受。\",\n    \"articleid\": \"100000009373583\",\n    \"twitter:site\": \"@nytimes\",\n    ...\n  },\n  \"network_used_mb\": 1.041460037231445,\n  \"raw\": \"\u003Chead>...\u003C\u002Fhead>\u003Cbody>...\u003C\u002Fbody>\",\n  \"valid_until\": \"2024-09-11T14:11:37.790570Z\"\n}\n```\n\n随后，索引后的数据存储在 Azure AI 搜索中：\n\n| 字段 | 类型 | 描述 |\n|-|-|-|\n| `chunck_number` | `Edm.Int32` | 分块编号，从 `0` 到 *`x`* |\n| `content` | `Edm.String` | 分块内容 |\n| `created_at` | `Edm.DateTimeOffset` | 来源抓取日期 |\n| `id` | `Edm.String` | 分块 ID |\n| `title` | `Edm.String` | 来源页面标题 |\n| `url` | `Edm.String` | 来源页面 URL |\n\n## 高级用法\n\n### 白名单\n\n白名单选项允许限制到特定域名，并忽略子路径。它是一个正则表达式列表：\n\n```txt\ndomain1,regexp1,regexp2 domain2,regexp3\n```\n\n例如：\n\n要将 `learn.microsoft.com` 加入白名单：\n\n```txt\nlearn\\.microsoft\\.com\n```\n\n要将 `learn.microsoft.com` 和 `go.microsoft.com` 加入白名单，但仅保留 `\u002Fen-us` 路径，忽略其他子路径：\n\n```txt\nlearn\\.microsoft\\.com,^\u002F(?!en-us).* go\\.microsoft\\.com\n```\n\n### 源环境变量\n\n为了方便配置 CLI，可以从 `.env` 文件中加载环境变量。例如，对于 `--azure-storage-access-key` 选项：\n\n```bash\nAZURE_STORAGE_ACCESS_KEY=xxx\n```\n\n对于接受多个值的参数，使用空格分隔的列表。例如，对于 `--whitelist` 选项：\n\n```bash\nWHITELIST=learn\\.microsoft\\.com go\\.microsoft\\.com\n```\n\n### 应用程序缓存目录\n\n缓存目录取决于操作系统：\n\n- `~\u002F.config\u002Fscrape-it-now`（Unix）\n- `~\u002FLibrary\u002FApplication Support\u002Fscrape-it-now`（macOS）\n- `C:\\Users\\\u003Cuser>\\AppData\\Roaming\\scrape-it-now`（Windows）\n\n### 浏览器二进制文件安装\n\n浏览器二进制文件会在每次运行时自动下载或更新。浏览器为 Chromium，不可配置（如有需要其他浏览器，请提交 issue），大小约为 450MB。缓存存储在缓存目录中。\n\n### 本地磁盘存储的工作原理\n\n本地磁盘存储同时用于 Blob 和队列。不建议在生产环境中使用，因为它不易扩展且不具备容错能力。它适用于测试和开发，或者在无法使用 Azure 服务的情况下。\n\n实现方式：\n\n- 本地磁盘 Blob 存储使用目录结构来保存 Blob 数据。每个 Blob 都以文件的形式存储，文件名即为 Blob 的名称。租约机制通过锁文件实现。默认情况下，文件会存储在与命令执行目录相对的某个目录中。\n- 本地磁盘队列使用 SQLite 数据库来存储消息。数据库文件存储在缓存目录中。SQL 数据库实现了可见性超时和删除标记功能，以确保与无状态队列服务（如 Azure 队列存储）的一致性。\n\n### 使用代理实现匿名性\n\n应用程序本身并未实现代理功能。仅靠应用层无法实现网络安全性。请使用 VPN（例如您自己的或第三方提供的）或代理服务（例如住宅代理、Tor）来确保匿名性，并配置系统防火墙以限制应用程序对网络的访问权限。\n\n### 打包为容器镜像\n\n由于该应用已发布到 PyPI，因此可以轻松地打包成容器镜像。每次启动时，应用都会下载并缓存所需的依赖项（如浏览器等）。您也可以预先下载这些依赖项，只需运行 `scrape-it-now scrape install` 命令即可。\n\n为了提升性能，还可以通过运行多个容器实例来并行化爬取和索引任务。这可以通过 [KEDA](https:\u002F\u002Fkeda.sh) 实现，具体方法是配置一个 [Azure 存储队列缩放器](https:\u002F\u002Fkeda.sh\u002Fdocs\u002F2.16\u002Fscalers\u002Fazure-storage-queue)。","# Scrape It Now 快速上手指南\n\nScrape It Now 是一款专为 AI 应用设计的网页爬虫工具。它支持并行抓取，能够自动拦截广告、处理动态 JavaScript 内容，并将网页转换为高质量的 Markdown 格式，同时支持构建语义搜索索引。\n\n## 环境准备\n\n### 系统要求\n- **Python 版本**：必须使用 **Python 3.13** 或更高版本。\n  - *提示*：如果系统未安装该版本，推荐使用 [pyenv](https:\u002F\u002Fgithub.com\u002Fpyenv\u002Fpyenv) 进行多版本管理。\n- **操作系统**：支持 Linux、macOS 和 Windows（需配置相应运行环境）。\n\n### 前置依赖\n- **浏览器内核**：工具内部使用 Playwright 和 Chromium 处理动态内容，首次运行时会自动下载所需依赖（可能需要几分钟）。\n- **存储后端（可选）**：\n  - **本地模式**：无需额外配置，直接使用本地磁盘。\n  - **云端模式**：如需高性能并行处理，需准备 Azure Storage (Queue & Blob) 和 Azure AI Search 账号及密钥。\n\n## 安装步骤\n\n### 方式一：通过 PyPI 安装（推荐）\n\n使用 pip 直接安装最新稳定版：\n\n```bash\npython3 -m pip install scrape-it-now\n```\n\n验证安装并查看帮助：\n\n```bash\nscrape-it-now --help\n```\n\n### 方式二：从源码安装\n\n适用于需要开发或体验最新特性的用户：\n\n```bash\n# 克隆仓库\ngit clone https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now.git\ncd scrape-it-now\n\n# 安装依赖（需确保已安装 make 工具）\nmake install dev\n\n# 验证安装\nscrape-it-now --help\n```\n\n## 基本使用\n\n本工具支持两种运行模式：**本地磁盘模式**（无需云账号，适合测试）和 **Azure 云端模式**（适合生产环境并行抓取）。以下以最简单的**本地磁盘模式**为例。\n\n### 1. 抓取网站内容\n\n设置环境变量启用本地存储，然后运行抓取任务。以下示例将抓取 `nytimes.com` 首页及其链接页面，并转换为 Markdown。\n\n```bash\n# 配置使用本地磁盘作为存储和队列\nexport BLOB_PROVIDER=local_disk\nexport QUEUE_PROVIDER=local_disk\n\n# 执行抓取任务\nscrape-it-now scrape run https:\u002F\u002Fnytimes.com\n```\n\n**运行说明：**\n- 首次运行会自动下载浏览器依赖。\n- 工具会自动去重链接、拦截广告，并将内容保存为 Markdown。\n- 默认会递归抓取一定深度的页面（可通过 `--max-depth` 调整）。\n\n**常用参数：**\n- `-si` \u002F `--save-images`：同时保存图片资源。\n- `-ss` \u002F `--save-screenshot`：保存页面截图。\n- `-md` \u002F `--max-depth`：设置最大抓取深度（例如 `-md 2`）。\n\n### 2. 查看任务状态\n\n抓取过程中或完成后，可以查看当前任务的进度统计（如已处理 URL 数、网络用量等）。\n\n```bash\n# 确保环境变量一致\nexport BLOB_PROVIDER=local_disk\n\n# 查看状态（[job_name] 可省略，默认显示最近的任务，或指定具体任务 ID）\nscrape-it-now scrape status\n```\n\n输出示例：\n```json\n{\"created_at\":\"2024-11-08T13:18:52.839060Z\",\"last_updated\":\"2024-11-08T13:19:16.528370Z\",\"network_used_mb\":2.67,\"processed\":1,\"queued\":311}\n```\n\n### 3. 构建 AI 搜索索引（可选）\n\n如果你需要将抓取到的内容转换为向量索引以供 AI 检索，可以使用 `index` 命令。此步骤通常需要配置 Azure OpenAI 和 Azure AI Search。\n\n```bash\n# 配置 Azure OpenAI (用于生成嵌入向量)\nexport AZURE_OPENAI_API_KEY=xxx\nexport AZURE_OPENAI_ENDPOINT=xxx\nexport AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME=xxx\n# ... 其他 OpenAI 配置\n\n# 配置 Azure Search\nexport AZURE_SEARCH_API_KEY=xxx\nexport AZURE_SEARCH_ENDPOINT=xxx\n\n# 使用本地抓取的數據構建索引\nexport BLOB_PROVIDER=local_disk\nexport QUEUE_PROVIDER=local_disk\n\n# 运行索引任务（替换 [job_name] 为实际任务 ID）\nscrape-it-now index run [job_name]\n```\n\n### 数据存储结构\n\n在本地模式下，抓取的数据将保存在当前目录下的文件夹中，结构如下：\n\n```text\n[job_name]-scraping\u002F\n├── scraped\u002F            # 抓取的页面数据\n│   ├── [page_id]\u002F      # 单个页面的资源\n│   │   ├── screenshot.jpeg\n│   │   └── [image_id].jpg\n│   └── [page_id].json  # 页面元数据和 Markdown 内容\n├── state\u002F              # 任务状态缓存（用于断点续传和并行控制）\n└── job.json            # 任务汇总统计\n```\n\n你可以直接读取 `.json` 文件中的 `content` 字段获取清洗后的 Markdown 内容。","某金融科技团队需要每日监控全球数百家新闻网站，将最新的市场动态提取并导入内部 AI 知识库，以辅助量化交易策略的生成。\n\n### 没有 scrape-it-now 时\n- **数据清洗耗时巨大**：爬虫抓取的原始 HTML 包含大量广告、导航栏和脚本代码，开发人员需编写复杂的正则表达式手动清洗，才能提取出纯文本。\n- **动态内容无法获取**：许多现代新闻站采用 JavaScript 异步加载，传统静态爬虫只能抓到空白页面，导致关键行情数据缺失。\n- **重复抓取浪费资源**：缺乏智能去重机制，程序会反复抓取未更新的页面，不仅浪费带宽和计算资源，还容易触发目标网站的反爬封锁。\n- **非结构化数据难利用**：抓取的内容是杂乱的文本片段，无法直接用于大模型检索，团队需额外开发分块（Chunking）和向量化流程。\n\n### 使用 scrape-it-now 后\n- **一键输出高质量 Markdown**：scrape-it-now 内置 Pandoc 引擎，自动过滤广告并直接将网页转换为结构清晰的 Markdown，省去了所有清洗代码。\n- **完美支持动态渲染**：基于 Playwright 和 Chromium，它能像真实用户一样执行 JavaScript，确保完整捕获异步加载的实时财经数据。\n- **智能去重与并行加速**：工具自动检测页面变更，仅抓取新内容，并支持多任务并行处理，将原本数小时的采集任务缩短至分钟级。\n- **原生适配 AI 检索**：内置 Indexer 模块可自动对内容进行语义分块并生成 OpenAI 嵌入向量，直接构建可搜索的 AI 知识库索引。\n\nscrape-it-now 将繁琐的“采集 - 清洗 - 格式化 - 向量化”全流程自动化，让团队能专注于核心策略分析而非数据工程。","https:\u002F\u002Foss.gittoolsai.com\u002Fimages\u002Fclemlesne_scrape-it-now_053ff7ed.png","clemlesne","Clémence Lesné","https:\u002F\u002Foss.gittoolsai.com\u002Favatars\u002Fclemlesne_9b3ad16c.jpg","Founder - @dualeai","@dualeai","Paris, France","clemence@lesne.pro",null,"https:\u002F\u002Flesne.pro","https:\u002F\u002Fgithub.com\u002Fclemlesne",[86,90,94,98],{"name":87,"color":88,"percentage":89},"Python","#3572A5",95.9,{"name":91,"color":92,"percentage":93},"Makefile","#427819",1.5,{"name":95,"color":96,"percentage":97},"Shell","#89e051",1.4,{"name":99,"color":100,"percentage":101},"HTML","#e34c26",1.2,540,22,"2026-03-07T08:56:34","Apache-2.0","Linux, macOS, Windows","未说明",{"notes":109,"python":110,"dependencies":111},"该工具支持两种运行模式：1. 本地模式：使用本地磁盘存储数据和队列，无需云配置；2. 云端模式：需配置 Azure Storage (Blob\u002FQueue) 和 Azure AI Search。若启用动态 JS 内容抓取，首次运行会自动安装 Playwright 及其浏览器内核（Chromium）。若启用索引功能，需配置 Azure OpenAI 用于生成嵌入向量。","3.13+",[112,113,114,115,116,117,118,119],"playwright","chromium","pandoc","azure-storage-blob","azure-storage-queue","azure-search-documents","openai","sqlite",[14,53,13,15],[122,123,124,125,126],"ai","azure","cli","markdown","scraper","2026-03-27T02:49:30.150509","2026-04-06T07:05:49.103152",[130,135,140,145,150,154],{"id":131,"question_zh":132,"answer_zh":133,"source_url":134},10691,"AI 搜索索引中增加了哪些新字段？","为了增强搜索能力，AI 搜索索引已更新并增加了以下列：标题 (Title)、分块编号 (Chunk number)、抓取日期 (Scraped date)、第一级路径 (First path) 以及网络位置 (Netloc，包含域名和端口)。","https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fissues\u002F30",{"id":136,"question_zh":137,"answer_zh":138,"source_url":139},10692,"是否支持通过代理或 Tor 进行浏览抓取？","目前该功能已被中止（Aborted）。虽然曾考虑过支持 HTTP 代理、Tor 连接以及自动获取代理列表等功能，但维护者决定暂不实现这些特性，相关讨论已在文档中记录。","https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fissues\u002F16",{"id":141,"question_zh":142,"answer_zh":143,"source_url":144},10688,"如何安装 scrape-it-now？","项目已发布详细的安装指南和预编译的二进制文件。您可以查看 README 中的安装部分获取说明，或者直接下载 releases 页面发布的二进制文件使用 CLI。","https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fissues\u002F7",{"id":146,"question_zh":147,"answer_zh":148,"source_url":149},10689,"开发环境安装时提示 .python-version 指向的版本不存在怎么办？","这通常是因为本地 pyenv 环境中缺少对应的虚拟环境名称。如果您使用 pyenv，可以通过运行命令 `pyenv virtualenv 3.12 scrapeitnow312` 来创建名为 scrapeitnow312 的虚拟环境（基于 Python 3.12），从而匹配项目配置。","https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fissues\u002F14",{"id":151,"question_zh":152,"answer_zh":153,"source_url":149},10690,"为什么无法使用 pip-sync 或提示缺少 pip-tools？","不需要手动单独安装 pip-tools。项目提供的 `make install` 命令会自动处理依赖安装，其中包含安装 pip-tools 的步骤。请直接运行 `make install` 即可完成环境配置。",{"id":155,"question_zh":156,"answer_zh":157,"source_url":158},10693,"CI\u002FCD 任务因 actions\u002Fcache 版本过时失败如何解决？","GitHub 已弃用旧版缓存后端。解决方案是将工作流文件 (.github\u002Fworkflows\u002F*.yml) 中的 `actions\u002Fcache@v4.1.2` 更新为浮动的重大版本号 `actions\u002Fcache@v4` 或最新的补丁版本（如 `@v4.2.0`）。同时建议移除旧的 SHA 锁定，并配置 Dependabot 以自动监控后续更新。","https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fissues\u002F45",[160,165,170,175,180,185,190,195,200,205,210,215,220,225,230,235,240,245,250,255],{"id":161,"version":162,"summary_zh":163,"released_at":164},71275,"v3.0.4","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv3.0.3...v3.0.4","2024-12-05T15:19:53",{"id":166,"version":167,"summary_zh":168,"released_at":169},71276,"v3.0.3","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv3.0.2...v3.0.3","2024-12-04T22:37:35",{"id":171,"version":172,"summary_zh":173,"released_at":174},71277,"v3.0.2","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv3.0.1...v3.0.2","2024-11-09T12:56:07",{"id":176,"version":177,"summary_zh":178,"released_at":179},71278,"v3.0.1","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv3.0.0...v3.0.1","2024-11-09T12:40:53",{"id":181,"version":182,"summary_zh":183,"released_at":184},71279,"v3.0.0","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv2.1.3...v3.0.0","2024-11-09T12:23:54",{"id":186,"version":187,"summary_zh":188,"released_at":189},71280,"v2.1.3","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv2.1.2...v2.1.3","2024-11-07T18:48:51",{"id":191,"version":192,"summary_zh":193,"released_at":194},71281,"v2.1.2","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv2.1.1...v2.1.2","2024-11-07T18:45:27",{"id":196,"version":197,"summary_zh":198,"released_at":199},71282,"v2.1.1","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv2.1.0...v2.1.1","2024-11-07T18:05:41",{"id":201,"version":202,"summary_zh":203,"released_at":204},71283,"v2.1.0","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv2.0.3...v2.1.0","2024-11-07T16:00:52",{"id":206,"version":207,"summary_zh":208,"released_at":209},71284,"v2.0.3","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv2.0.2...v2.0.3","2024-10-28T10:19:11",{"id":211,"version":212,"summary_zh":213,"released_at":214},71285,"v2.0.2","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv2.0.1...v2.0.2","2024-10-28T10:08:12",{"id":216,"version":217,"summary_zh":218,"released_at":219},71286,"v2.0.1","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv2.0.0...v2.0.1","2024-10-27T14:36:41",{"id":221,"version":222,"summary_zh":223,"released_at":224},71287,"v2.0.0","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv1.6.0...v2.0.0","2024-10-27T14:04:37",{"id":226,"version":227,"summary_zh":228,"released_at":229},71288,"v1.6.0","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv1.5.1...v1.6.0","2024-08-23T18:44:51",{"id":231,"version":232,"summary_zh":233,"released_at":234},71289,"v1.5.1","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv1.5.0...v1.5.1","2024-08-22T20:11:52",{"id":236,"version":237,"summary_zh":238,"released_at":239},71290,"v1.5.0","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv1.4.4...v1.5.0","2024-08-20T12:14:28",{"id":241,"version":242,"summary_zh":243,"released_at":244},71291,"v1.4.4","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv1.4.3...v1.4.4","2024-08-17T21:41:53",{"id":246,"version":247,"summary_zh":248,"released_at":249},71292,"v1.4.3","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv1.4.2...v1.4.3","2024-08-17T21:18:02",{"id":251,"version":252,"summary_zh":253,"released_at":254},71293,"v1.4.2","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002Fv1.4.1...v1.4.2","2024-08-17T21:05:01",{"id":256,"version":257,"summary_zh":258,"released_at":259},71294,"v1.4.1","**Full Changelog**: https:\u002F\u002Fgithub.com\u002Fclemlesne\u002Fscrape-it-now\u002Fcompare\u002F1.4.0...1.4.1","2024-08-17T20:34:38"]