{"full_name":"firecrawl/firecrawl","owner":"firecrawl","name":"firecrawl","url":"https://github.com/firecrawl/firecrawl","description":"Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥","homepage":"https://firecrawl.dev","topics":[],"language":"TypeScript","license":"AGPL-3.0","avatar":"https://avatars.githubusercontent.com/u/135057108?v=4","created_at":"2024-04-15T21:02:29Z","pushed_at":"2026-11-10T03:06:24Z","metrics":{"stars":200296,"forks":10538,"watchers":498,"issues":1028,"prs":4438,"releases":35,"issues_open":null,"issues_closed":null,"prs_open":null,"prs_closed":null,"prs_merged":null,"discussions":76},"growth":{"day":{"stars":364,"forks":18,"base_day":"2026-11-09"},"week":{"stars":2376,"forks":80,"base_day":"2026-11-03"},"month":{"stars":11446,"forks":510,"base_day":"2026-10-11"}},"history":{"dates":["2026-09-06","2026-09-07","2026-09-08","2026-09-09","2026-09-10","2026-09-11","2026-09-12","2026-09-13","2026-09-14","2026-09-15","2026-09-16","2026-09-17","2026-09-18","2026-09-19","2026-09-20","2026-09-21","2026-09-22","2026-09-23","2026-09-24","2026-09-25","2026-09-26","2026-09-27","2026-09-28","2026-09-29","2026-09-30","2026-10-01","2026-10-02","2026-10-03","2026-10-04","2026-10-05","2026-10-06","2026-10-07","2026-10-08","2026-10-09","2026-10-10","2026-10-11","2026-10-12","2026-10-13","2026-10-14","2026-10-15","2026-10-16","2026-10-17","2026-10-18","2026-10-19","2026-10-20","2026-10-21","2026-10-22","2026-10-23","2026-10-24","2026-10-25","2026-10-26","2026-10-27","2026-10-28","2026-10-29","2026-10-30","2026-10-31","2026-11-01","2026-11-02","2026-11-03","2026-11-04","2026-11-09","2026-11-10"],"stars":[171768,172258,172729,173174,173351,173562,173974,175222,175654,176076,176461,176828,177575,177969,178317,178765,179144,179511,180361,180792,181219,181618,181954,182223,182902,183326,183779,184215,184646,185045,185908,186536,187524,187881,188208,188850,189127,189444,189583,189873,190181,190785,191147,191513,191854,192164,192438,193050,193371,193858,194218,194613,194941,195632,196006,196365,196721,197048,197920,198265,199932,200296],"forks":[9508,9519,9534,9548,9551,9556,9574,9620,9628,9637,9653,9669,9693,9704,9714,9726,9748,9762,9790,9809,9820,9833,9842,9850,9879,9886,9902,9909,9921,9928,9962,9983,10002,10009,10012,10028,10040,10044,10056,10069,10085,10130,10143,10126,10143,10154,10168,10194,10209,10227,10253,10269,10285,10323,10349,10362,10381,10406,10458,10473,10520,10538],"watchers":[441,443,444,446,447,446,447,454,453,456,458,460,461,463,462,462,462,461,467,469,468,468,469,471,469,470,471,475,476,478,480,480,479,481,483,484,485,487,485,487,490,489,490,492,490,491,492,492,494,495,495,494,488,492,495,496,494,495,494,495,498,498],"issues_open":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"prs_open":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null]},"boards":{"daily":96,"weekly":51,"monthly":49,"daily_rate":196,"weekly_rate":195,"monthly_rate":196,"total":33},"same_owner":[{"full_name":"firecrawl/anydoc","name":"anydoc","boards":{}},{"full_name":"firecrawl/pdf-inspector","name":"pdf-inspector","boards":{}}],"issues":[{"number":1169,"title":"[Bug] Add an identifiable user agent so that I can prevent it from crawling my site","comments":18,"created_at":"2025-02-11T18:39:25Z"},{"number":839,"title":"[Feat] Ability to scrape embedded pdfs","comments":14,"created_at":"2024-10-28T21:01:44Z"},{"number":4459,"title":"RFC: deterministic policy layer for agent tool calls (credit caps, interact egress, self-host SSRF)","comments":5,"created_at":"2026-08-30T15:46:02Z"},{"number":4137,"title":"[Self-Host] Building with Podman instead of Docker","comments":5,"created_at":"2026-07-27T09:52:59Z"},{"number":4113,"title":"[Self-Host] nuq-worker crash-loops the api container when scraped content contains NUL bytes (Postgres 22P05)","comments":5,"created_at":"2026-07-23T16:24:07Z"},{"number":3552,"title":"[Bug] firecrawl agent fails with \"Refusal: Error: Agent reached max credits\" on every SPA careers URL, regardless of model or credit cap","comments":5,"created_at":"2026-05-18T04:39:53Z"},{"number":5022,"title":"[Bug] removeBase64Images deletes text and table cells when a non-base64 data:image comes before a base64 image on the same line","comments":4,"created_at":"2026-10-08T16:57:57Z"},{"number":3968,"title":"[Bug] Cached PDF scrape returns markdown for a different URL","comments":4,"created_at":"2026-07-08T11:05:59Z"}],"discussions":[{"number":5637,"title":"Incremental / Change-Aware Crawling","comments":0,"category":"Ideas","created_at":"2026-11-08T19:20:27Z"},{"number":5314,"title":"Help - Absolute Beginner","comments":2,"category":"General","created_at":"2026-10-21T19:47:46Z"},{"number":5168,"title":"Showcase: ATS Zero-Browser Gateway — Fast protocol scraper & MCP server for Workday & ATS platforms","comments":0,"category":"Show and tell","created_at":"2026-10-15T20:48:58Z"},{"number":4855,"title":"A short guide to choosing Firecrawl scrape or crawl","comments":1,"category":"Show and tell","created_at":"2026-09-30T03:57:42Z"},{"number":4769,"title":"Feature idea: return a token count with scraped content — served context windows are much smaller than model cards claim","comments":1,"category":"Ideas","created_at":"2026-09-26T12:30:05Z"},{"number":4726,"title":"EnvShield adoption request.","comments":0,"category":"General","created_at":"2026-09-22T13:26:55Z"},{"number":4692,"title":"Rust-based document converter for scraped docs → structured Markdown (Founder's Lock)","comments":0,"category":"General","created_at":"2026-09-18T09:24:48Z"},{"number":4666,"title":"Is it useful for real time data?","comments":2,"category":"General","created_at":"2026-09-16T22:42:20Z"}],"latest_release":{"name":"Firecrawl v2.11.0","tag":"v2.11.0","published_at":"2026-06-19T15:09:30Z","body":"# Firecrawl v2.11.0\r\n \r\n## Improvements\r\n \r\n- **Firecrawl Research Index** — Added a specialized index for agentic AI/ML research: search across 3M+ arXiv papers and the GitHub code behind them (issues, merged PRs, and READMEs, refreshed daily), fetch a paper's details or related work, and check claims against full text. It has state-of-the-art recall on arXivQA, outperforming the next best provider by 18% at comparable cost. Available via the API, SDKs, MCP, and CLI.\r\n- **Keyless access for core endpoints** — Use `/scrape`, `/search`, `/interact`, and `/parse` without an API key from official MCP, CLI, and SDK clients.\r\n- **Automatic PII redaction** — Added a `redactPII` option that strips personal and sensitive data like names, emails, phone numbers, addresses, and secrets out of scraped content before it's returned.\r\n- **`deterministicJson` format** — Added a format that returns structured JSON without running an LLM on every request. Firecrawl generates a reusable extractor for your schema and caches it per site, so repeat scrapes are cheaper and return consistent results.\r\n- **Video discovery on any page** — Expanded the `video` format to find videos on any page, not just supported providers like YouTube, returning each video's URL, title, thumbnail, duration, and more.\r\n- **Attach your own browser automation** — Added a CDP WebSocket URL (`cdpUrl`) to browser session responses, so you can drive a live Firecrawl browser session directly with Playwright, Puppeteer, or any other CDP client.\r\n- **Smarter monitor alerts** — Added a `goal` to monitors so an LLM judges each detected change as meaningful or noise against what you actually care about, cutting alert spam and surfacing the changes that matter first in summary emails.\r\n- **Field-level JSON diffs for monitors** — Monitors that scrape in JSON mode now compare the actual field values between runs instead of the rendered page, so you see exactly which fields changed rather than noise from layout shifts.\r\n- **Monitor email confirmation** — Added an opt-in confirmation flow with one-click unsubscribe for external monitor recipients; team members are auto-confirmed, and `Monitor` responses now report each recipient's subscription status.\r\n- **AM/PM monitor schedules** — Added support for 12-hour schedule inputs like `daily at 9am` and `daily at 5:30pm`, converted to the correct UTC cron expression.\r\n- **Monitor webhook delivery status** — Added delivery status to each monitor check, so you can see whether its webhook was attempted, delivered, or failed — and why.\r\n- **Steadier monitor checks** — Monitors now wait for pages to finish rendering before diffing, cutting false alerts caused by partially-loaded pages.\r\n- **PDF size cap** — Raised the PDF download and scrape size cap from 30 MB to 50 MB.\r\n- **Python SDK `crawl()` scrape kwargs** — Added direct scrape kwargs (`formats`, `headers`, `include_tags`, `exclude_tags`, etc.) to `crawl()` and `start_crawl()`, removing the need to wrap them in `ScrapeOptions(...)`.\r\n- **Clearer `.data` errors** — Improved the error raised when accessing `.data` on a search result to point at `.web`, `.news`, and `.images` with their counts, instead of returning a silent `None`.\r\n- **Python format defaults** — Removed the required `type=` argument on `JsonFormat` and `ChangeTrackingFormat`, defaulting it like `ScreenshotFormat`.\r\n- **`ChangeTrackingFormat` casing** — Added acceptance of both `change_tracking` and `changeTracking` for the format `type` so payloads round-trip between snake_case and camelCase clients.\r\n## Fixes\r\n \r\n- Resolved security advisories across the API and SDKs by upgrading `axios`, `esbuild`, `ws`, `openssl`, and other dependencies.\r\n- Fixed scrape workers stalling for tens of seconds on very large LLM-extractor inputs, which previously caused dropped jobs and worker restarts.\r\n- Fixed Wikipedia scrapes missing `metadata.ogImage` on roughly half of Wikimedia URLs.\r\n- Fixed crawl and batch cancellation not draining the"},"readme":true,"readme_zh":false,"news":[{"title":"Show HN: Get structured website data with just a prompt","url":"https://news.ycombinator.com/item?id=42770766","hn":"https://news.ycombinator.com/item?id=42770766","source":"Hacker News","lang":"en","date":"2025-01-20","points":2,"comments":0},{"title":"Firecrawl Announces $14.5 Million in Series A Funding to Put Web Data on Tap for AI Agents","url":"https://www.manilatimes.net/2025/08/19/tmt-newswire/globenewswire/firecrawl-announces-145-million-in-series-a-funding-to-put-web-data-on-tap-for-ai-agents/2170125","source":"manilatimes","lang":"en","date":"2025-08-19"},{"title":"AI crawler Firecrawl raises $14.5M, is still looking to hire agents as employees","url":"https://techcrunch.com/2025/08/19/ai-crawler-firecrawl-raises-14-5m-is-still-looking-to-hire-agents-as-employees/","source":"TechCrunch","lang":"en","date":"2025-08-19"},{"title":"Firecrawl raises $14.5M to grow AI-ready web data infrastructure","url":"https://siliconangle.com/2025/08/19/firecrawl-raises-14-5m-grow-ai-ready-web-data-infrastructure/","source":"SiliconANGLE","lang":"en","date":"2025-08-19"},{"title":"Firecrawl: Easy web data extraction for AI applications","url":"https://www.infoworld.com/article/4009041/firecrawl-easy-web-data-extraction-for-ai-applications.html","source":"InfoWorld","lang":"en","date":"2025-06-19"},{"title":"Y Combinator startup Firecrawl is ready to pay $1M to hire three AI agents as employees","url":"https://techcrunch.com/2025/05/17/y-combinator-startup-firecrawl-is-ready-to-pay-1m-to-hire-three-ai-agents-as-employees/","source":"TechCrunch","lang":"en","date":"2025-05-17"},{"title":"Firecrawl: A Powerful Web Scraping Tool for Turning Websites into Large Language Model (LLM) Ready Markdown or Structured Data","url":"https://www.marktechpost.com/2024/06/20/firecrawl-a-powerful-web-scraping-tool-for-turning-websites-into-large-language-model-llm-ready-markdown-or-structured-data/","source":"marktechpost","lang":"en","date":"2024-06-20"},{"title":"Firecrawl融资7500万美元：给AI Agent建一个「数据源总入口」","url":"https://news.google.com/rss/articles/CBMiVkFVX3lxTE9xcmFZOTZreTVWdWIyeVpxdHVfaW5zTmdOZU1odnpYQm1Qal9pWUFtaWo3bVE4TkZpU2JENi1KVHdaNnBOVUpEdl9lZmhfNm1pWVRpekJB?oc=5","source":"BlockBeats","lang":"zh","date":"2026-09-22"},{"title":"Firecrawl – 适用于 RAG 与 AI 智能体的终极网页数据 API","url":"https://news.google.com/rss/articles/CBMiZEFVX3lxTE9SRTA0d2prdnhwYjl1YUdLVWFhRHRUVDlOV1NaSXpoc2xxZVhsWE9qSlZsSkFnbmpOZ3dWVXFaNzRUTmFBeElPd3NHTzFUS2t3WFktYXhSMmhtUFByUHctMm1DY0g?oc=5","source":"quasa.io","lang":"zh","date":"2026-09-21"},{"title":"5 Claude, OpenClaw, and Hermes skills that deserve to be in your toolkit","url":"https://news.google.com/rss/articles/CBMigAFBVV95cUxQRDRrc1ozMnpRSVJlLXJxNFFaVDlSeHVnaC1vaFN6MElfWHNDWWFMVlZSZHRxbTBVX3hPY1NYWnctVm9MdU1IdEdYRTBrZzdxa2dxNXZuZlllYW9mU25Sb2RIYmdmOFBnTlRGdHpTX05jRGh0aWU0NFJ6ekRZOVdUeQ?oc=5","source":"MakeUseOf","lang":"en","date":"2026-07-21"},{"title":"10 GitHub Repos for Scraping the Internet in 2026 (Real Stars)","url":"https://news.google.com/rss/articles/CBMiiAFBVV95cUxPMWZEd05SYXh2UUhNa2I4S2JBZjRfVktYUnhTV1FBdW5JMFgxRGVqWXdybGUxUkZyQTZGTGpYTlZNdXhoWTF3Ymd5T1dnTWJORExBdjJNTXpoQmQ3UnNQaVREMVRHMnZZWjdtTDZFcVllb3hISjVuVkN0VGY4NUV0RE50cWdDR0NR?oc=5","source":"Pasquale Pillitteri","lang":"en","date":"2026-06-30"},{"title":"2026 年抓取互联网的 10 个 GitHub 仓库 (附真实星标)","url":"https://news.google.com/rss/articles/CBMihAFBVV95cUxORGZEaDBndmMzUDdFYUdOTnRmeG9QQUw0RHRIZUFCTlNTMncyTjJIUWRWOVpLejNMa011VU5XeXNaZzQ1R0dEWk1IMWQweUFNZ2hyQjJMdlE3cy0xSDYxYTB0QllyOFh0QU1nNWxXVjRTazVzdDd1MTJ5R2VnbXdVRFpVenA?oc=5","source":"Pasquale Pillitteri","lang":"zh","date":"2026-06-30"},{"title":"Firecrawl Launches Open Lovable with UGC Integration","url":"https://news.google.com/rss/articles/CBMijwFBVV95cUxPSmJXSWhTaTY4NUt3eFhfOW80S1VPWm5DYV9Zc2lNc3cycFBRcWdnbDR0OU5ibU15Rm84ek9YcGJDam9ma0NQZDlqTVlZZWd1enE0elkwcG5zMlZpNmxrWWF3R28zZHZ1VWJXb2dZMDZLZXFReHBpdE1Nc1p3UDFueXQ5MFMtT3dDakN3UUt1TQ?oc=5","source":"Dealroom","lang":"en","date":"2026-06-29"},{"title":"给OpenClaw开天眼！解决了10个跨境电商网站爬虫难题","url":"https://news.google.com/rss/articles/CBMiVkFVX3lxTE9ESS1TZmdRZE9iN0NEdU1QY2NtREc0M2Q0QWRCYjRLajYzQU01M3ZHVTNaZTluZllHTHpYdjkyMHhTYS02QTVhR1JRS2F5Z2lTSVprQktn?oc=5","source":"腾讯新闻","lang":"zh","date":"2026-03-03"},{"title":"Firecrawl Brings Autonomous Web Search and Extraction Onchain with x402","url":"https://news.google.com/rss/articles/CBMigAFBVV95cUxOM2dpT0dSTlBnQVV6UnFjb3FwQWw4cV9NYUxJUlpURXF6RFpnam91RzFzekJGMDZaNzdBeDBCT3lkUFRERmZncDFrTGhHbUc1XzY4RmVaQVQ2ZGJXZlMxZ1drdUtOWk84M1VBNVBEejBwRVVTRWJWU2o4ejczMVZGdQ?oc=5","source":"Coinbase","lang":"en","date":"2025-08-28"},{"title":"RAG实践总结：将互联网数据融入LLMs的工具推荐","url":"https://news.google.com/rss/articles/CBMicEFVX3lxTFBZUERJLVlSRGtnLWdYQjdWZWZ3Zmh6N05IQW9reU9Va0lvTjdyck54dngwbHFKaGhIdkZjNERMZmhXOEtZVUdtNk9EbC1kWndEaHc4SlFpOFRwTTNueWhQaF9MVEpPNGlXMEdFbE1xbXc?oc=5","source":"53AI","lang":"zh","date":"2024-05-31"}],"stale":false,"snapshot_at":"2026-11-10T09:35:25Z","history_start":"2026-09-06"}