快速开始

欢迎使用 Firecrawl

Firecrawl 是一项 API 服务，可接收 URL、进行爬取，并将其转换为干净的 Markdown。我们会爬取所有可访问的子页面，并为每个页面提供干净的 Markdown。无需提供 sitemap。

如何使用？

我们提供托管版的易用 API。你可以在此处找到 playground 和文档。你也可以选择自托管后端。从以下资源开始：

API：文档
SDKs： Python、Node、Go、Rust
LLM 框架： LangChain（Python）、LangChain（JS）、LlamaIndex、Crew.ai、Composio、PraisonAI、Superinterface、Vectorize
低代码框架： Dify、Langflow、Flowise AI、Cargo、Pipedream
其他： Zapier、Pabbly Connect
需要某个 SDK 或集成？欢迎提交 issue 告诉我们。

自托管： 请参考此处的指南。

API Key

要使用 API，你需要在 Firecrawl 注册并获取 API Key。

爬取

用于爬取一个 URL 及其所有可访问的子页面。该操作会提交一个爬取任务，并返回任务 ID 以便查询爬取状态。

安装

pip install firecrawl-py

npm install firecrawl

go get github.com/mendableai/firecrawl-go

# 在 Cargo.toml 中添加以下内容

[dependencies]
firecrawl = "^0.1"
tokio = { version = "^1", features = ["full"] }
serde = { version = "^1.0", features = ["derive"] }
serde_json = "^1.0"
uuid = { version = "^1.10", features = ["v4"] }

[build-dependencies]
tokio = { version = "1", features = ["full"] }

用法

from firecrawl import Firecrawl

app = Firecrawl(api_key="YOUR_API_KEY")

crawl_result = app.crawl_url('docs.firecrawl.dev', {'crawlerOptions': {'excludes': ['blog/*']}})

# 获取 Markdown
for result in crawl_result:
    print(result['markdown'])

import { Firecrawl } from "firecrawl";

// 使用你的 API 密钥初始化 Firecrawl
const app = new Firecrawl({ apiKey: "YOUR_API_KEY" });

// 抓取网站
const crawlResult = await app.crawlUrl("docs.firecrawl.dev", {
  crawlerOptions: { excludes: ["blog/*"] },
});

// 输出 Markdown
console.log(crawlResult.map((result) => result.markdown));

import (
  "fmt"
  "log"

  "github.com/mendableai/firecrawl-go"
)

func main() {
  // 使用你的 API 密钥初始化 Firecrawl
  app, err := firecrawl.NewFirecrawlApp("YOUR_API_KEY")
  if err != nil {
    log.Fatalf("初始化 Firecrawl 失败: %v", err)
  }

  // 抓取网站
  params := map[string]any{
    "crawlerOptions": map[string]any{
      "excludes": []string{"blog/*"},
    },
  }
  crawlResult, err := app.CrawlURL("docs.firecrawl.dev", params)
  if err != nil {
    log.Fatalf("抓取时发生错误: %v", err)
  }

  // 获取 Markdown
  for _, result := range crawlResult {
    fmt.Println(result.Markdown)
  }
}

use firecrawl::Firecrawl;

#[tokio::main]
async fn main() {
  // 使用 API 密钥初始化 Firecrawl
  let api_key = "YOUR_API_KEY";
  let api_url = "https://api.firecrawl.dev";
  let app = Firecrawl::new(api_key, api_url).expect("初始化 Firecrawl 失败");

  // 抓取 URL
  let crawl_params = json!({
    "crawlerOptions": {
        "excludes": ["blog/*"]
    }
  });

  let crawl_result = app
      .crawl_url("https://example.com", Some(crawl_params), true, 2, None)
      .await;

  // 打印抓取结果
  match crawl_result {
      Ok(data) => println!("抓取结果:\n{}", data),
      Err(e) => eprintln!("抓取失败: {}", e),
  }
}

curl -X POST https://api.firecrawl.dev/v0/crawl \
    -H 'Content-Type: application/json' \
    -H 'Authorization: Bearer YOUR_API_KEY' \
    -d '{
      "url": "https://docs.firecrawl.dev"
    }'

如果你不使用 SDK，或更倾向于使用 webhook 或其他轮询方式，可以将 wait_until_done 设置为 false。这将返回一个 jobId。对于 cURL，/crawl 始终会返回一个 jobId，可用它来检查抓取状态。

{ "jobId": "1234-5678-9101" }

检查爬取任务

用于查看爬取任务的状态并获取其结果。

status = app.check_crawl_status(job_id)

const status = await app.checkCrawlStatus(jobId);

status, err := app.CheckCrawlStatus(jobId)
if err != nil {
  log.Fatalf("Failed to check crawl status: %v", err)
}

let status = match app.check_crawl_status(jobId).await {
    Ok(status) => status,
    Err(e) => panic!("Failed to check crawl status: {:?}", e),
};

println!("Crawl Status: {:?}", status);

curl -X GET https://api.firecrawl.dev/v0/crawl/status/1234-5678-9101 \
  -H 'Content-Type: application/json' \
  -H 'Authorization: Bearer YOUR_API_KEY'

响应

{
  "status": "已完成",
  "current": 22,
  "total": 22,
  "data": [
    {
      "content": "原始内容",
      "markdown": "# Markdown 内容",
      "provider": "web-scraper",
      "metadata": {
        "title": "Firecrawl | 为你的 LLM 稳定可靠地抓取网页",
        "description": "用于客户体验与销售的 AI"
        "language": null,
        "sourceURL": "https://docs.firecrawl.dev/"
      }
    }
  ]
}

抓取

要抓取单个 URL，请使用 scrape_url 方法。该方法以 URL 作为参数，返回包含抓取结果的字典。

from firecrawl import Firecrawl

app = Firecrawl(api_key="YOUR_API_KEY")

content = app.scrape_url("https://docs.firecrawl.dev")

import { Firecrawl } from 'firecrawl-js';

const app = new Firecrawl({ apiKey: 'YOUR_API_KEY' });

const content = await app.scrapeUrl('https://docs.firecrawl.dev');

import (
  "log"

  "github.com/mendableai/firecrawl-go"
)

func main() {
  app, err := firecrawl.NewFirecrawlApp("YOUR_API_KEY")
  if err != nil {
    log.Fatalf("Failed to initialize Firecrawl: %v", err)
  }

  content, err := app.ScrapeURL("docs.firecrawl.dev", nil)
  if err != nil {
    log.Fatalf("Failed to scrape URL: %v", err)
  }
}

use firecrawl::Firecrawl;

#[tokio::main]
async fn main() {
    // 使用 API 密钥初始化 Firecrawl
    let api_key = "YOUR_API_KEY";
    let api_url = "https://api.firecrawl.dev";
    let app = Firecrawl::new(api_key, api_url).expect("Failed to initialize Firecrawl");

    // 抓取 URL
    let scrape_result = app.scrape_url("https://example.com", None).await;

    // 打印抓取结果
  match scrape_result {
    Ok(data) => println!("Scrape Result:\n{}", data["markdown"]),
    Err(e) => eprintln!("Scrape failed: {}", e),
  }
}

curl -X POST https://api.firecrawl.dev/v0/scrape \
    -H 'Content-Type: application/json' \
    -H 'Authorization: Bearer YOUR_API_KEY' \
    -d '{
      "url": "https://docs.firecrawl.dev"
    }'

{
  "success": true,
  "data": {
    "markdown": "<字符串>",
    "content": "<字符串>",
    "html": "<字符串>",
    "rawHtml": "<字符串>",
    "metadata": {
      "title": "<字符串>",
      "description": "<字符串>",
      "language": "<字符串>",
      "sourceURL": "<字符串>",
      "<其他元数据>": "<字符串>",
      "pageStatusCode": 123,
      "pageError": "<字符串>"
    },
    "llm_extraction": {},
    "warning": "<字符串>"
  }
}

提取

借助 LLM 提取功能，你可以轻松从任意 URL 提取结构化数据。我们也支持 Pydantic 架构，便于定义数据结构。使用方法如下：

class ArticleSchema(BaseModel):
    title: str
    points: int 
    by: str
    commentsURL: str

class TopArticlesSchema(BaseModel):
top: List[ArticleSchema] = Field(..., max_items=5, description="Top 5 stories")

data = app.scrape_url('https://news.ycombinator.com', {
'extractorOptions': {
'extractionSchema': TopArticlesSchema.model_json_schema(),
'mode': 'llm-extraction'
},
'pageOptions':{
'onlyMainContent': True
}
})
print(data["llm_extraction"])

import { Firecrawl } from "firecrawl";
import { z } from "zod";

const app = new Firecrawl({
  apiKey: "fc-YOUR_API_KEY",
});

// 定义提取内容所用的模式
const schema = z.object({
  top: z
    .array(
      z.object({
        title: z.string(),
        points: z.number(),
        by: z.string(),
        commentsURL: z.string(),
      })
    )
    .length(5)
    .describe("Top 5 stories on Hacker News"),
});

const scrapeResult = await app.scrapeUrl("https://news.ycombinator.com", {
  extractorOptions: { extractionSchema: schema },
});

console.log(scrapeResult.data["llm_extraction"]);

import (
  "fmt"
  "log"

  "github.com/mendableai/firecrawl-go"
)

app, err := NewFirecrawlApp(TEST_API_KEY, API_URL)
if err != nil {
  log.Fatalf("初始化 Firecrawl 失败：%v", err)
}

params := map[string]any{
  "extractorOptions": ExtractorOptions{
    Mode:             "llm-extraction",
    ExtractionPrompt: "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
    ExtractionSchema: map[string]any{
      "type": "object",
      "properties": map[string]any{
        "company_mission": map[string]string{"type": "string"},
        "supports_sso":    map[string]string{"type": "boolean"},
        "is_open_source":  map[string]string{"type": "boolean"},
      },
      "required": []string{"company_mission", "supports_sso", "is_open_source"},
    },
  },
}

scrapeResult, err := app.ScrapeURL("https://news.ycombinator.com", params)
if err != nil {
  log.Fatalf("抓取 URL 失败：%v", err)
}
fmt.Println(scrapeResult.LLMExtraction)

use firecrawl::Firecrawl;

#[tokio::main]
async fn main() {
    // 使用 API 密钥初始化 Firecrawl
    let api_key = "YOUR_API_KEY";
    let api_url = "https://api.firecrawl.dev";
    let app = Firecrawl::new(api_key, api_url).expect("Failed to initialize Firecrawl");

    // 定义用于提取内容的架构
    let json_schema = json!({
        "type": "object",
        "properties": {
            "top": {
                "type": "array",
                "items": {
                    "type": "object",
                    "properties": {
                        "title": {"type": "string"},
                        "points": {"type": "number"},
                        "by": {"type": "string"},
                        "commentsURL": {"type": "string"}
                    },
                    "required": ["title", "points", "by", "commentsURL"]
                },
                "minItems": 5,
                "maxItems": 5,
                "description": "Top 5 stories on Hacker News"
            }
        },
        "required": ["top"]
    });

    let llm_extraction_params = json!({
        "extractorOptions": {
            "extractionSchema": json_schema,
            "mode": "llm-extraction"
        },
        "pageOptions": {
            "onlyMainContent": true
        }
    });

    let llm_extraction_result = app
        .scrape_url("https://news.ycombinator.com", Some(llm_extraction_params))
        .await;
    match llm_extraction_result {
        Ok(data) => println!("LLM Extraction Result:\n{}", data["llm_extraction"]),
        Err(e) => eprintln!("LLM Extraction failed: {}", e),
    }
}

curl -X POST https://api.firecrawl.dev/v0/scrape \
    -H 'Content-Type: application/json' \
    -H 'Authorization: Bearer YOUR_API_KEY' \
    -d '{
      "url": "https://docs.firecrawl.dev/",
      "extractorOptions": {
        "mode": "llm-extraction",
        "extractionPrompt": "Based on the information on the page, extract the information from the schema. ",
        "extractionSchema": {
          "type": "object",
          "properties": {
            "company_mission": {
                      "type": "string"
            },
            "supports_sso": {
                      "type": "boolean"
            },
            "is_open_source": {
                      "type": "boolean"
            },
            "is_in_yc": {
                      "type": "boolean"
            }
          },
          "required": [
            "company_mission",
            "supports_sso",
            "is_open_source",
            "is_in_yc"
          ]
        }
      }
    }'

参与贡献

我们欢迎各类贡献！在提交 pull request 之前，请先阅读我们的贡献指南。

Get Started

Core Endpoints

More

Quickstarts

Developer Guides

Webhooks

Use Cases

Other

Contributing

欢迎使用 Firecrawl

如何使用？

API Key

爬取

安装

用法

检查爬取任务

响应

抓取

返回

提取

参与贡献

​欢迎使用 Firecrawl

​如何使用？

​API Key

​爬取

​安装

​用法

​检查爬取任务

​响应

​抓取

​返回

​提取

​参与贡献

欢迎使用 Firecrawl

如何使用？

API Key

爬取

安装

用法

检查爬取任务

响应

抓取

返回

提取

参与贡献