Firecrawl Batch Scrape

Submit an asynchronous batch scrape job.

curl --request POST \
  --url https://api.aisa.one/apis/v1/firecrawl/batch-scrape \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --header 'Idempotency-Key: <idempotency-key>' \
  --data '
{
  "urls": [
    "https://docs.firecrawl.dev",
    "https://docs.firecrawl.dev/introduction"
  ],
  "maxConcurrency": 10,
  "onlyMainContent": true,
  "includeTags": [
    "<string>"
  ],
  "excludeTags": [
    "<string>"
  ],
  "maxAge": 15768000000,
  "minAge": 15768000000,
  "timeout": 150500
}
'

import requests

url = "https://api.aisa.one/apis/v1/firecrawl/batch-scrape"

payload = {
    "urls": ["https://docs.firecrawl.dev", "https://docs.firecrawl.dev/introduction"],
    "maxConcurrency": 10,
    "onlyMainContent": True,
    "includeTags": ["<string>"],
    "excludeTags": ["<string>"],
    "maxAge": 15768000000,
    "minAge": 15768000000,
    "timeout": 150500
}
headers = {
    "Idempotency-Key": "<idempotency-key>",
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {
    'Idempotency-Key': '<idempotency-key>',
    Authorization: 'Bearer <token>',
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
    urls: ['https://docs.firecrawl.dev', 'https://docs.firecrawl.dev/introduction'],
    maxConcurrency: 10,
    onlyMainContent: true,
    includeTags: ['<string>'],
    excludeTags: ['<string>'],
    maxAge: 15768000000,
    minAge: 15768000000,
    timeout: 150500
  })
};

fetch('https://api.aisa.one/apis/v1/firecrawl/batch-scrape', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.aisa.one/apis/v1/firecrawl/batch-scrape",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'urls' => [
        'https://docs.firecrawl.dev',
        'https://docs.firecrawl.dev/introduction'
    ],
    'maxConcurrency' => 10,
    'onlyMainContent' => true,
    'includeTags' => [
        '<string>'
    ],
    'excludeTags' => [
        '<string>'
    ],
    'maxAge' => 15768000000,
    'minAge' => 15768000000,
    'timeout' => 150500
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json",
    "Idempotency-Key: <idempotency-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.aisa.one/apis/v1/firecrawl/batch-scrape"

	payload := strings.NewReader("{\n  \"urls\": [\n    \"https://docs.firecrawl.dev\",\n    \"https://docs.firecrawl.dev/introduction\"\n  ],\n  \"maxConcurrency\": 10,\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"maxAge\": 15768000000,\n  \"minAge\": 15768000000,\n  \"timeout\": 150500\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Idempotency-Key", "<idempotency-key>")
	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/firecrawl/batch-scrape")
  .header("Idempotency-Key", "<idempotency-key>")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"urls\": [\n    \"https://docs.firecrawl.dev\",\n    \"https://docs.firecrawl.dev/introduction\"\n  ],\n  \"maxConcurrency\": 10,\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"maxAge\": 15768000000,\n  \"minAge\": 15768000000,\n  \"timeout\": 150500\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.aisa.one/apis/v1/firecrawl/batch-scrape")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Idempotency-Key"] = '<idempotency-key>'
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"urls\": [\n    \"https://docs.firecrawl.dev\",\n    \"https://docs.firecrawl.dev/introduction\"\n  ],\n  \"maxConcurrency\": 10,\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"maxAge\": 15768000000,\n  \"minAge\": 15768000000,\n  \"timeout\": 150500\n}"

response = http.request(request)
puts response.read_body

{
  "id": "iaj_01HZY8Q2M4K7N9V3T6W1X0B2C3",
  "object": "integration_async_job",
  "endpoint": "/apis/v1/firecrawl/crawl",
  "createdAt": "2023-11-07T05:31:56Z",
  "completedAt": "2023-11-07T05:31:56Z",
  "pricing": {
    "currency": "USD",
    "authorizedMicrosUSD": 123,
    "finalMicrosUSD": 123,
    "billingMode": "metered_result"
  },
  "output": "<unknown>",
  "outputExpired": true,
  "error": {
    "code": "<string>",
    "message": "<string>"
  }
}

POST

https://api.aisa.one/apis/v1

firecrawl

batch-scrape

Submit an asynchronous batch scrape job.

curl --request POST \
  --url https://api.aisa.one/apis/v1/firecrawl/batch-scrape \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --header 'Idempotency-Key: <idempotency-key>' \
  --data '
{
  "urls": [
    "https://docs.firecrawl.dev",
    "https://docs.firecrawl.dev/introduction"
  ],
  "maxConcurrency": 10,
  "onlyMainContent": true,
  "includeTags": [
    "<string>"
  ],
  "excludeTags": [
    "<string>"
  ],
  "maxAge": 15768000000,
  "minAge": 15768000000,
  "timeout": 150500
}
'

import requests

url = "https://api.aisa.one/apis/v1/firecrawl/batch-scrape"

payload = {
    "urls": ["https://docs.firecrawl.dev", "https://docs.firecrawl.dev/introduction"],
    "maxConcurrency": 10,
    "onlyMainContent": True,
    "includeTags": ["<string>"],
    "excludeTags": ["<string>"],
    "maxAge": 15768000000,
    "minAge": 15768000000,
    "timeout": 150500
}
headers = {
    "Idempotency-Key": "<idempotency-key>",
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {
    'Idempotency-Key': '<idempotency-key>',
    Authorization: 'Bearer <token>',
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
    urls: ['https://docs.firecrawl.dev', 'https://docs.firecrawl.dev/introduction'],
    maxConcurrency: 10,
    onlyMainContent: true,
    includeTags: ['<string>'],
    excludeTags: ['<string>'],
    maxAge: 15768000000,
    minAge: 15768000000,
    timeout: 150500
  })
};

fetch('https://api.aisa.one/apis/v1/firecrawl/batch-scrape', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.aisa.one/apis/v1/firecrawl/batch-scrape",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'urls' => [
        'https://docs.firecrawl.dev',
        'https://docs.firecrawl.dev/introduction'
    ],
    'maxConcurrency' => 10,
    'onlyMainContent' => true,
    'includeTags' => [
        '<string>'
    ],
    'excludeTags' => [
        '<string>'
    ],
    'maxAge' => 15768000000,
    'minAge' => 15768000000,
    'timeout' => 150500
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json",
    "Idempotency-Key: <idempotency-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.aisa.one/apis/v1/firecrawl/batch-scrape"

	payload := strings.NewReader("{\n  \"urls\": [\n    \"https://docs.firecrawl.dev\",\n    \"https://docs.firecrawl.dev/introduction\"\n  ],\n  \"maxConcurrency\": 10,\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"maxAge\": 15768000000,\n  \"minAge\": 15768000000,\n  \"timeout\": 150500\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Idempotency-Key", "<idempotency-key>")
	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.aisa.one/apis/v1/firecrawl/batch-scrape")
  .header("Idempotency-Key", "<idempotency-key>")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"urls\": [\n    \"https://docs.firecrawl.dev\",\n    \"https://docs.firecrawl.dev/introduction\"\n  ],\n  \"maxConcurrency\": 10,\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"maxAge\": 15768000000,\n  \"minAge\": 15768000000,\n  \"timeout\": 150500\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.aisa.one/apis/v1/firecrawl/batch-scrape")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Idempotency-Key"] = '<idempotency-key>'
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"urls\": [\n    \"https://docs.firecrawl.dev\",\n    \"https://docs.firecrawl.dev/introduction\"\n  ],\n  \"maxConcurrency\": 10,\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"maxAge\": 15768000000,\n  \"minAge\": 15768000000,\n  \"timeout\": 150500\n}"

response = http.request(request)
puts response.read_body

{
  "id": "iaj_01HZY8Q2M4K7N9V3T6W1X0B2C3",
  "object": "integration_async_job",
  "endpoint": "/apis/v1/firecrawl/crawl",
  "createdAt": "2023-11-07T05:31:56Z",
  "completedAt": "2023-11-07T05:31:56Z",
  "pricing": {
    "currency": "USD",
    "authorizedMicrosUSD": 123,
    "finalMicrosUSD": 123,
    "billingMode": "metered_result"
  },
  "output": "<unknown>",
  "outputExpired": true,
  "error": {
    "code": "<string>",
    "message": "<string>"
  }
}

Scrape many URLs in a single asynchronous job. Provide a list of URLs; the batch runs in the background and you poll it for status and results. Lifecycle

Submit — POST /apis/v1/firecrawl/batch-scrape with the batch body and a required Idempotency-Key header. On success you get back 202 Accepted, a Location header pointing at the job resource (/apis/v1/firecrawl/batch-scrape/{jobId}), and a job object whose status starts as queued.
Poll — GET /apis/v1/firecrawl/batch-scrape/{jobId} and repeat until status is terminal: completed, failed, or cancelled. When completed, the scraped documents are in output.
List — GET /apis/v1/firecrawl/batch-scrape (optional limit, status, cursor query params).
Cancel — POST /apis/v1/firecrawl/batch-scrape/{jobId}/cancel.

Every URL must be a unique HTTPS URL and must not be a PDF; you can submit 1 to 1000 URLs per job. Billing is metered per Firecrawl credit — 1 credit per page scraped — so total cost scales with the pages actually processed. To follow links from a single root instead of listing URLs, use post_firecrawl-crawl.

Authorizations

Authorization

string

header

required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Headers

Idempotency-Key

string

required

Unique key (1 to 191 characters) that makes the submit idempotent. Re-submitting with the same key returns the original job.

Maximum string length: 191

Body

application/json

urls

string<uri>[]

required

1 to 1000 unique HTTPS URLs to scrape. PDF URLs are not supported.

Required array length: 1 - 1000 elements

Example:

[
  "https://docs.firecrawl.dev",
  "https://docs.firecrawl.dev/introduction"
]

maxConcurrency

integer

Maximum number of concurrent scrapes (1 to 20).

Required range: 1 <= x <= 20

onlyMainContent

boolean

default:true

Return only the main content of each page.

includeTags

string[]

HTML tags/selectors to keep.

Maximum array length: 50

Maximum string length: 128

excludeTags

string[]

HTML tags/selectors to drop.

Maximum array length: 50

Maximum string length: 128

maxAge

integer

Maximum acceptable cache age in milliseconds.

Required range: 0 <= x <= 31536000000

minAge

integer

Minimum cache age in milliseconds before a page is refetched.

Required range: 0 <= x <= 31536000000

timeout

integer

Per-page timeout in milliseconds.

Required range: 1000 <= x <= 300000

Response

202 - application/json

Batch scrape job accepted. The Location header points at the job resource; poll it until terminal.

An asynchronous integration job. Returned by the submit call (HTTP 202) and by the poll/detail call. Poll the job by its id until status is a terminal value (completed, failed, or cancelled).

string

Unique AIsa job identifier. Use it to poll, list, or cancel the job.

Example:

"iaj_01HZY8Q2M4K7N9V3T6W1X0B2C3"

object

string

Always "integration_async_job".

Example:

"integration_async_job"

endpoint

string

The submit endpoint this job belongs to.

Example:

"/apis/v1/firecrawl/crawl"

status

enum<string>

Customer-facing lifecycle status. queued and running are non-terminal; completed, failed, and cancelled are terminal.

Available options:

queued,

running,

completed,

failed,

cancelled

createdAt

string<date-time>

When the job was accepted.

completedAt

string<date-time> | null

When the job reached a terminal status. Null while the job is still queued or running.

pricing

object

Show child attributes

output

any | null

Job result payload. Present only once status is completed. For crawl this is the array of scraped pages; for batch scrape it is the array of scraped documents.

outputExpired

boolean

True when the result has been retained past its retention window and is no longer retrievable.

error

object | null

Present when status is failed. Null otherwise.

Show child attributes

Overview

Chat API

Search API

Perplexity API

Financial API

Twitter API

Reddit API

Instagram API

Pinterest API

Scholar API

WaveInflu API

Prediction Market API

CoinGecko

DataForSEO

AgentMail

Apollo

Essentials

Firecrawl Batch Scrape

Authorizations

Headers

Body

Response