Queue a deep web scrape job
Queue an async browser scrape for dynamic or long pages and poll the returned operation URL for results.
POST
/
api
/
v1
/
platform
/
scrapers
/
web
/
scrape
/
jobs
Queue a deep web scrape job
curl --request POST \
--url https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/jobs \
--header 'Content-Type: application/json' \
--header 'X-Access-Key: <api-key>' \
--header 'X-Secret-Key: <api-key>' \
--data '
{
"url": "https://example.com/page",
"formats": [
"text",
"metadata"
],
"jsonSchema": {},
"instructions": "<string>",
"maxAgeMs": 0,
"maxDurationMs": 120000,
"maxScrolls": 40,
"scrollWaitMs": 500,
"contentStableRounds": 3,
"expandClickableContent": true,
"maxExpansionClicks": 25
}
'const options = {
method: 'POST',
headers: {
'X-Access-Key': '<api-key>',
'X-Secret-Key': '<api-key>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
url: 'https://example.com/page',
formats: ['text', 'metadata'],
jsonSchema: {},
instructions: '<string>',
maxAgeMs: 0,
maxDurationMs: 120000,
maxScrolls: 40,
scrollWaitMs: 500,
contentStableRounds: 3,
expandClickableContent: true,
maxExpansionClicks: 25
})
};
fetch('https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/jobs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));import requests
url = "https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/jobs"
payload = {
"url": "https://example.com/page",
"formats": ["text", "metadata"],
"jsonSchema": {},
"instructions": "<string>",
"maxAgeMs": 0,
"maxDurationMs": 120000,
"maxScrolls": 40,
"scrollWaitMs": 500,
"contentStableRounds": 3,
"expandClickableContent": True,
"maxExpansionClicks": 25
}
headers = {
"X-Access-Key": "<api-key>",
"X-Secret-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text){
"data": {
"id": "<string>",
"operationId": "<string>",
"status": "queued",
"pollUrl": "<string>"
},
"meta": {
"requestId": "<string>"
}
}{
"statusCode": 400,
"error": "Bad Request",
"message": "Validation failed",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b",
"errors": [
{
"property": "body.query",
"message": "query must be a string",
"constraints": {
"isString": "query must be a string"
}
}
]
}{
"statusCode": 401,
"error": "Unauthorized",
"message": "Authentication is required.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 402,
"error": "Payment Required",
"message": "Insufficient credits.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 500,
"error": "Internal Server Error",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 503,
"error": "Service Unavailable",
"message": "Service temporarily unavailable.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}POST /api/v1/platform/scrapers/web/scrape/jobs
Queue a deep web scrape for long or dynamic pages and poll for the result.Use deep scrape when a page needs more rendering time than the synchronous scrape endpoint should spend, such as long comment threads, lazy-loaded content, or pages with repeated “load more” controls. The endpoint returns an operation ID immediately; poll
GET /api/operations/:id until the operation reaches a
terminal status.
The result uses the same requested formats as single-page scrape. Metadata may
include capture details such as how many scrolls were completed and why capture
stopped. A deep scrape is still bounded by the limits you send, so use the
capture metadata to decide whether to run again with higher limits.
Example
curl -X POST https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/jobs \
-H "X-Access-Key: ak_xxxxxxxxxxxxxxxx" \
-H "X-Secret-Key: sk_xxxxxxxxxxxxxxxx" \
-H "Content-Type: application/json" \
-H "Idempotency-Key: deep-scrape-2026-06-24-001" \
-d '{
"url": "https://example.com/thread",
"formats": ["markdown", "metadata"],
"maxDurationMs": 120000,
"maxScrolls": 40,
"contentStableRounds": 3,
"expandClickableContent": true
}'
formats: ["json"] with jsonSchema when you want schema-guided
extraction from the captured page content. The schema defines the returned keys
and shape.Authorizations
Headers
Body
application/json
⌘I
Queue a deep web scrape job
curl --request POST \
--url https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/jobs \
--header 'Content-Type: application/json' \
--header 'X-Access-Key: <api-key>' \
--header 'X-Secret-Key: <api-key>' \
--data '
{
"url": "https://example.com/page",
"formats": [
"text",
"metadata"
],
"jsonSchema": {},
"instructions": "<string>",
"maxAgeMs": 0,
"maxDurationMs": 120000,
"maxScrolls": 40,
"scrollWaitMs": 500,
"contentStableRounds": 3,
"expandClickableContent": true,
"maxExpansionClicks": 25
}
'const options = {
method: 'POST',
headers: {
'X-Access-Key': '<api-key>',
'X-Secret-Key': '<api-key>',
'Content-Type': 'application/json'
},
body: JSON.stringify({
url: 'https://example.com/page',
formats: ['text', 'metadata'],
jsonSchema: {},
instructions: '<string>',
maxAgeMs: 0,
maxDurationMs: 120000,
maxScrolls: 40,
scrollWaitMs: 500,
contentStableRounds: 3,
expandClickableContent: true,
maxExpansionClicks: 25
})
};
fetch('https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/jobs', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));import requests
url = "https://developer.thehog.ai/api/v1/platform/scrapers/web/scrape/jobs"
payload = {
"url": "https://example.com/page",
"formats": ["text", "metadata"],
"jsonSchema": {},
"instructions": "<string>",
"maxAgeMs": 0,
"maxDurationMs": 120000,
"maxScrolls": 40,
"scrollWaitMs": 500,
"contentStableRounds": 3,
"expandClickableContent": True,
"maxExpansionClicks": 25
}
headers = {
"X-Access-Key": "<api-key>",
"X-Secret-Key": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text){
"data": {
"id": "<string>",
"operationId": "<string>",
"status": "queued",
"pollUrl": "<string>"
},
"meta": {
"requestId": "<string>"
}
}{
"statusCode": 400,
"error": "Bad Request",
"message": "Validation failed",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b",
"errors": [
{
"property": "body.query",
"message": "query must be a string",
"constraints": {
"isString": "query must be a string"
}
}
]
}{
"statusCode": 401,
"error": "Unauthorized",
"message": "Authentication is required.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 402,
"error": "Payment Required",
"message": "Insufficient credits.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 500,
"error": "Internal Server Error",
"message": "An unexpected error occurred.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}{
"statusCode": 503,
"error": "Service Unavailable",
"message": "Service temporarily unavailable.",
"path": "/api/v1/search",
"timestamp": "2026-05-21T09:08:10.000Z",
"requestId": "506af9b3-01a9-43be-9eb4-8458fe3e4f5b"
}