> ## Documentation Index
> Fetch the complete documentation index at: https://docs.convocore.ai/llms.txt
> Use this file to discover all available pages before exploring further.
# Create Crawler Job
> Creates a new crawler or scrape job using the V3 API.
## Overview
Creates a new crawler job for a workspace and starts processing it in the background.
Crawler jobs are immutable after creation. If you need different settings, delete the job and create a new one.
## Supports
* Single-page scrape jobs
* Multi-URL scrape jobs
* Crawl jobs with `crawlOptions`
* Optional outbound webhooks for `page_scraped`, `job_completed`, and `job_failed`
## Billing
Credits are estimated at submission time and actually consumed per successful scraped page.
Use `useProxy: true` only when needed. Proxy scraping has higher per-page credit cost.
## OpenAPI
````yaml POST /workspaces/{workspaceId}/crawler/jobs
openapi: 3.0.3
info:
title: Convocore OpenAPI
description: Full API reference for Convocore
version: 1.0.4
servers:
- url: https://eu-gcp-api.vg-stuff.com/v3
description: EU Node.js API server
- url: https://na-gcp-api.vg-stuff.com/v3
description: NA Node.js API server
security: []
paths:
/workspaces/{workspaceId}/crawler/jobs:
post:
tags:
- Crawler
summary: Create crawler job
description: >-
Creates a new scrape or crawl job for the authenticated workspace.
Crawler jobs are immutable after submission. Updating a crawler job is
not supported; create a new job or delete the existing job instead.
operationId: createCrawlerJob
parameters:
- name: workspaceId
in: path
required: true
schema:
type: string
description: The workspace that owns the crawler job.
example: 360c48fb56eeaaaa322973c18
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
urls:
type: array
items:
type: string
format: uri
minItems: 1
description: >-
One or more source URLs to scrape or use as crawl entry
points.
crawl:
type: boolean
description: >-
If true, discovered URLs can be followed and scraped as part
of the same job.
crawlOptions:
type: object
properties:
maxPages:
type: integer
minimum: 1
maximum: 500
description: >-
Maximum number of pages the crawler is allowed to
process.
urlMatchers:
type: array
items:
type: string
description: URL path matchers that define which pages are in scope.
unMatchers:
type: array
items:
type: string
description: URL path matchers that should be excluded.
stayOnDomain:
type: boolean
description: If true, only pages on the same domain are crawled.
additionalProperties: false
deep:
type: boolean
description: If true, use deep scraping behavior.
useProxy:
type: boolean
description: >-
If true, the crawler uses proxy scraping and paid proxy
pricing.
refreshRate:
type: string
description: Optional refresh cadence for KB-linked scrapes.
toAgentId:
type: string
description: Optional single agent destination for KB import.
toAgentIds:
type: array
items:
type: string
description: Optional list of agent destinations for KB import.
webhook:
type: object
properties:
url:
type: string
format: uri
description: >-
The HTTPS endpoint that should receive crawler
callbacks.
events:
type: array
items:
type: string
enum:
- page_scraped
- job_completed
- job_failed
minItems: 1
description: >-
Which crawler events should be delivered to your
webhook. Defaults to page and final events.
secret:
type: string
description: >-
Optional secret mirrored back as the
`x-convocore-crawler-secret` header.
bearerToken:
type: string
description: Optional bearer token sent as the Authorization header.
headers:
type: object
additionalProperties:
type: string
description: >-
Optional additional headers to send with each webhook
request.
required:
- url
additionalProperties: false
description: >-
Optional outbound webhook that receives `page_scraped`,
`job_completed`, and `job_failed` events.
required:
- urls
additionalProperties: false
example:
urls:
- https://example.com
crawl: true
crawlOptions:
maxPages: 25
urlMatchers:
- /
stayOnDomain: true
webhook:
url: https://example.com/webhooks/crawler
events:
- page_scraped
- job_completed
- job_failed
responses:
'200':
description: Successful response
content:
application/json:
schema:
type: object
properties:
success:
type: boolean
message:
type: string
data:
type: object
properties:
id:
type: string
workspaceId:
type: string
status:
type: string
enum:
- queued
- processing
- completed
- failed
- cancelled
primaryUrl:
type: string
urls:
type: array
items:
type: string
crawl:
type: boolean
crawlOptions:
type: object
properties:
maxPages:
type: integer
minimum: 1
maximum: 500
description: >-
Maximum number of pages the crawler is allowed to
process.
urlMatchers:
type: array
items:
type: string
description: >-
URL path matchers that define which pages are in
scope.
unMatchers:
type: array
items:
type: string
description: URL path matchers that should be excluded.
stayOnDomain:
type: boolean
description: >-
If true, only pages on the same domain are
crawled.
additionalProperties: false
nullable: true
useProxy:
type: boolean
deep:
type: boolean
refreshRate:
type: string
nullable: true
toAgentId:
type: string
nullable: true
toAgentIds:
type: array
items:
type: string
done:
type: boolean
failed:
type: boolean
isCancelled:
type: boolean
message:
type: string
nullable: true
resultError:
type: string
nullable: true
createdAt:
type: string
nullable: true
ts:
type: number
currentPageIndex:
type: number
scrapedPagesNum:
type: number
failedPagesNum:
type: number
pageLimit:
type: number
creditsPerPage:
type: number
estimatedCredits:
type: number
activeScrapeUrl:
type: string
nullable: true
crawlerJobId:
type: string
nullable: true
webhook:
type: object
properties:
url:
type: string
format: uri
events:
type: array
items:
type: string
enum:
- page_scraped
- job_completed
- job_failed
hasSecret:
type: boolean
hasBearerToken:
type: boolean
headerKeys:
type: array
items:
type: string
required:
- url
- events
- hasSecret
- hasBearerToken
- headerKeys
additionalProperties: false
required:
- id
- workspaceId
- status
- primaryUrl
- urls
- crawl
- crawlOptions
- useProxy
- deep
- refreshRate
- toAgentId
- toAgentIds
- done
- failed
- isCancelled
- message
- resultError
- createdAt
- ts
- currentPageIndex
- scrapedPagesNum
- failedPagesNum
- pageLimit
- creditsPerPage
- estimatedCredits
- activeScrapeUrl
- crawlerJobId
additionalProperties: false
required:
- success
- message
- data
additionalProperties: false
default:
$ref: '#/components/responses/error'
security:
- Authorization: []
components:
responses:
error:
description: Error response
content:
application/json:
schema:
type: object
properties:
message:
type: string
code:
type: string
issues:
type: array
items:
type: object
properties:
message:
type: string
required:
- message
additionalProperties: false
required:
- message
- code
additionalProperties: false
securitySchemes:
Authorization:
type: http
scheme: bearer
````