{"schema_version":"1.0","documentation_url":"https://taskfuel.ai/catalog/manifest/","source":{"type":"provider_specifications","date":"2026-10-15","catalog_url":"https://taskfuel.ai/catalog/"},"capability":{"id":"crawl-site","name":"Crawl a website","description":"Follow links across a site and return every page’s content.","category":{"id":"web","name":"Websites & scraping"},"url":"https://taskfuel.ai/catalog/#cap-crawl-site","guide_url":null,"related_guide_urls":[],"detail_url":"https://taskfuel.ai/catalog/capabilities/crawl-site.json","providers":[{"name":"Bytemine","domains":["mpp.orthogonal.com/bytemine"]},{"name":"Olostep API","domains":["mpp.orthogonal.com/olostep"]},{"name":"Tavily","domains":["tavily.com","tavily.mpp.paywithlocus.com"]},{"name":"2s","domains":["2s.io"]},{"name":"AnyAPI","domains":["api.getanyapi.com"]},{"name":"Context.dev","domains":["mpp.orthogonal.com/context-dev"]},{"name":"ScrapeGraphAI","domains":["mpp.orthogonal.com/scrapegraphai"]},{"name":"Search1API","domains":["search1api.com"]},{"name":"StableEnrich","domains":["stableenrich.dev"]},{"name":"Tavily API","domains":["mpp.orthogonal.com/tavily"]}],"price":{"type":"from","amount":0.03,"currency":"USD","unit":null,"basis":"lowest_positive_declared_endpoint_amount","quote_required":true},"example_prompt":"Use TaskFuel to crawl the docs at <URL> and collect every page about authentication.","endpoint_count":13},"endpoints":[{"provider":{"name":"Context.dev","domain":"mpp.orthogonal.com/context-dev"},"method":"POST","url":"https://mpp.orthogonal.com/context-dev/web/crawl","name":"Performs a crawl starting from a given URL, extracts page content as Markdown, and returns results for all crawled…","description":"","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"fixed","amount":0.03,"quote_required":false}},{"provider":{"name":"Search1API","domain":"search1api.com"},"method":"POST","url":"https://api.search1api.com/deepcrawl","name":"Deep crawl a website across multiple pages","description":"Start an asynchronous crawl of an entire site and package the pages as documents. Use it for whole-site ingestion; a single page is POST /crawl. The call returns a task id immediately rather than the result — poll GET /deepcrawl/status/{taskId} until the task reports…","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"fixed","amount":0.06,"quote_required":false}},{"provider":{"name":"StableEnrich","domain":"stableenrich.dev"},"method":"POST","url":"https://stableenrich.dev/api/cloudflare/crawl","name":"Cloudflare Browser Rendering Crawl - Start a crawl job and return a token to poll for results. Respects robots.txt and…","description":"","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"fixed","amount":0.1,"quote_required":false}},{"provider":{"name":"2s","domain":"2s.io"},"method":"POST","url":"https://2s.io/api/search/crawl","name":"Crawl a site and return clean page content.","description":"Crawl a site and return clean page content. POST { url, limit?, maxDepth?, instructions? }. Follows links from the start URL (up to 10 pages, depth ≤2) and returns each page's extracted content. Optional natural-language instructions steer which pages to follow (e.g. \"only…","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"fixed","amount":0.12,"quote_required":false}},{"provider":{"name":"AnyAPI","domain":"api.getanyapi.com"},"method":"POST","url":"https://api.getanyapi.com/v1/run/web.crawl","name":"Website Crawl","description":"Crawl a website and get up to 10 of its pages as separate items, each with its URL, title, description, language, HTTP status, clean Markdown text and page metadata such as headings and structured data, ideal for feeding sites into LLMs and search indexes. Pages that failed or…","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"quote_required","amount":null,"quote_required":true}},{"provider":{"name":"Bytemine","domain":"mpp.orthogonal.com/bytemine"},"method":"POST","url":"https://mpp.orthogonal.com/bytemine/crawl/start","name":"Start a multi-page crawl job with sitemap discovery and configurable depth/page limits. Returns job_id. 1 credit per…","description":"","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"quote_required","amount":null,"quote_required":true}},{"provider":{"name":"Bytemine","domain":"mpp.orthogonal.com/bytemine"},"method":"POST","url":"https://mpp.orthogonal.com/bytemine/crawl/batch","name":"Scrape up to 100 URLs in parallel. Returns markdown, metadata, and tech stack per URL. 1 credit per URL.","description":"","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"quote_required","amount":null,"quote_required":true}},{"provider":{"name":"Olostep API","domain":"mpp.orthogonal.com/olostep"},"method":"POST","url":"https://mpp.orthogonal.com/olostep/v1/crawls","name":"Starts a new crawl. You receive a `id` to track the progress. The operation may take 1-10 mins depending upon the site…","description":"","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"quote_required","amount":null,"quote_required":true}},{"provider":{"name":"Olostep API","domain":"mpp.orthogonal.com/olostep"},"method":"POST","url":"https://mpp.orthogonal.com/olostep/v1/batches","name":"Starts a new batch. You receive an `id` that you can use to track the progress of the batch as shown…","description":"","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"quote_required","amount":null,"quote_required":true}},{"provider":{"name":"ScrapeGraphAI","domain":"mpp.orthogonal.com/scrapegraphai"},"method":"POST","url":"https://mpp.orthogonal.com/scrapegraphai/api/crawl","name":"Start a website crawl from a seed URL with configurable depth and page limits.","description":"","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"quote_required","amount":null,"quote_required":true}},{"provider":{"name":"Tavily","domain":"tavily.com"},"method":"POST","url":"https://api.tavily.com/crawl","name":"Initiate a web crawl from a base URL","description":"Tavily Crawl is a graph-based website traversal tool that can explore hundreds of paths in parallel with built-in extraction and intelligent discovery.","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"unlisted","amount":null,"quote_required":true}},{"provider":{"name":"Tavily","domain":"tavily.mpp.paywithlocus.com"},"method":"POST","url":"https://tavily.mpp.paywithlocus.com/tavily/crawl","name":"Crawl","description":"Crawl a website and extract content from discovered pages. Combines mapping and extraction in a single call.","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"quote_required","amount":null,"quote_required":true}},{"provider":{"name":"Tavily API","domain":"mpp.orthogonal.com/tavily"},"method":"POST","url":"https://mpp.orthogonal.com/tavily/crawl","name":"Tavily Crawl is a graph-based website traversal tool that can explore hundreds of paths in parallel with built-in…","description":"","supporting_call":false,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"quote_required","amount":null,"quote_required":true}},{"provider":{"name":"Bytemine","domain":"mpp.orthogonal.com/bytemine"},"method":"POST","url":"https://mpp.orthogonal.com/bytemine/crawl/continue","name":"Resume processing pending pages in an existing crawl job. Call repeatedly until status is 'completed'.","description":"","supporting_call":true,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"fixed","amount":0.03,"quote_required":false}},{"provider":{"name":"Bytemine","domain":"mpp.orthogonal.com/bytemine"},"method":"POST","url":"https://mpp.orthogonal.com/bytemine/crawl/results","name":"Retrieve paginated results from a completed crawl. Each result includes URL, markdown, metadata, and links. Free, no…","description":"","supporting_call":true,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"free","amount":0,"quote_required":false}},{"provider":{"name":"Bytemine","domain":"mpp.orthogonal.com/bytemine"},"method":"POST","url":"https://mpp.orthogonal.com/bytemine/crawl/status","name":"Check crawl job status including pages crawled, found, and completion. Free, no credits.","description":"","supporting_call":true,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"free","amount":0,"quote_required":false}},{"provider":{"name":"Olostep API","domain":"mpp.orthogonal.com/olostep"},"method":"GET","url":"https://mpp.orthogonal.com/olostep/v1/retrieve","name":"Retrieve page content of processed batches and crawls urls.","description":"","supporting_call":true,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"unlisted","amount":null,"quote_required":true}},{"provider":{"name":"Olostep API","domain":"mpp.orthogonal.com/olostep"},"method":"GET","url":"https://mpp.orthogonal.com/olostep/v1/batches/{batch_id}","name":"Retrieves the status and progress information about a batch. To retrieve the content for a batch, see here","description":"","supporting_call":true,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"unlisted","amount":null,"quote_required":true}},{"provider":{"name":"Olostep API","domain":"mpp.orthogonal.com/olostep"},"method":"GET","url":"https://mpp.orthogonal.com/olostep/v1/crawls/{crawl_id}/pages","name":"Fetches the list of pages for a specific crawl.","description":"","supporting_call":true,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"unlisted","amount":null,"quote_required":true}},{"provider":{"name":"Olostep API","domain":"mpp.orthogonal.com/olostep"},"method":"GET","url":"https://mpp.orthogonal.com/olostep/v1/batches/{batch_id}/items","name":"Retrieves the list of items processed for a batch. You can then use the `retrieve_id` to get the content with the…","description":"","supporting_call":true,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"unlisted","amount":null,"quote_required":true}},{"provider":{"name":"Olostep API","domain":"mpp.orthogonal.com/olostep"},"method":"GET","url":"https://mpp.orthogonal.com/olostep/v1/crawls/{crawl_id}","name":"Fetches information about a specific crawl.","description":"","supporting_call":true,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"unlisted","amount":null,"quote_required":true}},{"provider":{"name":"Search1API","domain":"search1api.com"},"method":"GET","url":"https://api.search1api.com/deepcrawl/status/{taskId}","name":"Check deepcrawl task status","description":"Poll a deepcrawl task started by POST /deepcrawl. Returns the task's current state and, once it finishes, where to retrieve the packaged result. Safe to call repeatedly. Free to call.","supporting_call":true,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"unlisted","amount":null,"quote_required":true}},{"provider":{"name":"StableEnrich","domain":"stableenrich.dev"},"method":"GET","url":"https://stableenrich.dev/api/cloudflare/jobs","name":"Poll Cloudflare crawl job status and retrieve results. Requires SIWX wallet auth — only the wallet that paid can poll.","description":"","supporting_call":true,"price":{"currency":"USD","unit":null,"basis":"provider_specification","type":"quote_required","amount":null,"quote_required":true}}]}